feat(ec2): seal account settings with revision-guarded writes (#1241)

Replace the unwired delete-based CleanupAccount with an owner-local sealed
state in the existing account-settings KV value. All settings mutations now
use revision-guarded create/update; a sealed record rejects every read and
mutation with Blocked and cannot be recreated. Records without a state
field remain active.

SealAccount CAS-writes the sealed value whether the key is present, absent
or tombstoned, and succeeds only on a leader-accepted write, returning its
revision. It first checks, through a read-only IAM account-status reader,
that the account is already TERMINATING; otherwise it refuses with
IncorrectState and writes nothing.

The daemon serves the internal subject ec2.v1.SealAccountSettings. Nothing
calls it yet.

.github/workflows/actions.lock

@@ -73,7 +73,7 @@ workflows:
     '.github/workflows/ui.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-node@v7.1.0'
-        - 'pnpm/action-setup@v6.0.10'
+        - 'pnpm/action-setup@v6.1.0'
 dependencies:
     'actions/cache@v6.1.0':
         ref: 'v6.1.0'
@@ -125,9 +125,9 @@ dependencies:
         commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
         owner_id: 761456
         repo_id: 258271226
-    'pnpm/action-setup@v6.0.10':
-        ref: 'v6.0.10'
-        commit: 'sha1-0977fd99725f1db4007ccb2928dbb4e90d06cc86'
+    'pnpm/action-setup@v6.1.0':
+        ref: 'v6.1.0'
+        commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
         owner_id: 21320719
         repo_id: 262289747
     'softprops/action-gh-release@v3.0.3':

.github/workflows/ui.yml

@@ -35,7 +35,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.0.10
+        uses: pnpm/action-setup@v6.1.0
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 

Makefile

@@ -379,9 +379,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.2 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -412,7 +412,7 @@ fix:
 
 govulncheck:
 	@echo "Running govulncheck..."
-	$(_Q)scripts/run-gate.sh govulncheck scripts/govulncheck-gate.sh
+	$(_Q)scripts/run-gate.sh govulncheck go tool govulncheck ./...
 	@echo "  govulncheck ok"
 
 # NilAway — advisory nil-panic analysis. Not in preflight due to false positives

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27.2-bookworm@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
+FROM golang:1.27.3-bookworm@sha256:5c3e9b43cfd3bb95c7937462e29d87ada57b481a96fa23ad0eb61778d7a00110 AS builder
 
 ARG VERSION
 

contracts/ec2/v1/README.md

@@ -40,7 +40,7 @@ The request (`account_id`) and reply (`vpc_id`, `error`) bodies are still declar
 ## Platform inventory
 
 `PlatformInventorySubject` (`ec2.v1.PlatformInventory`) is a request/reply route on which a daemon in the `spinifex-workers` queue group reports one account's key pairs, placement groups, launch templates and their `ec2-tags` entries from the EC2 owner stores.
-EC2 owns it: `domains/ec2/accountinventory` produces the reply and the daemon serves it; the only consumer today is `accountteardown`, which binds an adapter but does not call it.
+EC2 owns it: `domains/ec2/accountinventory` produces the reply and the daemon serves it; the only consumer today is `accountteardown`, which calls it only on an explicit operator dry run (`spx admin account delete --dry-run` or the admin API `DryRun`) and only logs a comparison with its legacy listings; no teardown calls it.
 
 The request names the account in its body (`account_id`, a 12-digit ID) and may carry an opaque `correlation_ref` that is echoed for logs.
 The daemon ignores `X-Account-ID` for this subject.
@@ -60,3 +60,25 @@ The subject name carries the contract major; `platform_inventory_test.go` preser
 Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
 A daemon that predates the subject has no subscription, so the request gets no responder; the consumer treats no responder, a timeout, an error reply, an undecodable reply, a reply for another account, and a missing, repeated or unknown-status kind as `scan-incomplete`, never clean.
 An incompatible change to the subject, fields or vocabulary needs a new major.
+
+## Seal account settings
+
+`SealAccountSettingsSubject` (`ec2.v1.SealAccountSettings`) is a request/reply route on which a daemon in the `spinifex-workers` queue group irreversibly seals one account's EC2 settings record.
+EC2 owns it: `domains/ec2/account` serves it through the daemon.
+`accountteardown` is the intended consumer from a later release; no teardown calls it yet.
+
+The request names the account in its body (`account_id`) and may carry an opaque `correlation_ref` that is echoed for logs.
+The daemon ignores `X-Account-ID` for this subject: the account sealed is the one named in the body.
+A seal is refused with `IncorrectState` unless IAM already holds the account as `TERMINATING`; a precondition read failure answers `InternalError`.
+Both refusals write nothing.
+
+A reply is sent only on success: `disposition` is always `sealed` and `revision` is the stream leader's accepted write for that seal.
+A failure is the standard awserrors error payload, not this reply shape.
+Sealing is idempotent: a seal on an already-sealed record re-writes the same value and returns a fresh, higher revision.
+
+### Compatibility boundary
+
+The subject name carries the contract major; `account_settings_seal_test.go` preserves the version-1 JSON shape.
+Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
+A daemon that predates the subject has no subscription, so the request gets no responder.
+An incompatible change to the subject or fields needs a new major.

contracts/ec2/v1/account_settings_seal.go

@@ -0,0 +1,28 @@
+package ec2v1
+
+// SealAccountSettingsSubject is the request/reply subject on which a daemon
+// seals one account's EC2 settings record. Sealing is irreversible and refused
+// unless IAM already holds the account as TERMINATING.
+const SealAccountSettingsSubject = "ec2.v1.SealAccountSettings"
+
+// AccountSettingsDispositionSealed is the only disposition a successful reply
+// carries: a reply is sent only on success, so there is no other value.
+const AccountSettingsDispositionSealed = "sealed"
+
+// SealAccountSettingsRequest asks for one account's EC2 settings to be
+// sealed. CorrelationRef is an opaque caller reference echoed for logs; it
+// grants nothing and is never a fence or authority to write.
+type SealAccountSettingsRequest struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// SealAccountSettingsReply confirms accountID's settings record was sealed at
+// Revision, the stream leader's accepted write for that seal. A failure is
+// the standard awserrors error payload, not this type.
+type SealAccountSettingsReply struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+	Disposition    string `json:"disposition"`
+	Revision       uint64 `json:"revision"`
+}

contracts/ec2/v1/account_settings_seal_test.go

@@ -0,0 +1,43 @@
+package ec2v1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSealAccountSettingsSubject(t *testing.T) {
+	require.Equal(t, "ec2.v1.SealAccountSettings", SealAccountSettingsSubject)
+}
+
+func TestSealAccountSettingsRequestJSONContract(t *testing.T) {
+	data, err := json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
+}
+
+func TestSealAccountSettingsReplyJSONContract(t *testing.T) {
+	reply := SealAccountSettingsReply{
+		AccountID:      "111122223333",
+		CorrelationRef: "req-1",
+		Disposition:    AccountSettingsDispositionSealed,
+		Revision:       42,
+	}
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	require.JSONEq(t, `{
+		"account_id": "111122223333",
+		"correlation_ref": "req-1",
+		"disposition": "sealed",
+		"revision": 42
+	}`, string(data))
+
+	data, err = json.Marshal(SealAccountSettingsReply{AccountID: "111122223333", Disposition: AccountSettingsDispositionSealed})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","disposition":"sealed","revision":0}`, string(data))
+}

docs/COMMANDS.md

@@ -109,7 +109,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Flags | Description |
 |---------|-------|-------------|
 | `spx admin kv digest` | `--store-dir` (default `<spinifex-dir>/nats/jetstream`), `--work-dir` (holds the temporary copy, default the system temp dir), `--stream` (repeatable filter), `--seqs` (also print a hash per sequence), `--json` (print one JSON document, the input `spx admin kv compare` reads) | Read-only per-replica consistency check. Copies this node's `$G` streams tree (without consumer state) to a temporary directory, opens the copy in a private in-process NATS server with no listener, and prints one line per stream with first/last sequence, message count and a SHA-256 over every stored message (sequence, subject, timestamp, headers, data). Run it on every node with `sudo` and diff the output with the `#` lines removed: replicas of one stream must match. A direct get over NATS is not used because any replica may answer it. Writes landing during the copy can change a replica's tail, so gate API writes or re-run before calling a tail difference divergence. Prefer `--json --seqs` on every node followed by `spx admin kv compare`, which tells a diverged replica from one that is merely behind. |
-| `spx admin kv compare <digest.json>...` | — (`-` reads any number of digests from stdin) | Reads the `spx admin kv digest --json --seqs` output of every node and checks that each stream's replicas hold the same content. A replica that is merely behind is not divergence; these are: the same sequence holding different content on two replicas, a message missing from one replica that nothing on it explains (no later write to the same subject, no expiry under `max_age` or a message TTL, no stream limit dropping its head), a stream held by fewer or more nodes than its replica count, replicas disagreeing on that count, or no sequence range common to every replica. Sequences past the slowest replica's last are not compared. Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input. Digests are taken from live stores seconds apart, so re-run before trusting a failure that a write landing between two copies could explain. `install-node.sh` gates cluster formation on it. |
+| `spx admin kv compare <digest.json>...` | — (`-` reads any number of digests from stdin) | Reads the `spx admin kv digest --json --seqs` output of every node and checks that each stream's replicas hold the same content. A replica that is merely behind is not divergence; these are: the same sequence holding different content on two replicas, a message missing from one replica that nothing on it explains (no later write to the same subject, no expiry under `max_age` or a message TTL, no stream limit dropping its head), a stream held by fewer or more nodes than its replica count, replicas disagreeing on that count, or no sequence range common to every replica. A stream with a `max_age` whose replicas were copied at different times and share no sequence range is reported as `skipped` (volatile) rather than diverged, since a short-lived bucket such as a leader lease turns over between copies; any sequence two of its replicas share must still match, and each message a replica lacks must be superseded, expired, or written after that replica was copied. Sequences past the slowest replica's last are not compared. Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input. Digests are taken from live stores seconds apart, so re-run before trusting a failure that a write landing between two copies could explain. `install-node.sh` gates cluster formation on it. |
 | `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
 | `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
 | `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s), `--unless-restarting` (drain on a real host shutdown or a plain target stop, and skip only when a restart of the stack is already queued; unset, it always drains) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -446,7 +446,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
 - `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
 - `gateway/requestaudit.go`: the outermost request-audit middleware records AWS fields and is updated from four places in the gateway's authentication flow through the gateway's own context key. Moving it would export about seven names used only by that flow, so it stays in `gateway` until the generic ingress registration seam defines how ingress middleware receives authentication results.
-- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; no engine path calls it yet, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters.
+- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet.
 - `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `handlers/rds` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
 - `domains/ec2/awsapi/instance` (dependency): imports `handlers/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
 - `domains/ec2/awsapi/spotinstance` (dependency): imports `handlers/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,7 +4,7 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
+FROM golang:1.27.3-alpine@sha256:f2c872bcee949b7be9c490efd02b48a8a2fbf947cdf533f1c58525ec9d8170fd AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.27.2
+go 1.27.3

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.27.2
+go 1.27.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8

govulncheck-accepted.txt

@@ -1,3 +0,0 @@
-# Reachable vulnerabilities accepted for a bounded time, read by scripts/govulncheck-gate.sh.
-# Format: <OSV-ID> <expiry YYYY-MM-DD> <reason>
-# The gate fails once the expiry passes: fix the dependency or renew deliberately.

scripts/build-microvm-image.sh

@@ -17,7 +17,7 @@ BUILD_DIR="$REPO_ROOT/build/microvm"
 INIT_SH="$BUILD_DIR/init.sh"
 INITTAB="$BUILD_DIR/inittab"
 
-ALPINE_VERSION="${ALPINE_VERSION:-3.20}"
+ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
 ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
 # haproxy: ALB (L7) data plane. nginx + nginx-mod-stream: NLB (L4) data plane —
 # the `stream` module (a separate Alpine package) load-balances TCP/UDP/TLS,

scripts/build-system-image.sh

@@ -284,6 +284,12 @@ if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
     CUST+=(--run-command 'sed -i "s|^#\(.*community\)|\1|" /etc/apk/repositories 2>/dev/null || true')
     CUST+=(--run-command 'grep -q community /etc/apk/repositories || grep main /etc/apk/repositories | head -1 | sed "s|/main|/community|" >> /etc/apk/repositories')
     CUST+=(--run-command 'apk update')
+    # The base cloud image's kernel is only as fresh as its release; upgrade so
+    # the AMI ships the branch's current linux-virt and security fixes.
+    CUST+=(--run-command 'apk upgrade')
+    # apk's mkinitfs trigger does not always fire in the appliance; a new kernel
+    # with the old initramfs cannot load virtio_blk and never mounts root.
+    CUST+=(--run-command 'k=$(ls /lib/modules | sort -V | tail -1) && mkinitfs "$k" && zcat /boot/initramfs-virt | cpio -t | grep -qE "^(usr/)?lib/modules/$k/.*/virtio_blk\.ko"')
     CUST+=(--run-command "apk add ${APK_PACKAGES}")
 elif [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APT_PACKAGES:-}" ]]; then
     echo "Will install packages: ${APT_PACKAGES}"

scripts/govulncheck-gate.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Fails on any reachable vulnerability that is not listed in
-# govulncheck-accepted.txt, and on any acceptance whose expiry has passed.
-# Each line is "<OSV-ID> <YYYY-MM-DD expiry> <reason>"; '#' starts a comment.
-# GOVULNCHECK_JSON substitutes a saved -format json report, for tests.
-set -euo pipefail
-
-accepted_file=${GOVULNCHECK_ACCEPTED:-govulncheck-accepted.txt}
-today=${GOVULNCHECK_TODAY:-$(date -u +%F)}
-
-if [ -n "${GOVULNCHECK_JSON:-}" ]; then
-	report=$(cat "$GOVULNCHECK_JSON")
-else
-	report=$(go tool govulncheck -format json ./...)
-fi
-
-# A finding whose first trace frame names a function is reachable from our
-# code; text mode reports only these, so module- and package-level ones pass.
-mapfile -t called < <(printf '%s\n' "$report" | jq -rs \
-	'[.[] | select(.finding) | .finding | select(.trace[0].function) | .osv] | unique | .[]')
-
-declare -A expiry reason
-if [ -f "$accepted_file" ]; then
-	while read -r id until why; do
-		[[ -z "$id" || "$id" == \#* ]] && continue
-		expiry[$id]=$until
-		reason[$id]=$why
-	done <"$accepted_file"
-fi
-
-fail=0
-for id in "${called[@]}"; do
-	if [ -z "${expiry[$id]:-}" ]; then
-		echo "  $id is reachable and not accepted; run 'go tool govulncheck ./...' for the trace"
-		fail=1
-	elif [[ "$today" > "${expiry[$id]}" ]]; then
-		echo "  $id acceptance expired on ${expiry[$id]}: ${reason[$id]}"
-		fail=1
-	else
-		echo "  $id accepted until ${expiry[$id]}: ${reason[$id]}"
-	fi
-done
-
-for id in "${!expiry[@]}"; do
-	if ! printf '%s\n' "${called[@]}" | grep -qx "$id"; then
-		echo "  $id is accepted but no longer reported; remove it from $accepted_file"
-	fi
-done
-
-exit "$fail"

scripts/govulncheck-gate_test.sh

@@ -1,53 +0,0 @@
-#!/bin/sh
-# Tests the accepted-vulnerability gate against saved reports, so the logic is
-# checked without running the scanner.
-set -u
-
-here=$(cd "$(dirname "$0")" && pwd)
-gate="$here/govulncheck-gate.sh"
-tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
-
-cat >"$tmp/accepted.txt" <<'EOF'
-# comment
-GO-2026-6597 2026-11-07 accepted for the test
-EOF
-cat >"$tmp/reachable.json" <<'EOF'
-{"config":{}}
-{"finding":{"osv":"GO-2026-6597","trace":[{"module":"m","package":"p","function":"Pull"}]}}
-{"finding":{"osv":"GO-2026-0001","trace":[{"module":"m"}]}}
-EOF
-cat >"$tmp/new.json" <<'EOF'
-{"finding":{"osv":"GO-2026-6597","trace":[{"module":"m","package":"p","function":"Pull"}]}}
-{"finding":{"osv":"GO-2026-7777","trace":[{"module":"m","package":"p","function":"F"}]}}
-EOF
-echo '{"config":{}}' >"$tmp/clean.json"
-
-fails=0
-check() {
-	want=$1 name=$2 report=$3 today=$4 accepted=$5 expect=$6
-	out=$(GOVULNCHECK_JSON="$tmp/$report" GOVULNCHECK_TODAY=$today GOVULNCHECK_ACCEPTED=$accepted bash "$gate" 2>&1)
-	rc=$?
-	if [ "$rc" != "$want" ] || ! printf '%s' "$out" | grep -q "$expect"; then
-		echo "FAIL [$name] rc=$rc want=$want output: $out"
-		fails=$((fails + 1))
-	else
-		echo "ok   [$name]"
-	fi
-}
-
-check 0 "accepted reachable finding passes" reachable.json 2026-10-07 "$tmp/accepted.txt" "GO-2026-6597 accepted until 2026-11-07"
-check 0 "expiry day itself still passes" reachable.json 2026-11-07 "$tmp/accepted.txt" "accepted until"
-check 1 "expired acceptance fails" reachable.json 2026-11-08 "$tmp/accepted.txt" "acceptance expired on 2026-11-07"
-check 1 "new reachable finding fails" new.json 2026-10-07 "$tmp/accepted.txt" "GO-2026-7777 is reachable and not accepted"
-out=$(GOVULNCHECK_JSON="$tmp/reachable.json" GOVULNCHECK_TODAY=2026-10-07 GOVULNCHECK_ACCEPTED="$tmp/accepted.txt" bash "$gate" 2>&1)
-if printf '%s' "$out" | grep -q "GO-2026-0001"; then
-	echo "FAIL [module-level finding is not reachable] output: $out"
-	fails=$((fails + 1))
-else
-	echo "ok   [module-level finding is not reachable]"
-fi
-check 0 "fixed finding warns that acceptance is stale" clean.json 2026-10-07 "$tmp/accepted.txt" "no longer reported"
-check 1 "missing acceptance file fails reachable finding" reachable.json 2026-10-07 "$tmp/none" "not accepted"
-
-[ "$fails" -eq 0 ] || exit 1

scripts/setup-ovn_test.sh

@@ -43,7 +43,13 @@ esac
 # Teardown, dry run. Every mutation must go through run(), so a dry run can
 # report the whole plan without performing any of it.
 # --------------------------------------------------------------------------
-out=$("$SETUP" --teardown --dry-run 2>&1) && rc=0 || rc=$?
+# The teardown plans the veth steps only when ovs-vsctl exists and the link is
+# present, so stub both or the order check depends on the machine running it.
+mkdir "$TMP/bin"
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/ip"
+cp "$TMP/bin/ip" "$TMP/bin/ovs-vsctl"
+chmod +x "$TMP/bin/ip" "$TMP/bin/ovs-vsctl"
+out=$(PATH="$TMP/bin:$PATH" "$SETUP" --teardown --dry-run 2>&1) && rc=0 || rc=$?
 [ "$rc" = 0 ] && pass "--teardown --dry-run exits 0" || fail "--teardown --dry-run exited $rc, want 0"
 
 case "$out" in

scripts/uninstall-spx_test.sh

@@ -81,9 +81,15 @@ mkcfg() { # mkcfg <dir> <node>...
 mkcfg "$TMP/three" node1 node2 node3
 mkcfg "$TMP/one" node1
 
+# The node count is read through sudo, so a pass-through stub keeps the refusal
+# from depending on whether the machine running this has sudo.
+mkdir "$TMP/bin"
+printf '#!/bin/sh\nexec "$@"\n' > "$TMP/bin/sudo"
+chmod +x "$TMP/bin/sudo"
+
 for args in "" "--yes" "--yes --dry-run"; do
     # shellcheck disable=SC2086
-    SPX_ETC_DIR="$TMP/three" "$UNINSTALL" $args >"$TMP/out" 2>"$TMP/err" && rc=0 || rc=$?
+    PATH="$TMP/bin:$PATH" SPX_ETC_DIR="$TMP/three" "$UNINSTALL" $args >"$TMP/out" 2>"$TMP/err" && rc=0 || rc=$?
     label=${args:-"(no flags)"}
     [ "$rc" = 1 ] \
         && pass "three-node config refuses with $label" \

spinifex/accountteardown/ec2_platform_inventory.go

@@ -12,8 +12,8 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ec2PlatformInventoryTimeout bounds one inventory request. The scan reads
-// four stores for one account; a reply later than this is scan-incomplete.
+// ec2PlatformInventoryTimeout bounds one inventory request; an earlier caller
+// deadline wins. A reply later than either is scan-incomplete.
 const ec2PlatformInventoryTimeout = 30 * time.Second
 
 // ec2PlatformInventory is EC2's read-only report of an account's key pairs,
@@ -39,8 +39,17 @@ func newEC2PlatformInventoryClient(nc *nats.Conn) *ec2PlatformInventoryClient {
 // responder, which is also how a daemon predating the subject answers, a
 // timeout, an error reply or a malformed reply each make every kind incomplete.
 func (c *ec2PlatformInventoryClient) PlatformInventory(ctx context.Context, accountID, correlationRef string) ec2v1.PlatformInventoryReply {
+	// The NATS request takes a timeout, not a context, so the caller's deadline
+	// shortens it here; a context already done is a timeout without a request.
+	timeout := c.timeout
+	if deadline, ok := ctx.Deadline(); ok {
+		timeout = min(timeout, time.Until(deadline))
+	}
+	if ctx.Err() != nil || timeout <= 0 {
+		return ec2v1.IncompletePlatformInventory(accountID, correlationRef, ec2v1.PlatformReasonTimeout)
+	}
 	reply, err := natsmsg.NATSRequest[ec2v1.PlatformInventoryReply](ctx, c.nc, ec2v1.PlatformInventorySubject,
-		&ec2v1.PlatformInventoryRequest{AccountID: accountID, CorrelationRef: correlationRef}, c.timeout, accountID)
+		&ec2v1.PlatformInventoryRequest{AccountID: accountID, CorrelationRef: correlationRef}, timeout, accountID)
 	if err != nil {
 		return ec2v1.IncompletePlatformInventory(accountID, correlationRef, inventoryFailureReason(err))
 	}

spinifex/accountteardown/ec2_platform_inventory_test.go

@@ -6,11 +6,14 @@ package accountteardown
 import (
 	"context"
 	"encoding/json"
+	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
 	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
@@ -67,9 +70,13 @@ func TestEC2PlatformInventoryClientReturnsEC2sReport(t *testing.T) {
 		KeyName: aws.String("deploy"), PublicKeyMaterial: []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"),
 	}, testAccountID)
 	require.NoError(t, err)
-	o.store.failDelete = func(key string) bool { return key == "keys/"+testAccountID+"/deploy" }
 	_, err = o.keys.DeleteKeyPair(ctx, &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, testAccountID)
 	require.NoError(t, err)
+	// A public key without metadata, as an older release's failed delete left.
+	_, err = o.store.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(platformBucket), Key: aws.String("keys/" + testAccountID + "/deploy"), Body: strings.NewReader("ssh-ed25519 AAAA"),
+	})
+	require.NoError(t, err)
 
 	reply := newEC2PlatformInventoryClient(o.nc).PlatformInventory(ctx, testAccountID, "req-1")
 
@@ -153,19 +160,10 @@ func TestEC2PlatformInventoryClientDistrustsAMalformedReply(t *testing.T) {
 	}, reply.Kinds)
 }
 
-// failingInventory fails the test if anything consults it.
-type failingInventory struct {
-	t *testing.T
-}
-
-func (f failingInventory) PlatformInventory(context.Context, string, string) ec2v1.PlatformInventoryReply {
-	f.t.Error("an engine path called the EC2 platform inventory")
-	return ec2v1.PlatformInventoryReply{}
-}
-
-// The cluster wiring binds the NATS adapter, and no engine path consults it:
-// precheck, inventory, dry run and a full teardown all leave it uncalled.
-func TestNoEnginePathCallsTheEC2PlatformInventory(t *testing.T) {
+// The cluster wiring binds the NATS adapter, and only the explicit dry run
+// consults it: precheck, the inventory the CLI prints before a real teardown,
+// a Teardown request marked dry run and a full teardown all leave it uncalled.
+func TestOnlyTheExplicitDryRunCallsTheEC2PlatformInventory(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	masterKey, err := handlers_iam.GenerateMasterKey()
 	require.NoError(t, err)
@@ -175,9 +173,10 @@ func TestNoEnginePathCallsTheEC2PlatformInventory(t *testing.T) {
 	require.NoError(t, err)
 	assert.IsType(t, &ec2PlatformInventoryClient{}, cluster.ec2Inventory)
 
+	inventory := &fakeInventory{reply: ec2v1.IncompletePlatformInventory(testAccountID, "req-1", ec2v1.PlatformReasonNoResponder)}
 	reaper := &fakeReaper{kind: "key-pair", stage: StagePlatform, resources: []Resource{{Kind: "key-pair", ID: "deploy"}}}
 	engine := quietEngine(newFakeAccounts(), reaper)
-	engine.ec2Inventory = failingInventory{t: t}
+	engine.ec2Inventory = inventory
 
 	require.NoError(t, engine.Precheck(testAccountID, testAccountName))
 	_, err = engine.Inventory(testCtx(t), testAccountID)
@@ -186,7 +185,49 @@ func TestNoEnginePathCallsTheEC2PlatformInventory(t *testing.T) {
 	dryRun.DryRun = true
 	_, err = engine.Teardown(testCtx(t), dryRun)
 	require.NoError(t, err)
+	assert.Empty(t, inventory.calls())
+
+	_, err = engine.DryRun(testCtx(t), testAccountID, "req-1")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"req-1"}, inventory.calls())
+
 	result, err := engine.Teardown(testCtx(t), request())
 	require.NoError(t, err)
 	assert.True(t, result.AccountDeleted)
+	assert.Equal(t, []string{"req-1"}, inventory.calls(), "a real teardown never consults the inventory")
+}
+
+// The shadow matches legacy listings to inventory kinds by the reapers' kind
+// names, so the registered platform reapers must use the contract's names.
+func TestPlatformReaperKindsAreTheInventoryKinds(t *testing.T) {
+	var kinds []string
+	for _, reaper := range EC2Reapers(nil, 1, nil) {
+		if reaper.Stage() == StagePlatform {
+			kinds = append(kinds, reaper.Kind())
+		}
+	}
+	for _, kind := range ec2ShadowKinds {
+		assert.Contains(t, kinds, kind.kind)
+	}
+}
+
+// A caller deadline shorter than the adapter's own timeout ends the request
+// at that deadline, and a context already done sends nothing.
+func TestEC2PlatformInventoryClientHonoursTheCallersDeadline(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	var requests atomic.Int32
+	serveInventory(t, nc, func() []byte { requests.Add(1); return nil })
+	client := newEC2PlatformInventoryClient(nc)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	started := time.Now()
+	assertAllIncomplete(t, client.PlatformInventory(ctx, testAccountID, ""), ec2v1.PlatformReasonTimeout)
+	assert.Less(t, time.Since(started), 5*time.Second)
+	assert.Equal(t, int32(1), requests.Load())
+
+	done, cancelDone := context.WithCancel(t.Context())
+	cancelDone()
+	assertAllIncomplete(t, client.PlatformInventory(done, testAccountID, ""), ec2v1.PlatformReasonTimeout)
+	assert.Equal(t, int32(1), requests.Load())
 }

spinifex/accountteardown/ec2_platform_shadow.go

@@ -0,0 +1,286 @@
+package accountteardown
+
+import (
+	"context"
+	"log/slog"
+	"regexp"
+	"slices"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ec2PlatformShadowBudget caps how long a dry run waits for EC2's platform
+// inventory after its own listings, never past the caller's deadline. A later
+// answer is scan-incomplete and the dry run returns exactly as it would without.
+const ec2PlatformShadowBudget = 10 * time.Second
+
+// ec2PlatformShadowMessage is the one operator audit record a dry run writes.
+const ec2PlatformShadowMessage = "Account teardown dry-run EC2 platform shadow comparison"
+
+// Shadow outcomes, per kind and for the whole record. Only agree means the
+// legacy listing and the inventory matched and nothing hidden was reported.
+const (
+	shadowAgree          = "agree"
+	shadowDiffer         = "differ"
+	shadowScanIncomplete = "scan-incomplete"
+)
+
+// Bases a kind's visible resources are compared on.
+const (
+	shadowBasisResourceID = "resource-id"
+	shadowBasisCount      = "count"
+)
+
+// shadowKind is one compared kind. The key-pair and placement-group reapers
+// list by name, which the inventory never carries, so those compare by count.
+type shadowKind struct {
+	kind   string
+	field  string
+	byName bool
+}
+
+var ec2ShadowKinds = []shadowKind{
+	{kind: ec2v1.PlatformKindKeyPair, field: "keyPair", byName: true},
+	{kind: ec2v1.PlatformKindPlacementGroup, field: "placementGroup", byName: true},
+	{kind: ec2v1.PlatformKindLaunchTemplate, field: "launchTemplate"},
+}
+
+// loggableResourceID admits only the AWS ID shape of the three kinds, so a
+// reply that put stored content where an ID belongs never reaches the log.
+var loggableResourceID = regexp.MustCompile(`^(key|pg|lt)-[0-9a-f]{1,32}$`)
+
+var knownShadowReasons = []string{
+	ec2v1.PlatformReasonStoreListFailed, ec2v1.PlatformReasonRecordReadFailed,
+	ec2v1.PlatformReasonOwnerRecordsUnresolved, ec2v1.PlatformReasonNoResponder,
+	ec2v1.PlatformReasonTimeout, ec2v1.PlatformReasonTransportFailed,
+	ec2v1.PlatformReasonErrorReply, ec2v1.PlatformReasonInvalidReply,
+	ec2v1.PlatformReasonKindNotReported,
+}
+
+var knownShadowClasses = []string{
+	ec2v1.PlatformClassPublicKeyWithoutMetadata, ec2v1.PlatformClassPublicKeyUnresolved,
+	ec2v1.PlatformClassUnreadableRecord, ec2v1.PlatformClassVersionBodyWithoutHeader,
+	ec2v1.PlatformClassNameIndexWithoutHeader, ec2v1.PlatformClassUnrecognisedTemplateEntry,
+}
+
+// shadowUnknownClass stands in for a count class this consumer does not know.
+const shadowUnknownClass = "unrecognised-class"
+
+func (e *Engine) ec2ShadowBudget() time.Duration {
+	if e.ec2PlatformShadowBudget > 0 {
+		return e.ec2PlatformShadowBudget
+	}
+	return ec2PlatformShadowBudget
+}
+
+// shadowEC2Platform compares EC2's platform inventory with the legacy listings
+// in result and logs the comparison. It reads result and never changes it.
+func (e *Engine) shadowEC2Platform(ctx context.Context, result *Result, correlationRef string) {
+	start := time.Now()
+	reply := e.awaitEC2PlatformInventory(ctx, result.AccountID, correlationRef)
+
+	reports := make(map[string]ec2v1.PlatformKindReport, len(reply.Kinds))
+	for _, report := range reply.Kinds {
+		reports[report.Kind] = report
+	}
+
+	outcome := shadowAgree
+	attrs := []slog.Attr{
+		slog.String("accountID", result.AccountID),
+		slog.String("correlationRef", correlationRef),
+	}
+	for _, kind := range ec2ShadowKinds {
+		attr, kindOutcome := compareShadowKind(kind, legacyIDs(result, kind.kind), reports[kind.kind], reply)
+		attrs = append(attrs, attr)
+		outcome = worseShadowOutcome(outcome, kindOutcome)
+	}
+	tagAttr, tagOutcome := shadowTagIndex(reports[ec2v1.PlatformKindTagIndex], reply)
+	attrs = append(attrs, tagAttr)
+	outcome = worseShadowOutcome(outcome, tagOutcome)
+
+	attrs = append(attrs, slog.String("outcome", outcome), slog.Int64("duration_ms", otelsetup.Millis(time.Since(start))))
+	e.logger().LogAttrs(ctx, slog.LevelInfo, ec2PlatformShadowMessage, attrs...)
+}
+
+// awaitEC2PlatformInventory bounds the inventory call by the shadow budget even
+// if the implementation ignores its context, and distrusts the reply's shape.
+func (e *Engine) awaitEC2PlatformInventory(ctx context.Context, accountID, correlationRef string) ec2v1.PlatformInventoryReply {
+	ctx, cancel := context.WithTimeout(ctx, e.ec2ShadowBudget())
+	defer cancel()
+
+	replies := make(chan ec2v1.PlatformInventoryReply, 1)
+	go func() { replies <- e.ec2Inventory.PlatformInventory(ctx, accountID, correlationRef) }()
+	select {
+	case reply := <-replies:
+		return normalisePlatformInventory(reply, accountID, correlationRef)
+	case <-ctx.Done():
+		return ec2v1.IncompletePlatformInventory(accountID, correlationRef, ec2v1.PlatformReasonTimeout)
+	}
+}
+
+func legacyIDs(result *Result, kind string) []string {
+	var ids []string
+	for _, stage := range result.Stages {
+		for _, resource := range stage.Deleted {
+			if resource.Kind == kind {
+				ids = append(ids, resource.ID)
+			}
+		}
+	}
+	return ids
+}
+
+// compareShadowKind compares one kind's legacy listing with the inventory's
+// live and dependency-held items, and reports its hidden records as counts.
+func compareShadowKind(kind shadowKind, legacy []string, report ec2v1.PlatformKindReport, reply ec2v1.PlatformInventoryReply) (slog.Attr, string) {
+	var visible, held []string
+	hidden := map[string]int{}
+	hiddenTotal := 0
+	for _, item := range reply.Items {
+		if item.Kind != kind.kind {
+			continue
+		}
+		switch item.Disposition {
+		case ec2v1.PlatformDispositionLive:
+			visible = append(visible, item.ResourceID)
+		case ec2v1.PlatformDispositionDependencyHeld:
+			visible = append(visible, item.ResourceID)
+			held = append(held, item.ResourceID)
+		default:
+			hidden[shadowUnknownClass]++
+			hiddenTotal++
+		}
+	}
+	for _, count := range reply.Counts {
+		if count.Kind != kind.kind || count.Count <= 0 {
+			continue
+		}
+		class := count.Class
+		if !slices.Contains(knownShadowClasses, class) {
+			class = shadowUnknownClass
+		}
+		hidden[class] += count.Count
+		hiddenTotal += count.Count
+	}
+
+	basis := shadowBasisResourceID
+	var onlyLegacy, onlyInventory []string
+	agree := len(legacy) == len(visible)
+	if kind.byName {
+		basis = shadowBasisCount
+	} else {
+		onlyLegacy, onlyInventory = setDifference(legacy, visible), setDifference(visible, legacy)
+		agree = len(onlyLegacy) == 0 && len(onlyInventory) == 0
+	}
+
+	comparison := shadowDiffer
+	if agree {
+		comparison = shadowAgree
+	}
+	outcome := comparison
+	if hiddenTotal > 0 {
+		outcome = shadowDiffer
+	}
+	if report.Status != ec2v1.PlatformStatusObservedClean && report.Status != ec2v1.PlatformStatusRecordsFound {
+		comparison, outcome = shadowScanIncomplete, shadowScanIncomplete
+	}
+
+	loggableOnlyLegacy, droppedLegacy := loggableIDs(onlyLegacy)
+	loggableOnlyInventory, droppedInventory := loggableIDs(onlyInventory)
+	loggableHeld, droppedHeld := loggableIDs(held)
+	return slog.Group(kind.field,
+		slog.String("status", report.Status),
+		slog.Any("incompleteReasons", fixedReasons(report.IncompleteReasons)),
+		slog.Time("scanStartedAt", report.ScanStartedAt),
+		slog.Time("scanEndedAt", report.ScanEndedAt),
+		slog.String("comparison", comparison),
+		slog.String("matchBasis", basis),
+		slog.Int("legacyCount", len(legacy)),
+		slog.Int("inventoryVisibleCount", len(visible)),
+		slog.Any("onlyInLegacy", loggableOnlyLegacy),
+		slog.Any("onlyInInventory", loggableOnlyInventory),
+		slog.Any("dependencyHeld", loggableHeld),
+		slog.Int("hiddenRecordCount", hiddenTotal),
+		slog.Any("hiddenRecords", hidden),
+		slog.Int("unloggableIDCount", droppedLegacy+droppedInventory+droppedHeld),
+	), outcome
+}
+
+// shadowTagIndex reports tag-index entries whose resource has no owner record,
+// which no legacy reaper lists.
+func shadowTagIndex(report ec2v1.PlatformKindReport, reply ec2v1.PlatformInventoryReply) (slog.Attr, string) {
+	var residue []string
+	for _, item := range reply.Items {
+		if item.Kind == ec2v1.PlatformKindTagIndex {
+			residue = append(residue, item.ResourceID)
+		}
+	}
+	outcome := shadowAgree
+	if len(residue) > 0 {
+		outcome = shadowDiffer
+	}
+	if report.Status != ec2v1.PlatformStatusObservedClean && report.Status != ec2v1.PlatformStatusRecordsFound {
+		outcome = shadowScanIncomplete
+	}
+	loggable, dropped := loggableIDs(residue)
+	return slog.Group("tagIndex",
+		slog.String("status", report.Status),
+		slog.Any("incompleteReasons", fixedReasons(report.IncompleteReasons)),
+		slog.Time("scanStartedAt", report.ScanStartedAt),
+		slog.Time("scanEndedAt", report.ScanEndedAt),
+		slog.Int("residueCount", len(residue)),
+		slog.Any("residue", loggable),
+		slog.Int("unloggableIDCount", dropped),
+	), outcome
+}
+
+func worseShadowOutcome(a, b string) string {
+	rank := map[string]int{shadowAgree: 0, shadowDiffer: 1, shadowScanIncomplete: 2}
+	if rank[b] > rank[a] {
+		return b
+	}
+	return a
+}
+
+// setDifference returns the sorted members of a absent from b.
+func setDifference(a, b []string) []string {
+	out := []string{}
+	for _, id := range a {
+		if !slices.Contains(b, id) && !slices.Contains(out, id) {
+			out = append(out, id)
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+func loggableIDs(ids []string) ([]string, int) {
+	out := []string{}
+	dropped := 0
+	for _, id := range ids {
+		if loggableResourceID.MatchString(id) {
+			out = append(out, id)
+			continue
+		}
+		dropped++
+	}
+	slices.Sort(out)
+	return out, dropped
+}
+
+// fixedReasons keeps only the contract's reason categories; anything else a
+// reply carried is reported as an invalid reply rather than logged.
+func fixedReasons(reasons []string) []string {
+	out := []string{}
+	for _, reason := range reasons {
+		if !slices.Contains(knownShadowReasons, reason) {
+			reason = ec2v1.PlatformReasonInvalidReply
+		}
+		if !slices.Contains(out, reason) {
+			out = append(out, reason)
+		}
+	}
+	return out
+}

spinifex/accountteardown/ec2_platform_shadow_test.go

@@ -0,0 +1,513 @@
+package accountteardown
+
+//test:in-package — the shadow comparison, the inventory binding it reads and
+// its budget are unexported.
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var shadowScanTime = time.Date(2026, 10, 16, 9, 0, 0, 0, time.UTC)
+
+// fakeInventory answers with reply, or never when blocked, and records the
+// correlation reference of every call.
+type fakeInventory struct {
+	reply   ec2v1.PlatformInventoryReply
+	blocked chan struct{}
+
+	mu   sync.Mutex
+	refs []string
+}
+
+func (f *fakeInventory) PlatformInventory(_ context.Context, _, correlationRef string) ec2v1.PlatformInventoryReply {
+	f.mu.Lock()
+	f.refs = append(f.refs, correlationRef)
+	f.mu.Unlock()
+	if f.blocked != nil {
+		<-f.blocked
+	}
+	return f.reply
+}
+
+func (f *fakeInventory) calls() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.refs...)
+}
+
+// countingReaper counts listings, so a dry run can be shown to list once.
+type countingReaper struct {
+	Reaper
+
+	lists int
+}
+
+func (r *countingReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
+	r.lists++
+	return r.Reaper.List(ctx, accountID)
+}
+
+func platformKinds(status string) []ec2v1.PlatformKindReport {
+	var kinds []ec2v1.PlatformKindReport
+	for _, kind := range ec2v1.PlatformKinds() {
+		kinds = append(kinds, ec2v1.PlatformKindReport{
+			Kind: kind, Status: status, ScanStartedAt: shadowScanTime, ScanEndedAt: shadowScanTime.Add(time.Second),
+		})
+	}
+	return kinds
+}
+
+func setKind(kinds []ec2v1.PlatformKindReport, kind, status string, reasons ...string) {
+	for i := range kinds {
+		if kinds[i].Kind == kind {
+			kinds[i].Status, kinds[i].IncompleteReasons = status, reasons
+		}
+	}
+}
+
+func item(kind, id, disposition string) ec2v1.PlatformItem {
+	return ec2v1.PlatformItem{ResourceID: id, Kind: kind, Disposition: disposition}
+}
+
+// legacyPlatform is a key pair, a placement group and a launch template as the
+// legacy reapers list them: the first two by name, the third by ID.
+func legacyPlatform() []Reaper {
+	return []Reaper{
+		&fakeReaper{kind: "instance", stage: StageCompute, resources: []Resource{{Kind: "instance", ID: "i-0123456789abcdef0"}}},
+		&fakeReaper{kind: "key-pair", stage: StagePlatform, resources: []Resource{{Kind: "key-pair", ID: "deploy"}}},
+		&fakeReaper{kind: "placement-group", stage: StagePlatform, resources: []Resource{{Kind: "placement-group", ID: "spread"}}},
+		&fakeReaper{kind: "launch-template", stage: StagePlatform, resources: []Resource{{Kind: "launch-template", ID: "lt-0aaaaaaaaaaaaaaaa"}}},
+	}
+}
+
+// matchingInventory reports exactly what legacyPlatform lists.
+func matchingInventory() ec2v1.PlatformInventoryReply {
+	return ec2v1.PlatformInventoryReply{
+		AccountID: testAccountID,
+		Kinds:     platformKinds(ec2v1.PlatformStatusRecordsFound),
+		Items: []ec2v1.PlatformItem{
+			item(ec2v1.PlatformKindKeyPair, "key-0aaaaaaaaaaaaaaaa", ec2v1.PlatformDispositionLive),
+			item(ec2v1.PlatformKindPlacementGroup, "pg-0aaaaaaaaaaaaaaaa", ec2v1.PlatformDispositionLive),
+			item(ec2v1.PlatformKindLaunchTemplate, "lt-0aaaaaaaaaaaaaaaa", ec2v1.PlatformDispositionLive),
+		},
+	}
+}
+
+// shadowEngine is a quiet engine whose log is captured as JSON in logs.
+func shadowEngine(inventory ec2PlatformInventory, reapers ...Reaper) (*Engine, *bytes.Buffer) {
+	engine := quietEngine(newFakeAccounts(), reapers...)
+	logs := &bytes.Buffer{}
+	engine.Logger = slog.New(slog.NewJSONHandler(logs, nil))
+	engine.ec2Inventory = inventory
+	return engine, logs
+}
+
+// shadowRecords returns every shadow comparison record in logs.
+func shadowRecords(t *testing.T, logs *bytes.Buffer) []map[string]any {
+	t.Helper()
+	var records []map[string]any
+	for line := range strings.SplitSeq(strings.TrimSpace(logs.String()), "\n") {
+		if line == "" {
+			continue
+		}
+		var record map[string]any
+		require.NoError(t, json.Unmarshal([]byte(line), &record))
+		if record["msg"] == ec2PlatformShadowMessage {
+			records = append(records, record)
+		}
+	}
+	return records
+}
+
+func shadowRecord(t *testing.T, logs *bytes.Buffer) map[string]any {
+	t.Helper()
+	records := shadowRecords(t, logs)
+	require.Len(t, records, 1, "one shadow record per dry run")
+	return records[0]
+}
+
+func group(t *testing.T, record map[string]any, name string) map[string]any {
+	t.Helper()
+	g, ok := record[name].(map[string]any)
+	require.True(t, ok, "record has a %s group", name)
+	return g
+}
+
+func strs(values ...string) []any {
+	out := []any{}
+	for _, v := range values {
+		out = append(out, v)
+	}
+	return out
+}
+
+func resultJSON(t *testing.T, result *Result) string {
+	t.Helper()
+	data, err := json.Marshal(result)
+	require.NoError(t, err)
+	return string(data)
+}
+
+// noShadowResult is what the same dry run returns with no inventory bound.
+func noShadowResult(t *testing.T, reapers ...Reaper) string {
+	t.Helper()
+	result, err := quietEngine(newFakeAccounts(), reapers...).Inventory(testCtx(t), testAccountID)
+	require.NoError(t, err)
+	return resultJSON(t, result)
+}
+
+// Visible resources that match on both sides agree. The legacy key-pair and
+// placement-group reapers list names, so those kinds compare by count.
+func TestDryRunShadowAgreesWhenBothSidesListTheSameResources(t *testing.T) {
+	reapers := legacyPlatform()
+	counted := make([]Reaper, 0, len(reapers))
+	for _, reaper := range reapers {
+		counted = append(counted, &countingReaper{Reaper: reaper})
+	}
+	inventory := &fakeInventory{reply: matchingInventory()}
+	engine, logs := shadowEngine(inventory, counted...)
+
+	result, err := engine.DryRun(testCtx(t), testAccountID, "req-agree")
+	require.NoError(t, err)
+
+	assert.Equal(t, noShadowResult(t, legacyPlatform()...), resultJSON(t, result))
+	assert.Equal(t, []string{"req-agree"}, inventory.calls())
+	for _, reaper := range counted {
+		assert.Equal(t, 1, reaper.(*countingReaper).lists, "%s listed once: the shadow reuses the dry run's listing", reaper.Kind())
+	}
+
+	record := shadowRecord(t, logs)
+	assert.Equal(t, "INFO", record["level"])
+	assert.Equal(t, testAccountID, record["accountID"])
+	assert.Equal(t, "req-agree", record["correlationRef"])
+	assert.Equal(t, shadowAgree, record["outcome"])
+	assert.Contains(t, record, "duration_ms")
+
+	keyPair := group(t, record, "keyPair")
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, keyPair["status"])
+	assert.Equal(t, shadowAgree, keyPair["comparison"])
+	assert.Equal(t, shadowBasisCount, keyPair["matchBasis"])
+	assert.InDelta(t, 1, keyPair["legacyCount"], 0)
+	assert.InDelta(t, 1, keyPair["inventoryVisibleCount"], 0)
+	assert.Equal(t, shadowScanTime.Format(time.RFC3339), keyPair["scanStartedAt"])
+
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.Equal(t, shadowAgree, launchTemplate["comparison"])
+	assert.Equal(t, shadowBasisResourceID, launchTemplate["matchBasis"])
+	assert.Equal(t, strs(), launchTemplate["onlyInLegacy"])
+	assert.Equal(t, strs(), launchTemplate["onlyInInventory"])
+
+	tagIndex := group(t, record, "tagIndex")
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, tagIndex["status"])
+	assert.InDelta(t, 0, tagIndex["residueCount"], 0)
+}
+
+// Visible resources one side has and the other does not are named by ID where
+// the legacy reaper lists IDs, and counted where it lists names.
+func TestDryRunShadowReportsVisibleResourcesOnlyOneSideSees(t *testing.T) {
+	reapers := legacyPlatform()
+	reapers[1].(*fakeReaper).resources = append(reapers[1].(*fakeReaper).resources, Resource{Kind: "key-pair", ID: "ci"})
+	reapers[3].(*fakeReaper).resources = []Resource{
+		{Kind: "launch-template", ID: "lt-0aaaaaaaaaaaaaaaa"}, {Kind: "launch-template", ID: "lt-0bbbbbbbbbbbbbbbb"},
+	}
+	reply := matchingInventory()
+	reply.Items = append(reply.Items, item(ec2v1.PlatformKindLaunchTemplate, "lt-0cccccccccccccccc", ec2v1.PlatformDispositionLive))
+	engine, logs := shadowEngine(&fakeInventory{reply: reply}, reapers...)
+
+	_, err := engine.DryRun(testCtx(t), testAccountID, "req-differ")
+	require.NoError(t, err)
+
+	record := shadowRecord(t, logs)
+	assert.Equal(t, shadowDiffer, record["outcome"])
+	keyPair := group(t, record, "keyPair")
+	assert.Equal(t, shadowDiffer, keyPair["comparison"])
+	assert.InDelta(t, 2, keyPair["legacyCount"], 0)
+	assert.InDelta(t, 1, keyPair["inventoryVisibleCount"], 0)
+	assert.Equal(t, strs(), keyPair["onlyInLegacy"], "legacy key-pair names are never logged")
+
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.Equal(t, shadowDiffer, launchTemplate["comparison"])
+	assert.Equal(t, strs("lt-0bbbbbbbbbbbbbbbb"), launchTemplate["onlyInLegacy"])
+	assert.Equal(t, strs("lt-0cccccccccccccccc"), launchTemplate["onlyInInventory"])
+	assert.Equal(t, shadowAgree, group(t, record, "placementGroup")["comparison"])
+}
+
+// Residue the legacy reapers cannot see makes the record differ even when the
+// visible resources agree: as counts per class, and tag-index entries by ID.
+func TestDryRunShadowReportsHiddenResidueTheLegacyReapersMiss(t *testing.T) {
+	reply := matchingInventory()
+	reply.Counts = []ec2v1.PlatformRecordCount{
+		{Kind: ec2v1.PlatformKindKeyPair, Disposition: ec2v1.PlatformDispositionResidue, Class: ec2v1.PlatformClassPublicKeyWithoutMetadata, Count: 1},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionResidue, Class: ec2v1.PlatformClassVersionBodyWithoutHeader, Count: 2},
+		{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionUnreadable, Class: ec2v1.PlatformClassUnreadableRecord, Count: 1},
+	}
+	reply.Items = append(reply.Items, item(ec2v1.PlatformKindTagIndex, "pg-0dddddddddddddddd", ec2v1.PlatformDispositionResidue))
+	engine, logs := shadowEngine(&fakeInventory{reply: reply}, legacyPlatform()...)
+
+	_, err := engine.DryRun(testCtx(t), testAccountID, "req-residue")
+	require.NoError(t, err)
+
+	record := shadowRecord(t, logs)
+	assert.Equal(t, shadowDiffer, record["outcome"])
+	keyPair := group(t, record, "keyPair")
+	assert.Equal(t, shadowAgree, keyPair["comparison"], "the visible key pairs still agree")
+	assert.InDelta(t, 1, keyPair["hiddenRecordCount"], 0)
+	assert.Equal(t, map[string]any{ec2v1.PlatformClassPublicKeyWithoutMetadata: float64(1)}, keyPair["hiddenRecords"])
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.InDelta(t, 3, launchTemplate["hiddenRecordCount"], 0)
+	assert.Equal(t, map[string]any{
+		ec2v1.PlatformClassVersionBodyWithoutHeader: float64(2),
+		ec2v1.PlatformClassUnreadableRecord:         float64(1),
+	}, launchTemplate["hiddenRecords"])
+	tagIndex := group(t, record, "tagIndex")
+	assert.InDelta(t, 1, tagIndex["residueCount"], 0)
+	assert.Equal(t, strs("pg-0dddddddddddddddd"), tagIndex["residue"])
+}
+
+// A placement group still holding instances is a plain listed resource to the
+// legacy dry run, which learns otherwise only when its delete fails.
+func TestDryRunShadowReportsADependencyHeldPlacementGroup(t *testing.T) {
+	reply := matchingInventory()
+	reply.Items[1].Disposition = ec2v1.PlatformDispositionDependencyHeld
+	engine, logs := shadowEngine(&fakeInventory{reply: reply}, legacyPlatform()...)
+
+	_, err := engine.DryRun(testCtx(t), testAccountID, "req-held")
+	require.NoError(t, err)
+
+	placementGroup := group(t, shadowRecord(t, logs), "placementGroup")
+	assert.Equal(t, shadowAgree, placementGroup["comparison"])
+	assert.InDelta(t, 1, placementGroup["inventoryVisibleCount"], 0)
+	assert.Equal(t, strs("pg-0aaaaaaaaaaaaaaaa"), placementGroup["dependencyHeld"])
+}
+
+// An owner store that could not be read is scan-incomplete, never agreement,
+// even where the IDs it did report match.
+func TestDryRunShadowNeverReportsAnIncompleteScanAsAgreement(t *testing.T) {
+	reply := matchingInventory()
+	setKind(reply.Kinds, ec2v1.PlatformKindLaunchTemplate, ec2v1.PlatformStatusScanIncomplete, ec2v1.PlatformReasonRecordReadFailed)
+	engine, logs := shadowEngine(&fakeInventory{reply: reply}, legacyPlatform()...)
+
+	result, err := engine.DryRun(testCtx(t), testAccountID, "req-store")
+	require.NoError(t, err)
+	assert.Equal(t, noShadowResult(t, legacyPlatform()...), resultJSON(t, result))
+
+	record := shadowRecord(t, logs)
+	assert.Equal(t, shadowScanIncomplete, record["outcome"])
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.Equal(t, shadowScanIncomplete, launchTemplate["comparison"])
+	assert.Equal(t, strs(ec2v1.PlatformReasonRecordReadFailed), launchTemplate["incompleteReasons"])
+	assert.Equal(t, shadowAgree, group(t, record, "keyPair")["comparison"])
+}
+
+// However the inventory fails, the dry run returns what it returns without it
+// and the record says scan-incomplete with the fixed reason.
+func TestDryRunShadowFailureLeavesTheDryRunUnchanged(t *testing.T) {
+	want := noShadowResult(t, legacyPlatform()...)
+	for _, reason := range []string{
+		ec2v1.PlatformReasonNoResponder, ec2v1.PlatformReasonTimeout,
+		ec2v1.PlatformReasonTransportFailed, ec2v1.PlatformReasonErrorReply,
+	} {
+		t.Run(reason, func(t *testing.T) {
+			engine, logs := shadowEngine(&fakeInventory{reply: ec2v1.IncompletePlatformInventory(testAccountID, "req-fail", reason)}, legacyPlatform()...)
+			result, err := engine.DryRun(testCtx(t), testAccountID, "req-fail")
+			require.NoError(t, err)
+			assert.Equal(t, want, resultJSON(t, result))
+
+			record := shadowRecord(t, logs)
+			assert.Equal(t, shadowScanIncomplete, record["outcome"])
+			for _, name := range []string{"keyPair", "placementGroup", "launchTemplate", "tagIndex"} {
+				g := group(t, record, name)
+				assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, g["status"], name)
+				assert.Equal(t, strs(reason), g["incompleteReasons"], name)
+			}
+		})
+	}
+
+	t.Run("reply for another account", func(t *testing.T) {
+		reply := matchingInventory()
+		reply.AccountID = "000000000099"
+		engine, logs := shadowEngine(&fakeInventory{reply: reply}, legacyPlatform()...)
+		result, err := engine.DryRun(testCtx(t), testAccountID, "req-fail")
+		require.NoError(t, err)
+		assert.Equal(t, want, resultJSON(t, result))
+		record := shadowRecord(t, logs)
+		assert.Equal(t, shadowScanIncomplete, record["outcome"])
+		assert.Equal(t, strs(ec2v1.PlatformReasonInvalidReply), group(t, record, "keyPair")["incompleteReasons"])
+	})
+
+	t.Run("no daemon serves the subject", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		engine, logs := shadowEngine(newEC2PlatformInventoryClient(nc), legacyPlatform()...)
+		result, err := engine.DryRun(testCtx(t), testAccountID, "req-fail")
+		require.NoError(t, err)
+		assert.Equal(t, want, resultJSON(t, result))
+		assert.Equal(t, strs(ec2v1.PlatformReasonNoResponder), group(t, shadowRecord(t, logs), "tagIndex")["incompleteReasons"])
+	})
+}
+
+// An inventory that never answers, even one ignoring its context, holds the
+// dry run for the shadow budget at most and is logged as a timeout.
+func TestDryRunShadowIsBoundedByItsBudget(t *testing.T) {
+	inventory := &fakeInventory{reply: matchingInventory(), blocked: make(chan struct{})}
+	t.Cleanup(func() { close(inventory.blocked) })
+	engine, logs := shadowEngine(inventory, legacyPlatform()...)
+	engine.ec2PlatformShadowBudget = 20 * time.Millisecond
+
+	started := time.Now()
+	result, err := engine.DryRun(testCtx(t), testAccountID, "req-slow")
+	require.NoError(t, err)
+	assert.Less(t, time.Since(started), 5*time.Second)
+	assert.Equal(t, noShadowResult(t, legacyPlatform()...), resultJSON(t, result))
+	assert.Equal(t, strs(ec2v1.PlatformReasonTimeout), group(t, shadowRecord(t, logs), "launchTemplate")["incompleteReasons"])
+}
+
+// The caller's deadline wins over the budget, so a dry run never outlasts the
+// bound its caller already set.
+func TestDryRunShadowStopsAtTheCallersDeadline(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	serveInventory(t, nc, func() []byte { return nil })
+	engine, logs := shadowEngine(newEC2PlatformInventoryClient(nc), legacyPlatform()...)
+	require.Greater(t, engine.ec2ShadowBudget(), 5*time.Second)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	started := time.Now()
+	result, err := engine.DryRun(ctx, testAccountID, "req-deadline")
+	require.NoError(t, err)
+	assert.Less(t, time.Since(started), 5*time.Second)
+	assert.Equal(t, noShadowResult(t, legacyPlatform()...), resultJSON(t, result))
+	assert.Equal(t, strs(ec2v1.PlatformReasonTimeout), group(t, shadowRecord(t, logs), "keyPair")["incompleteReasons"])
+}
+
+// A dry run whose own listing fails returns that error as before, and nothing
+// is compared against a listing that does not exist.
+func TestDryRunWithAFailedListingDoesNotCallTheInventory(t *testing.T) {
+	reapers := legacyPlatform()
+	reapers[2].(*fakeReaper).listErr = errors.New("describe failed")
+	inventory := &fakeInventory{reply: matchingInventory()}
+	engine, logs := shadowEngine(inventory, reapers...)
+
+	_, err := engine.DryRun(testCtx(t), testAccountID, "req-list")
+	require.ErrorContains(t, err, "inventory placement-group")
+	assert.Empty(t, inventory.calls())
+	assert.Empty(t, shadowRecords(t, logs))
+}
+
+// A reply that puts anything but the contract's fixed vocabulary where a
+// reason, class or ID belongs never reaches the log; the IDs are counted.
+func TestDryRunShadowLogsOnlyTheFixedVocabulary(t *testing.T) {
+	reply := matchingInventory()
+	setKind(reply.Kinds, ec2v1.PlatformKindKeyPair, ec2v1.PlatformStatusScanIncomplete, "read keys/000000000042/secret-key-name: denied")
+	reply.Counts = []ec2v1.PlatformRecordCount{{Kind: ec2v1.PlatformKindLaunchTemplate, Disposition: ec2v1.PlatformDispositionResidue, Class: "secret-template-name", Count: 1}}
+	reply.Items = append(reply.Items,
+		item(ec2v1.PlatformKindLaunchTemplate, "lt-secret-template-name", ec2v1.PlatformDispositionLive),
+		item(ec2v1.PlatformKindTagIndex, "pg-team=secret-tag-value", ec2v1.PlatformDispositionResidue),
+	)
+	reapers := legacyPlatform()
+	reapers[1].(*fakeReaper).resources = []Resource{{Kind: "key-pair", ID: "secret-key-name"}}
+	engine, logs := shadowEngine(&fakeInventory{reply: reply}, reapers...)
+
+	_, err := engine.DryRun(testCtx(t), testAccountID, "req-disclosure")
+	require.NoError(t, err)
+
+	assert.NotContains(t, logs.String(), "secret")
+	record := shadowRecord(t, logs)
+	assert.Equal(t, strs(ec2v1.PlatformReasonInvalidReply), group(t, record, "keyPair")["incompleteReasons"])
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.Equal(t, map[string]any{shadowUnknownClass: float64(1)}, launchTemplate["hiddenRecords"])
+	assert.InDelta(t, 1, launchTemplate["unloggableIDCount"], 0)
+	assert.InDelta(t, 1, group(t, record, "tagIndex")["unloggableIDCount"], 0)
+}
+
+// End to end over the real owner stores and the NATS adapter: the record names
+// the residue each reaper misses and nothing a customer named or stored.
+func TestDryRunShadowOverTheOwnerStores(t *testing.T) {
+	o := newPlatformOwners(t)
+	ctx := testCtx(t)
+	inv := accountinventory.New(o.keys, o.groups, o.templates, o.tags)
+	sub, err := o.nc.QueueSubscribe(ec2v1.PlatformInventorySubject, "spinifex-workers", func(msg *nats.Msg) {
+		natsmsg.ServeNATSRequestCtx(msg, inv.PlatformInventory)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	material := []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl")
+	for _, name := range []string{"secret-key-name", "secret-orphan-key"} {
+		_, err = o.keys.ImportKeyPair(ctx, &ec2.ImportKeyPairInput{KeyName: aws.String(name), PublicKeyMaterial: material}, testAccountID)
+		require.NoError(t, err)
+	}
+	_, err = o.keys.DeleteKeyPair(ctx, &ec2.DeleteKeyPairInput{KeyName: aws.String("secret-orphan-key")}, testAccountID)
+	require.NoError(t, err)
+	// Residue an older release's failed deletes left behind.
+	_, err = o.store.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(platformBucket), Key: aws.String("keys/" + testAccountID + "/secret-orphan-key"), Body: bytes.NewReader(material),
+	})
+	require.NoError(t, err)
+
+	_, err = o.groups.CreatePlacementGroup(ctx, &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("secret-group-name"), Strategy: aws.String(ec2.PlacementStrategySpread),
+	}, testAccountID)
+	require.NoError(t, err)
+	doomed, err := o.groups.CreatePlacementGroup(ctx, &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("secret-doomed-group"), Strategy: aws.String(ec2.PlacementStrategySpread),
+	}, testAccountID)
+	require.NoError(t, err)
+	lt, err := o.templates.CreateLaunchTemplate(ctx, &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("secret-template-name"),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("secret-user-data")))},
+	}, testAccountID)
+	require.NoError(t, err)
+	doomedID, ltID := aws.StringValue(doomed.PlacementGroup.GroupId), aws.StringValue(lt.LaunchTemplate.LaunchTemplateId)
+	_, err = o.tags.CreateTags(ctx, &ec2.CreateTagsInput{
+		Resources: []*string{aws.String(doomedID), aws.String(ltID)},
+		Tags:      []*ec2.Tag{{Key: aws.String("secret-tag-key"), Value: aws.String("secret-tag-value")}},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = o.groups.DeletePlacementGroup(ctx, &ec2.DeletePlacementGroupInput{GroupName: aws.String("secret-doomed-group")}, testAccountID)
+	require.NoError(t, err)
+	_, err = o.tags.CreateTags(ctx, &ec2.CreateTagsInput{
+		Resources: []*string{aws.String(doomedID)},
+		Tags:      []*ec2.Tag{{Key: aws.String("secret-tag-key"), Value: aws.String("secret-tag-value")}},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	engine, logs := shadowEngine(newEC2PlatformInventoryClient(o.nc), platformReapers(o.nc)...)
+	result, err := engine.DryRun(ctx, testAccountID, "req-owners")
+	require.NoError(t, err)
+	plain, err := quietEngine(newFakeAccounts(), platformReapers(o.nc)...).Inventory(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, resultJSON(t, plain), resultJSON(t, result))
+
+	assert.NotContains(t, logs.String(), "secret")
+	assert.NotContains(t, logs.String(), base64.StdEncoding.EncodeToString([]byte("secret-user-data")))
+	record := shadowRecord(t, logs)
+	assert.Equal(t, shadowDiffer, record["outcome"])
+	keyPair := group(t, record, "keyPair")
+	assert.Equal(t, shadowAgree, keyPair["comparison"])
+	assert.Equal(t, map[string]any{ec2v1.PlatformClassPublicKeyWithoutMetadata: float64(1)}, keyPair["hiddenRecords"])
+	assert.Equal(t, shadowAgree, group(t, record, "placementGroup")["comparison"])
+	launchTemplate := group(t, record, "launchTemplate")
+	assert.Equal(t, shadowAgree, launchTemplate["comparison"])
+	assert.Equal(t, shadowBasisResourceID, launchTemplate["matchBasis"])
+	assert.Equal(t, strs(doomedID), group(t, record, "tagIndex")["residue"])
+}

spinifex/accountteardown/engine.go

@@ -29,9 +29,12 @@ type Engine struct {
 	// and this is the only view into it while it runs.
 	OnStage func(ctx context.Context, stage StageResult)
 
-	// ec2Inventory is EC2's read-only platform inventory. Bound by the cluster
-	// wiring but consulted by no teardown or inventory path yet.
+	// ec2Inventory is EC2's read-only platform inventory. Only DryRun consults
+	// it, and only to log a comparison; no teardown or Inventory call does.
 	ec2Inventory ec2PlatformInventory
+
+	// ec2PlatformShadowBudget overrides ec2PlatformShadowBudget when positive.
+	ec2PlatformShadowBudget time.Duration
 }
 
 // NewEngine builds an engine with the default timeouts.
@@ -87,6 +90,20 @@ func (e *Engine) Inventory(ctx context.Context, accountID string) (*Result, erro
 	return result, nil
 }
 
+// DryRun is the explicit operator dry run. It returns exactly what Inventory
+// returns; beside it, EC2's platform inventory is compared with the legacy
+// listings and only logged, under correlationRef, which authorises nothing.
+func (e *Engine) DryRun(ctx context.Context, accountID, correlationRef string) (*Result, error) {
+	result, err := e.Inventory(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	if e.ec2Inventory != nil {
+		e.shadowEC2Platform(ctx, result, correlationRef)
+	}
+	return result, nil
+}
+
 // Teardown empties the account and then removes it.
 //
 // The account is marked TERMINATING before anything is deleted. Without that

spinifex/accountteardown/reapers_ec2_account_settings_current_behaviour_test.go

@@ -0,0 +1,50 @@
+package accountteardown
+
+//test:in-package — drives the real final-stage reapers against the real
+// EC2 account-settings service to pin that teardown leaves the bucket alone.
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// No reaper in this package references the EC2 account-settings bucket or
+// calls a settings delete. Running the real engine and the real StageAccount
+// reapers against an account with settings enabled shows the consequence:
+// the settings record outlives the account record it belongs to.
+func TestCurrentBehaviour_AccountTermination_EC2AccountSettingsSurviveTeardown(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	ctx := testCtx(t)
+
+	settings, err := ec2account.NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	_, err = settings.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	_, err = settings.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	require.NoError(t, err)
+
+	accounts := newFakeAccounts()
+	iam := &accountIAM{account: &handlers_iam.Account{AccountID: testAccountID, AccountName: testAccountName}}
+	engine := quietEngine(accounts, AccountReapers(iam, nil, nil)...)
+
+	result, err := engine.Teardown(ctx, request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.True(t, accounts.deleted)
+	assert.Zero(t, result.StuckCount())
+
+	bucket, err := js.KeyValue(ctx, ec2account.KVBucketAccountSettings)
+	require.NoError(t, err)
+	_, err = bucket.Get(ctx, testAccountID)
+	require.NoError(t, err, "the settings key must still be present after the account is torn down")
+
+	out, err := settings.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, *out.EbsEncryptionByDefault)
+}

spinifex/accountteardown/reapers_ec2_platform_current_behaviour_test.go

@@ -34,27 +34,11 @@ import (
 
 const platformBucket = "test-bucket"
 
-// deleteFailingStore fails object deletes its predicate selects.
-type deleteFailingStore struct {
-	*objectstore.MemoryObjectStore
-
-	failDelete func(key string) bool
-}
-
-var _ objectstore.ObjectStore = (*deleteFailingStore)(nil)
-
-func (s *deleteFailingStore) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
-	if s.failDelete != nil && s.failDelete(aws.StringValue(in.Key)) {
-		return nil, errors.New("injected store failure")
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, in)
-}
-
 // platformOwners runs the real key, placement-group, launch-template and tag
 // services behind the subjects the platform reapers call.
 type platformOwners struct {
 	nc        *nats.Conn
-	store     *deleteFailingStore
+	store     *objectstore.MemoryObjectStore
 	keys      *ec2key.KeyServiceImpl
 	groups    *ec2placementgroup.PlacementGroupServiceImpl
 	templates *ec2launchtemplate.LaunchTemplateServiceImpl
@@ -81,7 +65,7 @@ func newPlatformOwners(t *testing.T) *platformOwners {
 	t.Helper()
 	_, nc, js := testutil.StartTestJetStream(t)
 	ctx := t.Context()
-	o := &platformOwners{nc: nc, store: &deleteFailingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}}
+	o := &platformOwners{nc: nc, store: objectstore.NewMemoryObjectStore()}
 
 	var err error
 	o.tagsKV, err = ec2tags.GetOrCreateTagsBucket(ctx, js)
@@ -92,8 +76,10 @@ func newPlatformOwners(t *testing.T) *platformOwners {
 	o.keys.SetCentralTagStore(o.tags)
 	o.groups, err = ec2placementgroup.NewPlacementGroupServiceImplWithNATS(ctx, nil, nc)
 	require.NoError(t, err)
+	o.groups.SetCentralTagStore(o.tags)
 	o.templates, err = ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(ctx, nil, nc)
 	require.NoError(t, err)
+	o.templates.SetCentralTagStore(o.tags)
 	o.pgKV, err = js.KeyValue(ctx, ec2placementgroup.KVBucketPlacementGroups)
 	require.NoError(t, err)
 	o.ltKV, err = js.KeyValue(ctx, ec2launchtemplate.KVBucketLaunchTemplates)
@@ -172,9 +158,18 @@ func TestCurrentBehaviour_AccountTermination_PlatformReapersReportDrainedOverHid
 	}, testAccountID)
 	require.NoError(t, err)
 
-	// Residue each owner's Describe skips.
-	publicKey := "keys/" + testAccountID + "/deploy"
-	o.store.failDelete = func(key string) bool { return key == publicKey }
+	// Residue each owner's Describe skips, as left by older releases or a
+	// crash between writes.
+	publicKey := "keys/" + testAccountID + "/old-deploy"
+	_, err = o.store.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(platformBucket), Key: aws.String(publicKey), Body: strings.NewReader("ssh-ed25519 AAAA"),
+	})
+	require.NoError(t, err)
+	staleTagged := "pg-0stale000000000"
+	_, err = o.tags.CreateTags(ctx, &ec2.CreateTagsInput{
+		Resources: []*string{aws.String(staleTagged)}, Tags: []*ec2.Tag{{Key: aws.String("team"), Value: aws.String("blue")}},
+	}, testAccountID)
+	require.NoError(t, err)
 	_, err = o.store.PutObject(ctx, &s3.PutObjectInput{
 		Bucket: aws.String(platformBucket), Key: aws.String("keys/" + testAccountID + "/key-0badbadbadbadbad0.json"),
 		Body: strings.NewReader("{not json"),
@@ -210,10 +205,6 @@ func TestCurrentBehaviour_AccountTermination_PlatformReapersReportDrainedOverHid
 	assert.ElementsMatch(t, []string{publicKey, "keys/" + testAccountID + "/key-0badbadbadbadbad0.json"}, o.objectKeys(t))
 	assert.Equal(t, []string{testAccountID + ".broken"}, accountKVKeys(t, o.pgKV))
 	assert.Equal(t, []string{orphanBody}, accountKVKeys(t, o.ltKV))
-	assert.Equal(t, sortedStrings(testAccountID+"."+pgID, testAccountID+"."+ltID), accountKVKeys(t, o.tagsKV))
-}
-
-func sortedStrings(s ...string) []string {
-	slices.Sort(s)
-	return s
+	// The owner deletes clear their own tags; only the stale entry remains.
+	assert.Equal(t, []string{testAccountID + "." + staleTagged}, accountKVKeys(t, o.tagsKV))
 }

spinifex/accountteardown/reapers_quota_override_current_behaviour_test.go

@@ -0,0 +1,61 @@
+package accountteardown
+
+//test:in-package — drives the real final-stage reapers against the real quota service to pin that teardown leaves the override bucket alone.
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// No reaper in this package references the quota override bucket or calls an
+// override delete. Running the real engine and the real StageAccount reapers
+// against an account with a raised vCPU override, with usage wired, shows the
+// consequence: the override outlives the account record it belongs to, while
+// the usage counter the reapers do own is removed.
+func TestCurrentBehaviour_AccountTermination_QuotaOverridesSurviveTeardown(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := testCtx(t)
+
+	usage, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountUsage, 1)
+	require.NoError(t, err)
+	_, err = usage.Put(ctx, testAccountID, []byte(`{"vcpus":4}`))
+	require.NoError(t, err)
+
+	overridesKV, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountQuota, 1)
+	require.NoError(t, err)
+	svc := quota.New(quota.Limits{Enabled: true, VCPUs: 16}, usage)
+	svc.SetOverrides(overridesKV)
+	unlimited := quota.Unlimited
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccountID, quota.Overrides{VCPUs: &unlimited}, "operator"))
+
+	accounts := newFakeAccounts()
+	iam := &accountIAM{account: &handlers_iam.Account{AccountID: testAccountID, AccountName: testAccountName}}
+	engine := quietEngine(accounts, AccountReapers(iam, nil, usage)...)
+
+	result, err := engine.Teardown(ctx, request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.True(t, accounts.deleted)
+	assert.Zero(t, result.StuckCount())
+
+	// The reaper this package does own removed the usage counter, which shows
+	// which store teardown actually clears.
+	_, err = usage.Get(ctx, testAccountID)
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "teardown's quota-usage reaper must remove the vCPU counter")
+
+	_, err = overridesKV.Get(ctx, testAccountID)
+	require.NoError(t, err, "the override key must still be present after the account is torn down")
+
+	over, effective, err := svc.GetAccountQuota(ctx, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	assert.Equal(t, quota.Unlimited, *over.VCPUs)
+	assert.Equal(t, quota.Unlimited, effective.VCPUs)
+}

spinifex/daemon/daemon.go

@@ -1026,6 +1026,7 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.GetSerialConsoleAccessStatus", handleNATSRequest(d.node, d.accountService.GetSerialConsoleAccessStatus), "spinifex-workers"},
 		{"ec2.EnableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.EnableSerialConsoleAccess), "spinifex-workers"},
 		{"ec2.DisableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.DisableSerialConsoleAccess), "spinifex-workers"},
+		{ec2v1.SealAccountSettingsSubject, handleNATSRequest(d.node, d.sealAccountSettings), "spinifex-workers"},
 		{clusterv1.NodeHealthSubject(d.node), d.handleHealthCheck, ""},
 		{clusterv1.NodesDiscoverSubject, d.handleNodeDiscover, ""},
 		{clusterv1.NodeStatusSubject, d.handleNodeStatus, ""},
@@ -1825,9 +1826,6 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("failed to get tags KV bucket: %w", err)
 	}
 	d.tagsService = ec2tags.NewTagsServiceImpl(d.config, tagsKV)
-	// Key pairs keep their creation tags in their own metadata; project them so
-	// describe-tags sees them, and clear them when the key pair is deleted.
-	d.keyService.SetCentralTagStore(d.tagsService)
 
 	d.eigwService, err = initServiceWithRetry("EIGW service", func() (*ec2eigw.EgressOnlyIGWServiceImpl, error) {
 		return ec2eigw.NewEgressOnlyIGWServiceImplWithNATS(d.ctx, d.config, d.natsConn)
@@ -1856,6 +1854,7 @@ func (d *Daemon) startCluster() error {
 	if err != nil {
 		return fmt.Errorf("failed to initialize launch template service: %w", err)
 	}
+	d.wirePlatformTagIndex()
 
 	d.spotInstanceService, err = initServiceWithRetry("spot instance service", func() (*ec2spotinstance.SpotInstanceServiceImpl, error) {
 		return ec2spotinstance.NewSpotInstanceServiceImplWithNATS(d.ctx, d.config, d.natsConn)
@@ -1994,6 +1993,12 @@ func (d *Daemon) startCluster() error {
 	if err != nil {
 		return fmt.Errorf("failed to initialize account settings service: %w", err)
 	}
+	accountStatusJS, err := jetstream.New(d.natsConn)
+	if err != nil {
+		return fmt.Errorf("jetstream handle for account status reader: %w", err)
+	}
+	var _ ec2account.AccountTerminationChecker = (*handlers_iam.AccountStatusReader)(nil)
+	d.accountService.SetAccountTerminationChecker(handlers_iam.NewAccountStatusReader(accountStatusJS))
 
 	// ELBv2 resolves listener certificate private keys through its own ACM
 	// Store over the same JetStream bucket the ACM service writes, so it needs
@@ -3628,3 +3633,12 @@ func probeGPU() gpuProbeResult {
 	r.Capable = len(r.Devices) > 0 && ((r.IOMMUActive && r.VFIOPresent) || hasMIG)
 	return r
 }
+
+// wirePlatformTagIndex gives the key-pair, placement-group and launch-template
+// owners the tag index their deletes clear; key pairs also project their
+// metadata-held creation tags into it so describe-tags sees them.
+func (d *Daemon) wirePlatformTagIndex() {
+	d.keyService.SetCentralTagStore(d.tagsService)
+	d.placementGroupService.SetCentralTagStore(d.tagsService)
+	d.launchTemplateService.SetCentralTagStore(d.tagsService)
+}

spinifex/daemon/daemon_handlers_account_settings_seal.go

@@ -0,0 +1,23 @@
+package daemon
+
+import (
+	"context"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+)
+
+// sealAccountSettings serves the irreversible EC2 settings seal. The account
+// comes from the request body, not X-Account-ID: the sealed account is part
+// of the versioned contract, so the header is ignored.
+func (d *Daemon) sealAccountSettings(ctx context.Context, req *ec2v1.SealAccountSettingsRequest, _ string) (*ec2v1.SealAccountSettingsReply, error) {
+	seal, err := d.accountService.SealAccount(ctx, req.AccountID)
+	if err != nil {
+		return nil, err
+	}
+	return &ec2v1.SealAccountSettingsReply{
+		AccountID:      req.AccountID,
+		CorrelationRef: req.CorrelationRef,
+		Disposition:    ec2v1.AccountSettingsDispositionSealed,
+		Revision:       seal.Revision,
+	}, nil
+}

spinifex/daemon/daemon_handlers_account_settings_seal_test.go

@@ -0,0 +1,101 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sealTestChecker answers AccountTerminating from a fixed set, standing in
+// for IAM in these daemon-level tests.
+type sealTestChecker struct {
+	terminating map[string]bool
+}
+
+func (c *sealTestChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	return c.terminating[accountID], nil
+}
+
+// newSealTestDaemon builds a daemon serving its real subscription table over
+// a real account settings service, wired with checker as its termination
+// precondition.
+func newSealTestDaemon(t *testing.T, checker ec2account.AccountTerminationChecker) (*Daemon, *nats.Conn) {
+	t.Helper()
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	acctSvc, err := ec2account.NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	acctSvc.SetAccountTerminationChecker(checker)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.accountService = acctSvc
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	return d, caller
+}
+
+const (
+	sealTestTerminating = "111122223333"
+	sealTestActive      = "444455556666"
+)
+
+func sealRequest(t *testing.T, nc *nats.Conn, body any, header string) []byte {
+	t.Helper()
+	data, err := json.Marshal(body)
+	require.NoError(t, err)
+	reply, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, data, header, 5*time.Second)
+	require.NoError(t, err)
+	return reply.Data
+}
+
+// The header is ignored: the body account is what gets sealed, even when
+// X-Account-ID names a different one.
+func TestSealAccountSettingsSubjectSealsTheBodyAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestTerminating, CorrelationRef: "req-1"}, sealTestActive)
+	require.Empty(t, replyErrCode(t, raw), string(raw))
+
+	var reply ec2v1.SealAccountSettingsReply
+	require.NoError(t, json.Unmarshal(raw, &reply))
+	assert.Equal(t, sealTestTerminating, reply.AccountID)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+	assert.Equal(t, ec2v1.AccountSettingsDispositionSealed, reply.Disposition)
+	assert.NotZero(t, reply.Revision)
+}
+
+// Only an account IAM holds as TERMINATING may be sealed; ACTIVE is refused
+// before any write.
+func TestSealAccountSettingsSubjectRefusesAnActiveAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestActive}, "")
+	assert.Equal(t, awserrors.ErrorIncorrectState, replyErrCode(t, raw))
+}
+
+// A request without a valid account is refused, not sealed.
+func TestSealAccountSettingsSubjectRefusesAMissingAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	assert.Equal(t, awserrors.ErrorMissingParameter, replyErrCode(t, sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{}, "")))
+}
+
+// A daemon that does not serve the subject leaves the request with no responder.
+func TestSealAccountSettingsSubjectHasNoResponderWithoutTheDaemon(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, []byte(`{"account_id":"111122223333"}`), "", time.Second)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+}

spinifex/daemon/daemon_handlers_platform_inventory_test.go

@@ -164,6 +164,9 @@ func newInventoryCluster(t *testing.T) *inventoryCluster {
 	d.placementGroupService = groups
 	d.launchTemplateService = templates
 	require.NoError(t, d.subscribeAll())
+	// The caller is a separate connection, so the server must hold the SUBs
+	// before it publishes or the request can see no responders.
+	require.NoError(t, nc.Flush())
 
 	c.caller, err = nats.Connect(ns.ClientURL())
 	require.NoError(t, err)

spinifex/daemon/platform_disclosure_current_behaviour_test.go

@@ -7,7 +7,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"slices"
-	"strings"
 	"testing"
 	"time"
 
@@ -56,12 +55,15 @@ func newDisclosureCluster(t *testing.T) *disclosureCluster {
 	require.NoError(t, err)
 	d.keyService = ec2key.NewKeyServiceImplWithStore(store, disclosureBucket)
 	d.tagsService = ec2tags.NewTagsServiceImplWithStore(cfg, store, tagsKV)
-	d.keyService.SetCentralTagStore(d.tagsService)
 	d.placementGroupService, err = ec2placementgroup.NewPlacementGroupServiceImplWithNATS(t.Context(), nil, nc)
 	require.NoError(t, err)
 	d.launchTemplateService, err = ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(t.Context(), nil, nc)
 	require.NoError(t, err)
+	d.wirePlatformTagIndex()
 	require.NoError(t, d.subscribeAll())
+	// The caller is a separate connection, so the server must hold the SUBs
+	// before it publishes or the request can see no responders.
+	require.NoError(t, nc.Flush())
 
 	caller, err := nats.Connect(ns.ClientURL())
 	require.NoError(t, err)
@@ -225,12 +227,13 @@ func TestCurrentBehaviour_AccountTermination_DirectNATSCallerCannotSeeHiddenResi
 	assert.Equal(t, before.taggedResources, after.taggedResources)
 }
 
-// Pins current behaviour that differs from AWS: deleting a placement group or
-// launch template leaves its tag-index entry, which DescribeTags keeps
-// returning with its values. A key-pair delete clears its entry.
-func TestCurrentBehaviour_AccountTermination_PlatformDeletesLeaveTagIndexEntries(t *testing.T) {
+// Deleting a key pair, placement group or launch template clears its
+// tag-index entry, so DescribeTags no longer returns its rows, as in AWS.
+func TestPlatformDeletesClearTagIndexEntries(t *testing.T) {
 	c := newDisclosureCluster(t)
 	s := c.seed(t, disclosureAccount, "a")
+	before := c.describe(t, disclosureAccount)
+	require.Equal(t, sortedIDs(s.keyID, s.groupID, s.templateID), before.taggedResources)
 
 	c.call(t, "ec2.DeleteKeyPair", disclosureAccount, &ec2.DeleteKeyPairInput{KeyName: aws.String(s.keyName)}, nil)
 	c.call(t, "ec2.DeletePlacementGroup", disclosureAccount, &ec2.DeletePlacementGroupInput{GroupName: aws.String(s.groupName)}, nil)
@@ -240,13 +243,8 @@ func TestCurrentBehaviour_AccountTermination_PlatformDeletesLeaveTagIndexEntries
 	assert.Empty(t, got.keyPairs)
 	assert.Empty(t, got.placementGroups)
 	assert.Empty(t, got.launchTemplates)
-	assert.Equal(t, sortedIDs(s.groupID, s.templateID), got.taggedResources)
-
+	assert.Empty(t, got.taggedResources)
 	var tags ec2.DescribeTagsOutput
 	c.call(t, "ec2.DescribeTags", disclosureAccount, &ec2.DescribeTagsInput{}, &tags)
-	for _, row := range tags.Tags {
-		assert.Equal(t, "owner", aws.StringValue(row.Key))
-		assert.Equal(t, "a", aws.StringValue(row.Value))
-		assert.True(t, strings.HasPrefix(aws.StringValue(row.ResourceId), "pg-") || strings.HasPrefix(aws.StringValue(row.ResourceId), "lt-"))
-	}
+	assert.Empty(t, tags.Tags)
 }

spinifex/daemon/vm_adapters_trace_test.go

@@ -73,7 +73,9 @@ func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 	sr := withRecordedSpans(t)
 	daemon := createTestDaemon(t, sharedNATSURL)
 
+	consumerEnded := make(chan struct{})
 	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		defer close(consumerEnded)
 		// Mirrors viperblockd's ebs.unmount handler: open a consumer span
 		// joining whatever trace context the producer injected.
 		_, span := natsmsg.StartConsumerSpan(msg)
@@ -89,6 +91,9 @@ func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 
 	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
 	require.NoError(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-linked"}))
+	// The handler replies before its deferred span.End runs, so the consumer
+	// span may not be recorded yet when UnmountOne returns.
+	<-consumerEnded
 
 	spans := sr.Ended()
 	require.Len(t, spans, 2, "expected one producer span and one consumer span")

spinifex/domains/admission/quota/cleanup.go

@@ -0,0 +1,93 @@
+package quota
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// OverrideCleanupDisposition is CleanupAccount's outcome for an account's
+// quota override key.
+type OverrideCleanupDisposition string
+
+const (
+	OverrideCleanupDeleted OverrideCleanupDisposition = "deleted"
+	OverrideCleanupAbsent  OverrideCleanupDisposition = "absent"
+	OverrideCleanupFailed  OverrideCleanupDisposition = "failed"
+)
+
+// evidenceRevisionGuardedDelete and evidenceObservedAbsent are the fixed
+// evidence bases CleanupAccount reports; see kvstore/bucket.go:112-120 for why
+// a plain read cannot support a stronger absence claim.
+const (
+	evidenceRevisionGuardedDelete = "revision-guarded-delete"
+	evidenceObservedAbsent        = "observed-absent-at-scan"
+)
+
+// cleanupMaxAttempts bounds retries against a revision that keeps moving
+// under concurrent writes, so cleanup fails closed instead of spinning.
+const cleanupMaxAttempts = 5
+
+// OverrideCleanup is CleanupAccount's result: what happened to an account's
+// quota override key, and the fixed evidence basis for that claim.
+type OverrideCleanup struct {
+	Disposition OverrideCleanupDisposition
+	Evidence    string
+	Revision    uint64
+}
+
+// CleanupAccount removes accountID's quota override key so a reused account ID
+// does not inherit a deleted account's raised or lowered limits. It is
+// unwired: nothing calls it yet.
+func (s *Service) CleanupAccount(ctx context.Context, accountID string) (*OverrideCleanup, error) {
+	if accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required for quota override cleanup")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
+	}
+	if accountID == awsidentifiers.GlobalAccountID {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "quota override cleanup must not target the global account")
+	}
+	if s == nil || s.overrides == nil {
+		return &OverrideCleanup{Disposition: OverrideCleanupFailed}, errors.New("quota: override bucket not configured")
+	}
+
+	// accountID is never resolved through any fallback here: cleanup must act
+	// on exactly accountID and never fall back to the global key.
+	for range cleanupMaxAttempts {
+		entry, err := s.overrides.Get(ctx, accountID)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				return &OverrideCleanup{Disposition: OverrideCleanupAbsent, Evidence: evidenceObservedAbsent}, nil
+			}
+			return &OverrideCleanup{Disposition: OverrideCleanupFailed}, fmt.Errorf("get quota override for cleanup: %w", err)
+		}
+		rev := entry.Revision()
+
+		err = s.overrides.Delete(ctx, accountID, jetstream.LastRevision(rev))
+		switch {
+		case err == nil:
+			return &OverrideCleanup{
+				Disposition: OverrideCleanupDeleted,
+				Evidence:    evidenceRevisionGuardedDelete,
+				Revision:    rev,
+			}, nil
+		case errors.Is(err, jetstream.ErrKeyNotFound):
+			return &OverrideCleanup{Disposition: OverrideCleanupAbsent, Evidence: evidenceObservedAbsent}, nil
+		case errors.Is(err, jetstream.ErrKeyRevisionMismatch):
+			// A concurrent write landed between the read and the guarded
+			// delete; retry against whatever revision now holds.
+			continue
+		default:
+			return &OverrideCleanup{Disposition: OverrideCleanupFailed}, fmt.Errorf("delete quota override for cleanup: %w", err)
+		}
+	}
+
+	return &OverrideCleanup{Disposition: OverrideCleanupFailed},
+		fmt.Errorf("quota override cleanup for account: exhausted %d attempts against concurrent writes", cleanupMaxAttempts)
+}

spinifex/domains/admission/quota/cleanup_test.go

@@ -0,0 +1,184 @@
+//test:in-package — CleanupAccount deletes through the unexported overrides field, and the retry/race behaviour is only observable from inside the package.
+package quota
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// overrideCleanupFakeKV wraps a real KeyValue so a test can inject a race
+// between CleanupAccount's read and its guarded delete, or a store failure,
+// deterministically rather than through goroutine timing.
+type overrideCleanupFakeKV struct {
+	jetstream.KeyValue
+
+	deleteCalls  int
+	beforeDelete func(attempt int) error
+	getErr       error
+}
+
+func (f *overrideCleanupFakeKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
+	if f.getErr != nil {
+		return nil, f.getErr
+	}
+	return f.KeyValue.Get(ctx, key)
+}
+
+func (f *overrideCleanupFakeKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	f.deleteCalls++
+	if f.beforeDelete != nil {
+		if err := f.beforeDelete(f.deleteCalls); err != nil {
+			return err
+		}
+	}
+	return f.KeyValue.Delete(ctx, key, opts...)
+}
+
+func TestCleanupAccount_DeletesTheOverrideKey(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	ctx := t.Context()
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator"))
+
+	result, err := svc.CleanupAccount(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupDeleted, result.Disposition)
+	assert.Equal(t, evidenceRevisionGuardedDelete, result.Evidence)
+	assert.NotZero(t, result.Revision)
+
+	_, err = svc.overrides.Get(ctx, testAccount)
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+}
+
+func TestCleanupAccount_RepeatCallIsIdempotent(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	ctx := t.Context()
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator"))
+
+	first, err := svc.CleanupAccount(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupDeleted, first.Disposition)
+
+	second, err := svc.CleanupAccount(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupAbsent, second.Disposition)
+	assert.Equal(t, evidenceObservedAbsent, second.Evidence)
+}
+
+func TestCleanupAccount_AbsentKeyIsIdempotent(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	result, err := svc.CleanupAccount(t.Context(), "333333333333")
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupAbsent, result.Disposition)
+	assert.Equal(t, evidenceObservedAbsent, result.Evidence)
+}
+
+func TestCleanupAccount_OtherAccountsUntouched(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	ctx := t.Context()
+	accountA, accountB := testAccount, "222222222222"
+	require.NoError(t, svc.PutAccountQuota(ctx, accountA, Overrides{VCPUs: ptr(32)}, "operator"))
+	require.NoError(t, svc.PutAccountQuota(ctx, accountB, Overrides{VCPUs: ptr(8)}, "operator"))
+
+	result, err := svc.CleanupAccount(ctx, accountA)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupDeleted, result.Disposition)
+
+	over, _, err := svc.GetAccountQuota(ctx, accountB)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	assert.Equal(t, 8, *over.VCPUs)
+}
+
+func TestCleanupAccount_RefusesGlobalAccountID(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	result, err := svc.CleanupAccount(t.Context(), awsidentifiers.GlobalAccountID)
+	assert.Error(t, err)
+	assert.Nil(t, result)
+}
+
+func TestCleanupAccount_RefusesEmptyAccountID(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	result, err := svc.CleanupAccount(t.Context(), "")
+	assert.Error(t, err)
+	assert.Nil(t, result)
+}
+
+func TestCleanupAccount_RefusesInvalidAccountID(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	result, err := svc.CleanupAccount(t.Context(), "not-an-account")
+	assert.Error(t, err)
+	assert.Nil(t, result)
+}
+
+func TestCleanupAccount_WithoutBucketFails(t *testing.T) {
+	svc := New(baseLimits(), nil)
+	result, err := svc.CleanupAccount(t.Context(), testAccount)
+	assert.Error(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, OverrideCleanupFailed, result.Disposition)
+}
+
+// A concurrent Put landing between CleanupAccount's read and its
+// revision-guarded delete must not be lost silently: the delete is rejected
+// and retried against the revision the write left behind.
+func TestCleanupAccount_ConcurrentPutDuringCleanupRetriesAndDeletes(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	ctx := t.Context()
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator"))
+
+	fake := &overrideCleanupFakeKV{KeyValue: svc.overrides}
+	fake.beforeDelete = func(attempt int) error {
+		if attempt != 1 {
+			return nil
+		}
+		// Simulate a racing override write landing after CleanupAccount's
+		// read but before its guarded delete, through the real KV so the
+		// delete's expected revision is now stale.
+		return svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator")
+	}
+	svc.overrides = fake
+
+	result, err := svc.CleanupAccount(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideCleanupDeleted, result.Disposition)
+	assert.Equal(t, 2, fake.deleteCalls, "the first delete is rejected by the race and retried once")
+}
+
+// A revision that keeps moving under sustained concurrent writes must not
+// spin forever: cleanup gives up and reports failed, bounded by attempts.
+func TestCleanupAccount_PersistentConcurrentWritesFailsBounded(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	ctx := t.Context()
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator"))
+
+	fake := &overrideCleanupFakeKV{KeyValue: svc.overrides}
+	fake.beforeDelete = func(int) error {
+		return svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator")
+	}
+	svc.overrides = fake
+
+	result, err := svc.CleanupAccount(ctx, testAccount)
+	assert.Error(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, OverrideCleanupFailed, result.Disposition)
+	assert.Equal(t, cleanupMaxAttempts, fake.deleteCalls)
+}
+
+// A store error must report failed, never absent: absence is a claim about
+// what the store holds, which an error gives no basis to make.
+func TestCleanupAccount_StoreErrorIsFailedNotAbsent(t *testing.T) {
+	svc, _ := newOverrideService(t)
+	fake := &overrideCleanupFakeKV{KeyValue: svc.overrides, getErr: errors.New("kv unavailable")}
+	svc.overrides = fake
+
+	result, err := svc.CleanupAccount(t.Context(), testAccount)
+	assert.Error(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, OverrideCleanupFailed, result.Disposition)
+}

spinifex/domains/ec2/account/seal.go

@@ -0,0 +1,135 @@
+package account
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// sealMaxAttempts bounds SealAccount's retry loop against a revision that
+// keeps moving under concurrent writes, so it fails closed instead of
+// spinning.
+const sealMaxAttempts = 5
+
+// sealedRecord is the fixed value a seal writes. Clearing both booleans
+// retains no tenant setting; an older reader that ignores state sees the
+// AWS defaults.
+var sealedRecord = &AccountSettingsRecord{State: AccountSettingsStateSealed}
+
+// AccountSettingsSeal is SealAccount's result: the revision at which the
+// stream leader accepted the sealed value.
+type AccountSettingsSeal struct {
+	Revision uint64
+}
+
+// AccountTerminationChecker reports whether IAM holds accountID as TERMINATING.
+// A missing record is (false, nil); only a failed read is an error.
+type AccountTerminationChecker interface {
+	AccountTerminating(ctx context.Context, accountID string) (bool, error)
+}
+
+// SetAccountTerminationChecker wires the precondition; until it is set, SealAccount refuses.
+func (s *AccountSettingsServiceImpl) SetAccountTerminationChecker(c AccountTerminationChecker) {
+	s.terminationChecker = c
+}
+
+// checkTerminating runs before any settings-KV call, so a refusal never writes.
+// Not TERMINATING, including a missing IAM record, is IncorrectState; a read
+// failure or an unwired checker is InternalError.
+func (s *AccountSettingsServiceImpl) checkTerminating(ctx context.Context, accountID string) error {
+	if s.terminationChecker == nil {
+		slog.ErrorContext(ctx, "Account settings seal has no termination checker wired", "accountID", accountID)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+
+	terminating, err := s.terminationChecker.AccountTerminating(ctx, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read account status for seal precondition", "accountID", accountID, "err", err)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	if !terminating {
+		return awserrors.Errorf(awserrors.ErrorIncorrectState, "account %s is not TERMINATING", accountID)
+	}
+	return nil
+}
+
+// isRevisionConflict reports a lost CAS race SealAccount should retry. Besides
+// the two sentinels, Create's retry against a delete marker returns a bare
+// *jetstream.APIError carrying a wrong-last-sequence code.
+func isRevisionConflict(err error) bool {
+	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		return true
+	}
+	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
+		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
+			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
+	}
+	return false
+}
+
+// SealAccount seals accountID's settings record so no later mutation can
+// recreate it. It never relies on a read for its claim: the only success is a
+// leader-accepted write, evidenced by the returned revision.
+func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID string) (*AccountSettingsSeal, error) {
+	if accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required to seal settings")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
+	}
+	if accountID == awsidentifiers.GlobalAccountID {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "settings seal must not target the global account")
+	}
+
+	if err := s.checkTerminating(ctx, accountID); err != nil {
+		return nil, err
+	}
+
+	data, err := json.Marshal(sealedRecord)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to encode sealed account settings record", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+
+	// settingsKey is never used here: a seal must act on exactly accountID
+	// and never fall back to the global key.
+	for range sealMaxAttempts {
+		var rev uint64
+		entry, getErr := s.settingsKV.Get(ctx, accountID)
+		switch {
+		case getErr == nil:
+			rev = entry.Revision()
+		case errors.Is(getErr, jetstream.ErrKeyNotFound):
+			rev = 0
+		default:
+			slog.ErrorContext(ctx, "Failed to read account settings for seal", "accountID", accountID, "err", getErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+
+		var newRev uint64
+		var writeErr error
+		if rev == 0 {
+			newRev, writeErr = s.settingsKV.Create(ctx, accountID, data)
+		} else {
+			newRev, writeErr = s.settingsKV.Update(ctx, accountID, data, rev)
+		}
+
+		switch {
+		case writeErr == nil:
+			return &AccountSettingsSeal{Revision: newRev}, nil
+		case isRevisionConflict(writeErr):
+			continue
+		default:
+			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+	}
+
+	slog.ErrorContext(ctx, "Account settings seal exhausted attempts against concurrent writes", "accountID", accountID, "attempts", sealMaxAttempts)
+	return nil, errors.New(awserrors.ErrorInternalError)
+}

spinifex/domains/ec2/account/seal_test.go

@@ -0,0 +1,559 @@
+package account
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubTerminationChecker answers AccountTerminating from a fixed set, for
+// tests that exercise the KV paths rather than IAM itself.
+type stubTerminationChecker struct {
+	terminating map[string]bool
+	err         error
+}
+
+func (c *stubTerminationChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	if c.err != nil {
+		return false, c.err
+	}
+	return c.terminating[accountID], nil
+}
+
+// terminatingChecker answers true for exactly accountID.
+func terminatingChecker(accountID string) *stubTerminationChecker {
+	return &stubTerminationChecker{terminating: map[string]bool{accountID: true}}
+}
+
+// lastSettingsSeq reports the account-settings stream's last sequence, so a
+// test can assert a failed attempt wrote nothing.
+func lastSettingsSeq(t *testing.T, svc *AccountSettingsServiceImpl) uint64 {
+	t.Helper()
+	stream, err := svc.js.Stream(context.Background(), "KV_"+KVBucketAccountSettings)
+	require.NoError(t, err)
+	info, err := stream.Info(context.Background())
+	require.NoError(t, err)
+	return info.State.LastSeq
+}
+
+// mutationCase is one of the four settings mutations, called uniformly so a
+// test can table over them.
+type mutationCase struct {
+	name string
+	call func(svc *AccountSettingsServiceImpl, ctx context.Context) error
+}
+
+func mutationTable() []mutationCase {
+	return []mutationCase{
+		{"EnableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+			return err
+		}},
+		{"DisableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+			return err
+		}},
+		{"EnableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+			return err
+		}},
+		{"DisableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
+			return err
+		}},
+	}
+}
+
+// seedSettingsState prepares testAccountID's record for a race test: present
+// (an active record), absent (never written) or tombstone (written then
+// deleted, leaving a delete marker Get hides).
+func seedSettingsState(t *testing.T, svc *AccountSettingsServiceImpl, state string) {
+	t.Helper()
+	ctx := context.Background()
+	switch state {
+	case "present":
+		_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+		require.NoError(t, err)
+	case "tombstone":
+		_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+		require.NoError(t, err)
+		require.NoError(t, svc.settingsKV.Delete(ctx, testAccountID))
+	case "absent":
+	default:
+		t.Fatalf("unknown seed state %q", state)
+	}
+}
+
+// sealRaceKV wraps a real KeyValue so a test can inject a deterministic race
+// between a caller's read and its guarded write (Create or Update), instead
+// of relying on goroutine timing.
+type sealRaceKV struct {
+	jetstream.KeyValue
+
+	writeCalls  int
+	beforeWrite func(attempt int) error
+}
+
+func (f *sealRaceKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Create(ctx, key, value, opts...)
+}
+
+func (f *sealRaceKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Update(ctx, key, value, revision)
+}
+
+// staleGetKV always answers Get with a fixed snapshot taken at construction,
+// so a caller's CAS keeps retrying against a revision that can never win.
+type staleGetKV struct {
+	jetstream.KeyValue
+
+	entry jetstream.KeyValueEntry
+}
+
+func (f *staleGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return f.entry, nil
+}
+
+// staleNotFoundKV always answers Get as absent, regardless of what the real
+// bucket holds.
+type staleNotFoundKV struct {
+	jetstream.KeyValue
+}
+
+func (f *staleNotFoundKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, jetstream.ErrKeyNotFound
+}
+
+// errGetKV always fails Get with a fixed store error.
+type errGetKV struct {
+	jetstream.KeyValue
+
+	err error
+}
+
+func (f *errGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, f.err
+}
+
+func decodeSettings(t *testing.T, entry jetstream.KeyValueEntry) AccountSettingsRecord {
+	t.Helper()
+	var rec AccountSettingsRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	return rec
+}
+
+// A seal landing between a mutation's read and its guarded write is not
+// outlived: the mutation's retry sees the seal and is rejected, and nothing
+// it wrote survives.
+func TestSealRaceMutation_SealLandsBetweenReadAndWrite(t *testing.T) {
+	for _, m := range mutationTable() {
+		for _, state := range []string{"present", "absent", "tombstone"} {
+			t.Run(m.name+"_"+state, func(t *testing.T) {
+				svc := setupTestAccountService(t)
+				ctx := context.Background()
+				seedSettingsState(t, svc, state)
+
+				realKV := svc.settingsKV
+				sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: realKV}
+				sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+				var sealed *AccountSettingsSeal
+				race := &sealRaceKV{KeyValue: realKV}
+				race.beforeWrite = func(attempt int) error {
+					if attempt != 1 {
+						return nil
+					}
+					var sealErr error
+					sealed, sealErr = sealer.SealAccount(ctx, testAccountID)
+					return sealErr
+				}
+				svc.settingsKV = race
+
+				err := m.call(svc, ctx)
+				require.Error(t, err)
+				assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+				require.NotNil(t, sealed)
+
+				entry, err := realKV.Get(ctx, testAccountID)
+				require.NoError(t, err)
+				assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+				assert.Equal(t, sealed.Revision, entry.Revision(), "no write landed after the seal")
+			})
+		}
+	}
+}
+
+// A mutation landing inside a seal's read-write window does not survive it:
+// the seal retries against the mutation's write and wins, leaving the
+// sealed value with both booleans false.
+func TestSealRaceMutation_MutationLandsBetweenSealReadAndWrite(t *testing.T) {
+	for _, m := range mutationTable() {
+		for _, state := range []string{"present", "absent", "tombstone"} {
+			t.Run(m.name+"_"+state, func(t *testing.T) {
+				svc := setupTestAccountService(t)
+				ctx := context.Background()
+				seedSettingsState(t, svc, state)
+
+				realKV := svc.settingsKV
+				race := &sealRaceKV{KeyValue: realKV}
+				race.beforeWrite = func(attempt int) error {
+					if attempt != 1 {
+						return nil
+					}
+					_ = m.call(svc, ctx)
+					return nil
+				}
+
+				sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: race}
+				sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+				seal, err := sealer.SealAccount(ctx, testAccountID)
+				require.NoError(t, err)
+
+				entry, err := realKV.Get(ctx, testAccountID)
+				require.NoError(t, err)
+				rec := decodeSettings(t, entry)
+				assert.Equal(t, AccountSettingsStateSealed, rec.State)
+				assert.False(t, rec.EbsEncryptionByDefault)
+				assert.False(t, rec.SerialConsoleAccess)
+				assert.Equal(t, seal.Revision, entry.Revision())
+			})
+		}
+	}
+}
+
+// None of the four mutations can act on an already-sealed record.
+func TestMutationOnSealedRecordIsRejected(t *testing.T) {
+	for _, m := range mutationTable() {
+		t.Run(m.name, func(t *testing.T) {
+			svc := setupTestAccountService(t)
+			ctx := context.Background()
+			svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+			_, err := svc.SealAccount(ctx, testAccountID)
+			require.NoError(t, err)
+
+			before, err := svc.settingsKV.Get(ctx, testAccountID)
+			require.NoError(t, err)
+
+			err = m.call(svc, ctx)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+
+			after, err := svc.settingsKV.Get(ctx, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, before.Revision(), after.Revision())
+		})
+	}
+}
+
+// Every settings mutation races one seal on a real bucket; afterwards the
+// record is sealed and stays sealed, so no success among them could have
+// landed after the seal.
+func TestSealConcurrentMutations(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	mutations := mutationTable()
+	errs := make([]error, len(mutations))
+	var wg sync.WaitGroup
+	for i, m := range mutations {
+		wg.Add(1)
+		go func(i int, m mutationCase) {
+			defer wg.Done()
+			errs[i] = m.call(svc, ctx)
+		}(i, m)
+	}
+
+	var sealResult *AccountSettingsSeal
+	var sealErr error
+	wg.Go(func() {
+		sealResult, sealErr = svc.SealAccount(ctx, testAccountID)
+	})
+	wg.Wait()
+
+	require.NoError(t, sealErr)
+	require.NotNil(t, sealResult)
+	for i, err := range errs {
+		if err != nil {
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "mutation %s: got %v", mutations[i].name, err)
+		}
+	}
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+	assert.Equal(t, sealResult.Revision, entry.Revision())
+
+	_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+}
+
+// A Get pinned to a revision the record has already moved past exhausts the
+// mutation's retries without ever writing.
+func TestMutationStaleReadExhaustsWithoutWriting(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	stale, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &staleGetKV{KeyValue: svc.settingsKV, entry: stale}
+
+	_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+// A Get that always reports absent, while a sealed record actually exists,
+// never gets to recreate it: every Create attempt conflicts.
+func TestMutationStaleAbsentDoesNotRecreate(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &staleNotFoundKV{KeyValue: svc.settingsKV}
+
+	_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+// A revision that keeps moving under sustained concurrent writes must not
+// let the seal spin forever: it gives up, bounded by its attempt budget,
+// and its own write never lands.
+func TestSealSustainedConflictFailsWithoutWriting(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	realKV := svc.settingsKV
+	race := &sealRaceKV{KeyValue: realKV}
+	race.beforeWrite = func(int) error {
+		data, merr := json.Marshal(&AccountSettingsRecord{SerialConsoleAccess: true})
+		require.NoError(t, merr)
+		_, perr := realKV.Put(ctx, testAccountID, data)
+		return perr
+	}
+	svc.settingsKV = race
+
+	_, err = svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, sealMaxAttempts, race.writeCalls)
+
+	entry, err := realKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotEqual(t, AccountSettingsStateSealed, decodeSettings(t, entry).State, "the seal's own write never landed")
+}
+
+// A store error on the seal's read reports failed, never written.
+func TestSealStoreErrorWritesNothing(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &errGetKV{KeyValue: svc.settingsKV, err: errors.New("kv unavailable")}
+
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
+	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
+	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
+	assert.False(t, isRevisionConflict(nil))
+}
+
+// An account IAM does not hold as TERMINATING refuses the seal before any
+// settings-KV write, whether the key is present or absent.
+func TestSealRefusedForActiveAccount(t *testing.T) {
+	for _, state := range []string{"present", "absent"} {
+		t.Run(state, func(t *testing.T) {
+			svc := setupTestAccountService(t)
+			ctx := context.Background()
+			if state == "present" {
+				_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+				require.NoError(t, err)
+			}
+			svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+			before := lastSettingsSeq(t, svc)
+			_, err := svc.SealAccount(ctx, testAccountID)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+			assert.Equal(t, before, lastSettingsSeq(t, svc))
+		})
+	}
+}
+
+func TestSealRefusedForMissingIAMRecord(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealRefusedWhenIAMReadFails(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealRefusedWithNoCheckerWired(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealAcceptedForTerminatingAccount(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	seal, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotZero(t, seal.Revision)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+}
+
+// A sealed record stays sealed after the service is rebuilt over the same
+// JetStream, and every one of the six settings actions still refuses it.
+func TestSealSurvivesServiceReconstruction(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+
+	svc, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	first, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	rebuilt, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+
+	entry, err := rebuilt.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+
+	_, err = rebuilt.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.GetSerialConsoleAccessStatus(ctx, &ec2.GetSerialConsoleAccessStatusInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+
+	rebuilt.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	second, err := rebuilt.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Greater(t, second.Revision, first.Revision)
+}
+
+func TestSealAbsentKeyCreatesSealedRecord(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	_, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+
+	seal, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+}
+
+func TestSealRefusesEmptyInvalidAndGlobalAccount(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	_, err := svc.SealAccount(ctx, "")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+
+	_, err = svc.SealAccount(ctx, "not-an-account")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+
+	_, err = svc.SealAccount(ctx, awsidentifiers.GlobalAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+}
+
+func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	accountB := "222222222222"
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, accountB)
+	require.NoError(t, err)
+
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err = svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, accountB)
+	require.NoError(t, err)
+	assert.True(t, *out.EbsEncryptionByDefault)
+}

spinifex/domains/ec2/account/service_impl.go

@@ -5,12 +5,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"log/slog"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -20,14 +21,19 @@ import (
 const (
 	KVBucketAccountSettings        = "spinifex-ec2-account-settings"
 	KVBucketAccountSettingsVersion = 1
-	KeyEbsEncryptionDefault        = "ebs-encryption-default"
-	KeySerialConsoleAccess         = "serial-console-access"
 )
 
+// AccountSettingsState is a settings record's lifecycle state. "" is active;
+// a record written before the state field existed has none and stays active.
+type AccountSettingsState string
+
+const AccountSettingsStateSealed AccountSettingsState = "sealed"
+
 // AccountSettingsRecord represents stored account settings.
 type AccountSettingsRecord struct {
-	EbsEncryptionByDefault bool `json:"ebs_encryption_by_default"`
-	SerialConsoleAccess    bool `json:"serial_console_access"`
+	EbsEncryptionByDefault bool                 `json:"ebs_encryption_by_default"`
+	SerialConsoleAccess    bool                 `json:"serial_console_access"`
+	State                  AccountSettingsState `json:"state,omitempty"`
 }
 
 // AccountSettingsServiceImpl implements account settings operations with NATS JetStream persistence.
@@ -35,6 +41,10 @@ type AccountSettingsServiceImpl struct {
 	config     *config.Config
 	js         jetstream.JetStream
 	settingsKV jetstream.KeyValue
+
+	// terminationChecker is the seal precondition; SealAccount refuses until
+	// it is wired.
+	terminationChecker AccountTerminationChecker
 }
 
 var _ AccountSettingsService = (*AccountSettingsServiceImpl)(nil)
@@ -72,50 +82,99 @@ func settingsKey(accountID string) string {
 	return accountID
 }
 
-// getSettings retrieves current account settings.
+// errSettingsSealed marks a mutation or read that found a sealed record, so
+// callers map it to Blocked without logging it as a fault.
+var errSettingsSealed = errors.New(awserrors.ErrorBlocked)
+
+// errSettingsContended marks a mutation whose CAS retries were exhausted
+// under concurrent writers, bounding the retry loop instead of spinning.
+var errSettingsContended = errors.New("account settings: CAS retries exhausted")
+
+// checkActive rejects a sealed or unknown-state record; "" is active and
+// passes through. A mutation must never act on a record that failed this.
+func checkActive(rec *AccountSettingsRecord) error {
+	switch rec.State {
+	case "":
+		return nil
+	case AccountSettingsStateSealed:
+		return errSettingsSealed
+	default:
+		return fmt.Errorf("account settings: unknown state %q", rec.State)
+	}
+}
+
+// mapSettingsError turns a mutateSettings or getSettings failure into the AWS
+// error a caller returns: sealed is an expected, unlogged Blocked; everything
+// else is InternalError, logged with the account ID.
+func mapSettingsError(ctx context.Context, accountID string, err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, errSettingsSealed):
+		return errors.New(awserrors.ErrorBlocked)
+	case errors.Is(err, errSettingsContended):
+		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
+	case errors.Is(err, kvutil.ErrRead):
+		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrDecode):
+		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrEncode):
+		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrWrite):
+		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
+	default:
+		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
+	}
+	return errors.New(awserrors.ErrorInternalError)
+}
+
+// mutateSettings applies set to accountID's record at the revision it read.
+// changed is always true, so every success is a leader-accepted write over an
+// active value; a seal landing between the read and the write fails the CAS.
+func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
+	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
+		CreateIfAbsent: true,
+		Exhausted:      func(string, int) error { return errSettingsContended },
+	}, func(rec *AccountSettingsRecord) (bool, error) {
+		if err := checkActive(rec); err != nil {
+			return false, err
+		}
+		set(rec)
+		return true, nil
+	})
+	return mapSettingsError(ctx, accountID, err)
+}
+
+// getSettings retrieves current account settings, rejecting a sealed or
+// unknown-state record so a read never answers from behind a seal. Absent is
+// the AWS defaults; a read is a direct get and can be stale.
 func (s *AccountSettingsServiceImpl) getSettings(ctx context.Context, accountID string) (*AccountSettingsRecord, error) {
 	entry, err := s.settingsKV.Get(ctx, settingsKey(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return &AccountSettingsRecord{
-				EbsEncryptionByDefault: false,
-				SerialConsoleAccess:    false,
-			}, nil
+			return &AccountSettingsRecord{}, nil
 		}
-		return nil, fmt.Errorf("failed to get account settings: %w", err)
+		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 
 	var settings AccountSettingsRecord
 	if err := json.Unmarshal(entry.Value(), &settings); err != nil {
-		return nil, err
+		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+	if err := checkActive(&settings); err != nil {
+		return nil, mapSettingsError(ctx, accountID, err)
 	}
 
 	return &settings, nil
 }
 
-// saveSettings saves current account settings.
-func (s *AccountSettingsServiceImpl) saveSettings(ctx context.Context, settings *AccountSettingsRecord, accountID string) error {
-	data, err := json.Marshal(settings)
-	if err != nil {
-		return err
-	}
-
-	_, err = s.settingsKV.Put(ctx, settingsKey(accountID), data)
-	return err
-}
-
 // EnableEbsEncryptionByDefault enables EBS encryption by default for the account.
 func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, accountID string) (*ec2.EnableEbsEncryptionByDefaultOutput, error) {
 	slog.InfoContext(ctx, "EnableEbsEncryptionByDefault called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.EbsEncryptionByDefault = true
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = true }); err != nil {
 		return nil, err
 	}
 
@@ -128,14 +187,7 @@ func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Co
 func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, accountID string) (*ec2.DisableEbsEncryptionByDefaultOutput, error) {
 	slog.InfoContext(ctx, "DisableEbsEncryptionByDefault called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.EbsEncryptionByDefault = false
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = false }); err != nil {
 		return nil, err
 	}
 
@@ -176,14 +228,7 @@ func (s *AccountSettingsServiceImpl) GetSerialConsoleAccessStatus(ctx context.Co
 func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, accountID string) (*ec2.EnableSerialConsoleAccessOutput, error) {
 	slog.InfoContext(ctx, "EnableSerialConsoleAccess called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.SerialConsoleAccess = true
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = true }); err != nil {
 		return nil, err
 	}
 
@@ -196,14 +241,7 @@ func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Conte
 func (s *AccountSettingsServiceImpl) DisableSerialConsoleAccess(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, accountID string) (*ec2.DisableSerialConsoleAccessOutput, error) {
 	slog.InfoContext(ctx, "DisableSerialConsoleAccess called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.SerialConsoleAccess = false
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = false }); err != nil {
 		return nil, err
 	}
 

spinifex/domains/ec2/account/service_impl_test.go

@@ -2,11 +2,13 @@ package account
 
 import (
 	"context"
+	"encoding/json"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -143,3 +145,44 @@ func TestSettingsKey_EmptyAccountIDFallback(t *testing.T) {
 	assert.Equal(t, awsidentifiers.GlobalAccountID, settingsKey(""))
 	assert.Equal(t, "123456789012", settingsKey("123456789012"))
 }
+
+// A raw pre-seal record, with no state field at all, is readable and
+// mutable, and a mutation writes it back without adding one.
+func TestRecordWithoutStateIsActive(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":true,"serial_console_access":false}`))
+	require.NoError(t, err)
+
+	out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, *out.EbsEncryptionByDefault)
+
+	_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	require.NoError(t, err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotContains(t, string(entry.Value()), "state")
+}
+
+// A record with an unrecognised state fails closed rather than being treated
+// as active or sealed, for both a read and a mutation.
+func TestUnknownStateFailsClosed(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":false,"serial_console_access":false,"state":"quarantined"}`))
+	require.NoError(t, err)
+
+	_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+
+	_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	var rec AccountSettingsRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	assert.EqualValues(t, "quarantined", rec.State, "the unknown-state record was never overwritten")
+}

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -0,0 +1,277 @@
+// Package accountcleanup deletes the proven-safe subset of one account's EC2
+// platform resources through the owners' own delete paths: placement groups
+// with no members that no instance, spot request or live launch-template
+// version names; key pairs none of those name; and launch-template residue no
+// header owns. Live launch templates are never deleted: an EKS nodegroup may
+// name one and EC2 cannot see it. A reference check that cannot complete
+// deletes nothing of the kinds it guards. Every disposition is proved by an
+// owner-store re-scan, never by Describe.
+//
+// It adds no caller, checks no authority and holds no store or transport of
+// its own: the owners and reference readers are injected. Whoever binds it
+// decides who may invoke it. It is not exposed on any NATS subject.
+package accountcleanup
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// PlatformCleaner deletes one account's EC2 platform resources and reports
+// what the owner stores hold afterwards. It is destructive.
+type PlatformCleaner interface {
+	CleanupPlatform(ctx context.Context, req *Request) (*Result, error)
+}
+
+var _ PlatformCleaner = (*Cleanup)(nil)
+
+// KeyPairOwner is the key-pair owner's account scan and cleanup.
+type KeyPairOwner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2key.AccountScan, error)
+	CleanupAccount(ctx context.Context, accountID string, settledBefore time.Time, referenced func(keyName string) string) (*ec2key.AccountCleanup, error)
+}
+
+// PlacementGroupOwner is the placement-group owner's account scan and cleanup.
+type PlacementGroupOwner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2placementgroup.AccountScan, error)
+	CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*ec2placementgroup.AccountCleanup, error)
+}
+
+// LaunchTemplateOwner is the launch-template owner's account scan and residue
+// cleanup; live templates are never deleted by account cleanup.
+type LaunchTemplateOwner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2launchtemplate.AccountScan, error)
+	CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*ec2launchtemplate.AccountCleanup, error)
+}
+
+// TagIndexOwner lists and clears the account's tag-index entries.
+type TagIndexOwner interface {
+	ScanAccountResourceIDs(ctx context.Context, accountID string, prefixes ...string) ([]string, error)
+	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
+}
+
+var (
+	_ KeyPairOwner        = (*ec2key.KeyServiceImpl)(nil)
+	_ PlacementGroupOwner = (*ec2placementgroup.PlacementGroupServiceImpl)(nil)
+	_ LaunchTemplateOwner = (*ec2launchtemplate.LaunchTemplateServiceImpl)(nil)
+	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
+)
+
+// DefaultSettleWindow is how old headerless residue must be before it is
+// removed; anything younger may be a create still in flight.
+const DefaultSettleWindow = 10 * time.Minute
+
+// Cleanup composes the owner cleanups and re-scans.
+type Cleanup struct {
+	keys      KeyPairOwner
+	groups    PlacementGroupOwner
+	templates LaunchTemplateOwner
+	tags      TagIndexOwner
+	refs      ReferenceReaders
+
+	// SettleWindow overrides DefaultSettleWindow when positive.
+	SettleWindow time.Duration
+	now          func() time.Time
+}
+
+// New builds a cleanup over the four owners and the reference readers. A nil
+// reader counts as a failed check, so the kinds it guards delete nothing.
+func New(keys KeyPairOwner, groups PlacementGroupOwner, templates LaunchTemplateOwner, tags TagIndexOwner, refs ReferenceReaders) *Cleanup {
+	return &Cleanup{keys: keys, groups: groups, templates: templates, tags: tags, refs: refs, now: time.Now}
+}
+
+// Resource-ID prefixes of the three kinds in the tag index.
+const (
+	keyPairIDPrefix        = "key-"
+	placementGroupIDPrefix = "pg-"
+	launchTemplateIDPrefix = "lt-"
+)
+
+// CleanupPlatform deletes the account's key pairs, empty placement groups and
+// launch templates, removes settled residue and orphaned tag-index entries,
+// then re-scans every owner store to build the result. It is safe to re-run.
+func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
+	if req == nil || req.AccountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
+	}
+	if !awsidentifiers.IsAccountID(req.AccountID) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
+	}
+	start := time.Now()
+	window := c.SettleWindow
+	if window <= 0 {
+		window = DefaultSettleWindow
+	}
+	settledBefore := c.now().Add(-window)
+	acct := req.AccountID
+	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
+
+	refs := c.readReferences(ctx, acct)
+	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
+	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+
+	// Tags are listed before the owner re-scans, so a resource created in
+	// between is seen as owned and keeps its tags.
+	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	if tagErr != nil {
+		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
+		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+	}
+
+	owners := map[string]ownedIDs{
+		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
+		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
+		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	}
+	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+
+	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
+		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))
+	return res, nil
+}
+
+// ownedIDs is the set of IDs whose owner record a re-scan found, live or
+// unreadable; resolved is false unless the re-scan listed and read every
+// record, so an absent ID is not proof its tags are orphaned.
+type ownedIDs struct {
+	ids      map[string]bool
+	resolved bool
+}
+
+// kindRun accumulates one kind's report across its cleanup and re-scan.
+type kindRun struct {
+	report KindReport
+	found  bool
+}
+
+func (c *Cleanup) startKind(kind string) *kindRun {
+	return &kindRun{report: KindReport{Kind: kind, StartedAt: c.now().UTC()}}
+}
+
+func (k *kindRun) incomplete(reason string) {
+	if !slices.Contains(k.report.IncompleteReasons, reason) {
+		k.report.IncompleteReasons = append(k.report.IncompleteReasons, reason)
+	}
+}
+
+func (c *Cleanup) finishKind(k *kindRun) {
+	k.report.EndedAt = c.now().UTC()
+	switch {
+	case len(k.report.IncompleteReasons) > 0:
+		k.report.Status = ec2v1.PlatformStatusScanIncomplete
+	case k.found:
+		k.report.Status = ec2v1.PlatformStatusRecordsFound
+	default:
+		k.report.Status = ec2v1.PlatformStatusObservedClean
+	}
+}
+
+// attempts is what an owner said it did, by resource ID. kept holds items
+// left on purpose, with the fixed reason.
+type attempts struct {
+	deleted, failed map[string]bool
+	blocked         map[string]int
+	kept            map[string]string
+}
+
+func newAttempts() attempts {
+	return attempts{deleted: map[string]bool{}, failed: map[string]bool{}, blocked: map[string]int{}, kept: map[string]string{}}
+}
+
+// itemDisposition decides one ID from what the owner did and whether the
+// re-scan still found it. Only a complete re-scan that misses an ID is deleted;
+// an item kept on purpose stays blocked unless that re-scan shows it gone.
+func itemDisposition(id string, did attempts, present, rescanned bool) (string, string) {
+	switch {
+	case did.kept[id] != "" && (present || !rescanned):
+		return DispositionBlocked, did.kept[id]
+	case !rescanned:
+		return DispositionFailed, ReasonAbsenceUnproven
+	case !present:
+		return DispositionDeleted, ""
+	case did.blocked[id] > 0:
+		return DispositionBlocked, ReasonInstanceMembers
+	case did.failed[id]:
+		return DispositionFailed, ReasonOwnerDeleteFailed
+	case did.deleted[id]:
+		return DispositionFailed, ReasonStillPresent
+	default:
+		return DispositionResidue, ReasonAppearedDuringCleanup
+	}
+}
+
+func (r *Result) addItem(k *kindRun, id, disposition, reason string, members int) {
+	if disposition != DispositionDeleted {
+		k.found = true
+	}
+	r.Items = append(r.Items, Item{ResourceID: id, Kind: k.report.Kind, Disposition: disposition, Reason: reason, MemberCount: members})
+}
+
+func (r *Result) addCount(k *kindRun, class, disposition, reason string, n int) {
+	if n <= 0 {
+		return
+	}
+	if disposition != DispositionDeleted {
+		k.found = true
+	}
+	r.Counts = append(r.Counts, Count{Kind: k.report.Kind, Class: class, Disposition: disposition, Reason: reason, Count: n})
+}
+
+// residueCounts reports one headerless class: the proven decrease as deleted
+// (never more than the owner removed), and what remains split by cause, kept
+// records first, by reason in a fixed order.
+func (r *Result) residueCounts(k *kindRun, class string, before, after, removed, failed, unsettled int, kept map[string]int) {
+	r.addCount(k, class, DispositionDeleted, "", min(removed, max(0, before-after)))
+	left := after
+	for _, reason := range keptReasons {
+		n := min(left, kept[reason])
+		r.addCount(k, class, DispositionBlocked, reason, n)
+		left -= n
+	}
+	failedLeft := min(left, failed)
+	unsettledLeft := min(left-failedLeft, unsettled)
+	r.addCount(k, class, DispositionFailed, ReasonOwnerDeleteFailed, failedLeft)
+	r.addCount(k, class, DispositionResidue, ReasonRecentlyWritten, unsettledLeft)
+	r.addCount(k, class, DispositionFailed, ReasonStillPresent, left-failedLeft-unsettledLeft)
+}
+
+// keptReasons are the reasons an owner may keep a record, in report order.
+var keptReasons = []string{ReasonReferenceCheckFailed, ReasonInstanceReference, ReasonSpotRequestReference, ReasonLaunchTemplateReference}
+
+// sortedUnion returns the distinct IDs of all lists, sorted.
+func sortedUnion(lists ...[]string) []string {
+	var out []string
+	for _, l := range lists {
+		for _, id := range l {
+			if !slices.Contains(out, id) {
+				out = append(out, id)
+			}
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+func idPrefix(id string) string {
+	for _, prefix := range []string{keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix} {
+		if strings.HasPrefix(id, prefix) {
+			return prefix
+		}
+	}
+	return ""
+}

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -0,0 +1,777 @@
+package accountcleanup
+
+//test:in-package — the clock is injected to place residue outside the settle
+// window without waiting for it.
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testAccount  = "111122223333"
+	otherAccount = "999988887777"
+	testBucket   = "test-bucket"
+	testPubKey   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"
+	memberID     = "i-0123456789abcdef0"
+	staleTagID   = "pg-0stale000000000"
+)
+
+var errInjected = errors.New("injected failure")
+
+// faultyStore fails object deletes its predicate selects.
+type faultyStore struct {
+	*objectstore.MemoryObjectStore
+
+	failDelete func(key string) bool
+}
+
+var _ objectstore.ObjectStore = (*faultyStore)(nil)
+
+func (f *faultyStore) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if f.failDelete != nil && f.failDelete(aws.StringValue(in.Key)) {
+		return nil, errInjected
+	}
+	return f.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// owners are the real EC2 owner services over an embedded JetStream, wired to
+// the tag index as the daemon wires them.
+type owners struct {
+	store     *faultyStore
+	keys      *ec2key.KeyServiceImpl
+	groups    *ec2placementgroup.PlacementGroupServiceImpl
+	templates *ec2launchtemplate.LaunchTemplateServiceImpl
+	tags      *ec2tags.TagsServiceImpl
+	spot      *ec2spotinstance.SpotInstanceServiceImpl
+	instances *fakeInstances
+	js        jetstream.JetStream
+}
+
+// fakeInstances stands in for the cluster-wide instance listing, which needs
+// every daemon to answer; its own refusal is covered separately.
+type fakeInstances struct {
+	refs *References
+	err  error
+}
+
+func (f *fakeInstances) AccountInstanceReferences(context.Context, string) (*References, error) {
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.refs, nil
+}
+
+func newOwners(t *testing.T) *owners {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	o := &owners{store: &faultyStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}}
+	tagsKV, err := ec2tags.GetOrCreateTagsBucket(t.Context(), js)
+	require.NoError(t, err)
+	o.tags = ec2tags.NewTagsServiceImplWithStore(&config.Config{Predastore: config.PredastoreConfig{Bucket: testBucket}}, o.store, tagsKV)
+	o.keys = ec2key.NewKeyServiceImplWithStore(o.store, testBucket)
+	o.keys.SetCentralTagStore(o.tags)
+	o.groups, err = ec2placementgroup.NewPlacementGroupServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	o.groups.SetCentralTagStore(o.tags)
+	o.templates, err = ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	o.templates.SetCentralTagStore(o.tags)
+	o.spot, err = ec2spotinstance.NewSpotInstanceServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	o.instances = &fakeInstances{refs: newReferences()}
+	o.js = js
+	return o
+}
+
+// readers are the real spot and launch-template checks and the instance fake.
+func (o *owners) readers() ReferenceReaders {
+	return ReferenceReaders{Instances: o.instances, Spot: o.spot, Templates: o.templates}
+}
+
+func (o *owners) newCleanup() *Cleanup {
+	return New(o.keys, o.groups, o.templates, o.tags, o.readers())
+}
+
+// cleanup runs over the owners with residue of any age treated as settled.
+func (o *owners) cleanup(t *testing.T) *Result {
+	t.Helper()
+	return run(t, o.newCleanup())
+}
+
+func run(t *testing.T, c *Cleanup) *Result {
+	t.Helper()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, CorrelationRef: "ref-1"})
+	require.NoError(t, err)
+	return res
+}
+
+func (o *owners) importKey(t *testing.T, account, name string) string {
+	t.Helper()
+	out, err := o.keys.ImportKeyPair(t.Context(), &ec2.ImportKeyPairInput{
+		KeyName: aws.String(name), PublicKeyMaterial: []byte(testPubKey),
+		TagSpecifications: []*ec2.TagSpecification{{ResourceType: aws.String("key-pair"), Tags: secretTags()}},
+	}, account)
+	require.NoError(t, err)
+	return aws.StringValue(out.KeyPairId)
+}
+
+func (o *owners) createGroup(t *testing.T, account, name string) string {
+	t.Helper()
+	out, err := o.groups.CreatePlacementGroup(t.Context(), &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String(name), Strategy: aws.String(ec2.PlacementStrategyCluster),
+	}, account)
+	require.NoError(t, err)
+	id := aws.StringValue(out.PlacementGroup.GroupId)
+	o.tag(t, account, id)
+	return id
+}
+
+func (o *owners) addMember(t *testing.T, group string) {
+	t.Helper()
+	reserved, err := o.groups.ReserveClusterNode(t.Context(), &ec2placementgroup.ReserveClusterNodeInput{
+		GroupName: group, EligibleNodes: []string{"node-1"},
+	}, testAccount)
+	require.NoError(t, err)
+	_, err = o.groups.FinalizeClusterInstances(t.Context(), &ec2placementgroup.FinalizeClusterInstancesInput{
+		GroupName: group, NodeInstances: map[string][]string{reserved.TargetNode: {memberID}},
+	}, testAccount)
+	require.NoError(t, err)
+}
+
+func (o *owners) removeMember(t *testing.T, group string) {
+	t.Helper()
+	_, err := o.groups.RemoveInstance(t.Context(), &ec2placementgroup.RemoveInstanceInput{
+		GroupName: group, NodeName: "node-1", InstanceID: memberID,
+	}, testAccount)
+	require.NoError(t, err)
+}
+
+func (o *owners) createTemplate(t *testing.T, account, name string) string {
+	t.Helper()
+	out, err := o.templates.CreateLaunchTemplate(t.Context(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String(name),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			ImageId:  aws.String("ami-123"),
+			UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("secret-user-data"))),
+		},
+	}, account)
+	require.NoError(t, err)
+	id := aws.StringValue(out.LaunchTemplate.LaunchTemplateId)
+	o.tag(t, account, id)
+	return id
+}
+
+func secretTags() []*ec2.Tag {
+	return []*ec2.Tag{{Key: aws.String("secret-tag-key"), Value: aws.String("secret-tag-value")}}
+}
+
+func (o *owners) tag(t *testing.T, account, id string) {
+	t.Helper()
+	_, err := o.tags.CreateTags(t.Context(), &ec2.CreateTagsInput{Resources: []*string{aws.String(id)}, Tags: secretTags()}, account)
+	require.NoError(t, err)
+}
+
+func (o *owners) putObject(t *testing.T, key, body string) {
+	t.Helper()
+	_, err := o.store.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(testBucket), Key: aws.String("keys/" + testAccount + "/" + key), Body: strings.NewReader(body),
+	})
+	require.NoError(t, err)
+}
+
+func statuses(res *Result) map[string]string {
+	out := map[string]string{}
+	for _, k := range res.Kinds {
+		out[k.Kind] = k.Status
+	}
+	return out
+}
+
+func allStatus(status string) map[string]string {
+	return map[string]string{
+		ec2v1.PlatformKindKeyPair: status, ec2v1.PlatformKindPlacementGroup: status,
+		ec2v1.PlatformKindLaunchTemplate: status, ec2v1.PlatformKindTagIndex: status,
+	}
+}
+
+func item(id, kind, disposition, reason string) Item {
+	return Item{ResourceID: id, Kind: kind, Disposition: disposition, Reason: reason}
+}
+
+// One run deletes every unreferenced key pair and empty placement group with
+// its tag-index entry, removes settled hidden residue and orphaned tags,
+// blocks on a group holding an instance, and leaves another account alone.
+// The next run, once the member leaves, converges to clean.
+func TestCleanupPlatformDeletesBlocksAndConverges(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "secret-key-name")
+	emptyID := o.createGroup(t, testAccount, "secret-empty-group")
+	heldID := o.createGroup(t, testAccount, "secret-held-group")
+	o.addMember(t, "secret-held-group")
+	o.putObject(t, "secret-orphan-key", testPubKey)
+	o.tag(t, testAccount, staleTagID)
+	theirKey := o.importKey(t, otherAccount, "theirs")
+	o.createGroup(t, otherAccount, "theirs")
+	o.createTemplate(t, otherAccount, "theirs")
+
+	res := o.cleanup(t)
+
+	assert.Equal(t, testAccount, res.AccountID)
+	assert.Equal(t, "ref-1", res.CorrelationRef)
+	assert.ElementsMatch(t, []Item{
+		item(keyID, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(emptyID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, ""),
+		{ResourceID: heldID, Kind: ec2v1.PlatformKindPlacementGroup, Disposition: DispositionBlocked, Reason: ReasonInstanceMembers, MemberCount: 1},
+		item(staleTagID, ec2v1.PlatformKindTagIndex, DispositionDeleted, ""),
+	}, res.Items)
+	assert.Equal(t, []Count{{
+		Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassPublicKeyWithoutMetadata, Disposition: DispositionDeleted, Count: 1,
+	}}, res.Counts)
+	// The held group's own tags are its, not residue.
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindKeyPair:        ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusRecordsFound,
+		ec2v1.PlatformKindLaunchTemplate: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusRecordsFound,
+	}, statuses(res))
+
+	theirs, err := o.keys.ScanAccount(t.Context(), otherAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{theirKey}, theirs.LiveKeyPairIDs)
+	theirTags, err := o.tags.ScanAccountResourceIDs(t.Context(), otherAccount, "key-", "pg-", "lt-")
+	require.NoError(t, err)
+	assert.Len(t, theirTags, 3)
+
+	o.removeMember(t, "secret-held-group")
+	res = o.cleanup(t)
+	assert.Equal(t, []Item{item(heldID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items,
+		"items finished on an earlier run are not reported again")
+	assert.Empty(t, res.Counts)
+	assert.Equal(t, allStatus(ec2v1.PlatformStatusObservedClean), statuses(res))
+
+	res = o.cleanup(t)
+	assert.Empty(t, res.Items)
+	assert.Equal(t, allStatus(ec2v1.PlatformStatusObservedClean), statuses(res))
+}
+
+// The result carries IDs, kinds, dispositions, fixed reasons and counts only:
+// no names, tags, key material, user data, member instance IDs or error text.
+func TestCleanupPlatformResultDisclosesNoTenantData(t *testing.T) {
+	o := newOwners(t)
+	o.importKey(t, testAccount, "secret-key-name")
+	o.createGroup(t, testAccount, "secret-held-group")
+	o.addMember(t, "secret-held-group")
+	o.createTemplate(t, testAccount, "secret-template-name")
+	o.putObject(t, "secret-orphan-key", testPubKey)
+	o.putObject(t, "key-0badbadbadbadbad0.json", "{secret-broken")
+	failing := o.importKey(t, testAccount, "secret-failing-key")
+	o.store.failDelete = func(key string) bool { return strings.HasSuffix(key, failing+".json") }
+	o.importKey(t, testAccount, "secret-instance-key")
+	o.instances.refs.addKeys("secret-instance-key")
+	o.importKey(t, testAccount, "secret-spot-key")
+	o.openSpotRequest(t, "sir-0secret000000001", "secret-spot-key", "")
+	o.importKey(t, testAccount, "secret-template-key")
+	_, err := o.templates.CreateLaunchTemplate(t.Context(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("secret-referencing-template"),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{KeyName: aws.String("secret-template-key")},
+	}, testAccount)
+	require.NoError(t, err)
+
+	body, err := json.Marshal(o.cleanup(t))
+	require.NoError(t, err)
+	for _, leak := range []string{
+		"secret", "AAAAC3Nz", memberID, "node-1", "injected", "sir-",
+		base64.StdEncoding.EncodeToString([]byte("secret-user-data")),
+	} {
+		assert.NotContains(t, string(body), leak)
+	}
+	assert.Contains(t, string(body), failing, "the IDs themselves are reported")
+}
+
+// A delete that fails part way is failed, not deleted; the re-run completes
+// the partly deleted key pair and reports it deleted exactly once.
+func TestCleanupPlatformInterruptedRunConverges(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "deploy")
+	o.store.failDelete = func(key string) bool { return strings.HasSuffix(key, ".json") }
+
+	res := o.cleanup(t)
+	assert.Equal(t, []Item{item(keyID, ec2v1.PlatformKindKeyPair, DispositionFailed, ReasonOwnerDeleteFailed)}, res.Items)
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindKeyPair])
+	scan, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, scan.LiveKeyPairIDs, "the metadata anchors the unfinished delete")
+
+	o.store.failDelete = nil
+	res = o.cleanup(t)
+	assert.Equal(t, []Item{item(keyID, ec2v1.PlatformKindKeyPair, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, allStatus(ec2v1.PlatformStatusObservedClean), statuses(res))
+}
+
+// Records the owner cannot read are never deleted and stay reported as residue,
+// and orphaned tags whose owner may be one of them are left.
+func TestCleanupPlatformLeavesUnreadableRecords(t *testing.T) {
+	o := newOwners(t)
+	o.putObject(t, "key-0badbadbadbadbad0.json", "{not json")
+	o.putObject(t, "orphan", testPubKey)
+
+	res := o.cleanup(t)
+	assert.Empty(t, res.Items)
+	assert.ElementsMatch(t, []Count{
+		{Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassUnreadableRecord, Disposition: DispositionResidue, Reason: ReasonUnreadableRecord, Count: 1},
+		{Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassPublicKeyUnresolved, Disposition: DispositionResidue, Reason: ReasonOwnerRecordsUnresolved, Count: 1},
+	}, res.Counts)
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindKeyPair])
+	scan, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"key-0badbadbadbadbad0"}, scan.UnreadableKeyPairIDs)
+}
+
+// Residue younger than the settle window may be a create in flight: it is
+// reported, not removed.
+func TestCleanupPlatformKeepsUnsettledResidue(t *testing.T) {
+	o := newOwners(t)
+	o.putObject(t, "orphan", testPubKey)
+	c := o.newCleanup()
+
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount})
+	require.NoError(t, err)
+	assert.Equal(t, []Count{{
+		Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassPublicKeyWithoutMetadata,
+		Disposition: DispositionResidue, Reason: ReasonRecentlyWritten, Count: 1,
+	}}, res.Counts)
+}
+
+// lyingKeys reports every key pair deleted without deleting anything.
+type lyingKeys struct{ *ec2key.KeyServiceImpl }
+
+func (l lyingKeys) CleanupAccount(ctx context.Context, account string, _ time.Time, _ func(string) string) (*ec2key.AccountCleanup, error) {
+	scan, err := l.ScanAccount(ctx, account)
+	if err != nil {
+		return nil, err
+	}
+	return &ec2key.AccountCleanup{DeletedKeyPairIDs: scan.LiveKeyPairIDs}, nil
+}
+
+// An owner that reports a delete is not believed: the re-scan still finds the
+// record, so it is failed and still present.
+func TestCleanupPlatformProvesDeletesByRescan(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "deploy")
+
+	res := run(t, New(lyingKeys{o.keys}, o.groups, o.templates, o.tags, o.readers()))
+	assert.Contains(t, res.Items, item(keyID, ec2v1.PlatformKindKeyPair, DispositionFailed, ReasonStillPresent))
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindKeyPair])
+}
+
+// failingRescan fails every scan after the first, so the cleanup deletes but
+// cannot prove it.
+type failingRescan struct {
+	*ec2launchtemplate.LaunchTemplateServiceImpl
+
+	scans int
+}
+
+func (f *failingRescan) ScanAccount(ctx context.Context, account string) (*ec2launchtemplate.AccountScan, error) {
+	if f.scans++; f.scans > 1 {
+		return nil, errInjected
+	}
+	return f.LaunchTemplateServiceImpl.ScanAccount(ctx, account)
+}
+
+// A failed re-scan is incomplete, never clean: the template stays blocked, and
+// a tag of that kind no reported resource owns stays, as residue.
+func TestCleanupPlatformFailedRescanIsIncomplete(t *testing.T) {
+	o := newOwners(t)
+	ltID := o.createTemplate(t, testAccount, "web")
+	o.tag(t, testAccount, "lt-0stale000000000")
+
+	res := run(t, New(o.keys, o.groups, &failingRescan{LaunchTemplateServiceImpl: o.templates}, o.tags, o.readers()))
+	assert.Equal(t, []Item{
+		item(ltID, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown),
+		item("lt-0stale000000000", ec2v1.PlatformKindTagIndex, DispositionResidue, ReasonOwnerRecordsUnresolved),
+	}, res.Items)
+	assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+	assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindTagIndex])
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "lt-")
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"lt-0stale000000000", ltID}, tagged)
+}
+
+// failingTags fails every tag-index clear.
+type failingTags struct{ *ec2tags.TagsServiceImpl }
+
+func (failingTags) DeleteAllTags(context.Context, string, string) error { return errInjected }
+
+// A tag-index entry that cannot be cleared is failed and the kind keeps
+// records, never clean.
+func TestCleanupPlatformReportsUnclearedTags(t *testing.T) {
+	o := newOwners(t)
+	o.tag(t, testAccount, staleTagID)
+
+	res := run(t, New(o.keys, o.groups, o.templates, failingTags{o.tags}, o.readers()))
+	assert.Equal(t, []Item{item(staleTagID, ec2v1.PlatformKindTagIndex, DispositionFailed, ReasonOwnerDeleteFailed)}, res.Items)
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindTagIndex])
+}
+
+// failingCleanup fails the owner cleanup's own listing.
+type failingCleanup struct {
+	*ec2placementgroup.PlacementGroupServiceImpl
+}
+
+func (failingCleanup) CleanupAccount(context.Context, string, func(string, string) string) (*ec2placementgroup.AccountCleanup, error) {
+	return nil, errInjected
+}
+
+// An owner cleanup that cannot list deletes nothing and is incomplete; the
+// group it could not reach is still reported, from the re-scan.
+func TestCleanupPlatformOwnerListingFailureIsIncomplete(t *testing.T) {
+	o := newOwners(t)
+	groupID := o.createGroup(t, testAccount, "web")
+
+	res := run(t, New(o.keys, failingCleanup{o.groups}, o.templates, o.tags, o.readers()))
+	assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindPlacementGroup])
+	assert.Contains(t, res.Items, item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionResidue, ReasonAppearedDuringCleanup))
+}
+
+func TestCleanupPlatformValidatesTheRequest(t *testing.T) {
+	c := New(nil, nil, nil, nil, ReferenceReaders{})
+	for name, tc := range map[string]struct {
+		req  *Request
+		code string
+	}{
+		"nil":       {nil, awserrors.ErrorMissingParameter},
+		"empty":     {&Request{}, awserrors.ErrorMissingParameter},
+		"malformed": {&Request{AccountID: "acct-1"}, awserrors.ErrorInvalidParameterValue},
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := c.CleanupPlatform(t.Context(), tc.req)
+			require.Error(t, err)
+			got, _ := awserrors.ResolveErrorCode(err)
+			assert.Equal(t, tc.code, got)
+		})
+	}
+}
+
+func (o *owners) openSpotRequest(t *testing.T, id, keyName, groupName string) {
+	t.Helper()
+	spec := &ec2.LaunchSpecification{ImageId: aws.String("ami-123"), KeyName: aws.String(keyName)}
+	if groupName != "" {
+		spec.Placement = &ec2.SpotPlacement{GroupName: aws.String(groupName)}
+	}
+	_, err := o.spot.PutSpotInstanceRequests(t.Context(), &ec2spotinstance.PutSpotRequestsInput{Requests: []*ec2.SpotInstanceRequest{{
+		SpotInstanceRequestId: aws.String(id), State: aws.String(ec2.SpotInstanceStateOpen), LaunchSpecification: spec,
+	}}}, testAccount)
+	require.NoError(t, err)
+}
+
+func (o *owners) putTemplateRecord(t *testing.T, key, value string) {
+	t.Helper()
+	kv, err := o.js.KeyValue(t.Context(), ec2launchtemplate.KVBucketLaunchTemplates)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(value))
+	require.NoError(t, err)
+}
+
+// A live launch template is never deleted: an EKS nodegroup may name it and
+// EC2 cannot read EKS records. It stays blocked, described and tagged on every
+// run; headerless version bodies and names, which nothing can resolve, are
+// removed once settled.
+func TestCleanupPlatformNeverDeletesLiveLaunchTemplates(t *testing.T) {
+	o := newOwners(t)
+	ltID := o.createTemplate(t, testAccount, "web")
+	o.putTemplateRecord(t, testAccount+".lt-0orphan000000000.v1", `{"version_number":1}`)
+
+	for range 2 {
+		res := o.cleanup(t)
+		assert.Equal(t, []Item{item(ltID, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown)}, res.Items)
+		assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+		assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindTagIndex], "the template keeps its tags")
+	}
+	scan, err := o.templates.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{ltID}, scan.LiveTemplateIDs)
+	assert.Zero(t, scan.OrphanVersionBodies)
+	out, err := o.templates.DescribeLaunchTemplateVersions(t.Context(), &ec2.DescribeLaunchTemplateVersionsInput{LaunchTemplateId: aws.String(ltID)}, testAccount)
+	require.NoError(t, err)
+	assert.Len(t, out.LaunchTemplateVersions, 1, "the version body is untouched")
+}
+
+// A key pair a non-terminated instance names is blocked and kept, and so is an
+// orphan public key the instance names, because IMDS serves it by name. A
+// placement group a stopped instance names is blocked though it has no members.
+func TestCleanupPlatformBlocksOnInstanceReferences(t *testing.T) {
+	o := newOwners(t)
+	usedKey := o.importKey(t, testAccount, "in-use")
+	freeKey := o.importKey(t, testAccount, "free")
+	o.putObject(t, "served-orphan", testPubKey)
+	usedGroup := o.createGroup(t, testAccount, "stopped-member")
+	o.instances.refs.addKeys("in-use", "served-orphan")
+	o.instances.refs.addGroups("stopped-member")
+
+	res := o.cleanup(t)
+	assert.ElementsMatch(t, []Item{
+		item(usedKey, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonInstanceReference),
+		item(freeKey, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(usedGroup, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonInstanceReference),
+	}, res.Items)
+	assert.Equal(t, []Count{{
+		Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassPublicKeyWithoutMetadata,
+		Disposition: DispositionBlocked, Reason: ReasonInstanceReference, Count: 1,
+	}}, res.Counts)
+	scan, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{usedKey}, scan.LiveKeyPairIDs)
+	assert.Equal(t, 1, scan.OrphanPublicKeys)
+	material, err := o.keys.GetPublicKeyMaterial(testAccount, "served-orphan")
+	require.NoError(t, err)
+	assert.Equal(t, testPubKey, material)
+}
+
+// An open spot request's launch specification blocks the key pair and the
+// placement group it names until the request is cancelled.
+func TestCleanupPlatformBlocksOnSpotRequests(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "spot-key")
+	groupID := o.createGroup(t, testAccount, "spot-group")
+	o.openSpotRequest(t, "sir-0000000000000001", "spot-key", "spot-group")
+
+	res := o.cleanup(t)
+	assert.ElementsMatch(t, []Item{
+		item(keyID, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonSpotRequestReference),
+		item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonSpotRequestReference),
+	}, res.Items)
+
+	_, err := o.spot.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{
+		SpotInstanceRequestIds: []*string{aws.String("sir-0000000000000001")},
+	}, testAccount)
+	require.NoError(t, err)
+	res = o.cleanup(t)
+	assert.ElementsMatch(t, []Item{
+		item(keyID, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, ""),
+	}, res.Items)
+}
+
+// failingTemplates fails the launch-template reference check.
+type failingTemplates struct{}
+
+func (failingTemplates) AccountTemplateReferences(context.Context, string) (*ec2launchtemplate.TemplateReferences, error) {
+	return nil, errInjected
+}
+
+// failingSpot fails the spot reference check.
+type failingSpot struct{}
+
+func (failingSpot) AccountLaunchReferences(context.Context, string) (*ec2spotinstance.LaunchReferences, error) {
+	return nil, errInjected
+}
+
+// A reference check that cannot complete is never read as "no references":
+// key pairs, orphan public keys and placement groups are all kept, both kinds
+// are incomplete, and launch-template residue, which no check guards, is
+// still removed.
+func TestCleanupPlatformFailedReferenceCheckDeletesNothing(t *testing.T) {
+	for name, build := range map[string]func(o *owners) *Cleanup{
+		"instance listing fails": func(o *owners) *Cleanup {
+			o.instances.err = errInjected
+			return o.newCleanup()
+		},
+		"spot listing fails": func(o *owners) *Cleanup {
+			r := o.readers()
+			r.Spot = failingSpot{}
+			return New(o.keys, o.groups, o.templates, o.tags, r)
+		},
+		"launch-template reading fails": func(o *owners) *Cleanup {
+			r := o.readers()
+			r.Templates = failingTemplates{}
+			return New(o.keys, o.groups, o.templates, o.tags, r)
+		},
+		"instance reader absent": func(o *owners) *Cleanup {
+			r := o.readers()
+			r.Instances = nil
+			return New(o.keys, o.groups, o.templates, o.tags, r)
+		},
+		"launch-template reader absent": func(o *owners) *Cleanup {
+			r := o.readers()
+			r.Templates = nil
+			return New(o.keys, o.groups, o.templates, o.tags, r)
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			keyID := o.importKey(t, testAccount, "deploy")
+			o.putObject(t, "orphan", testPubKey)
+			groupID := o.createGroup(t, testAccount, "web")
+			o.putTemplateRecord(t, testAccount+".lt-0orphan000000000.v1", `{"version_number":1}`)
+
+			res := run(t, build(o))
+			assert.ElementsMatch(t, []Item{
+				item(keyID, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonReferenceCheckFailed),
+				item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonReferenceCheckFailed),
+			}, res.Items)
+			assert.ElementsMatch(t, []Count{
+				{Kind: ec2v1.PlatformKindKeyPair, Class: ec2v1.PlatformClassPublicKeyWithoutMetadata, Disposition: DispositionBlocked, Reason: ReasonReferenceCheckFailed, Count: 1},
+				{Kind: ec2v1.PlatformKindLaunchTemplate, Class: ec2v1.PlatformClassVersionBodyWithoutHeader, Disposition: DispositionDeleted, Count: 1},
+			}, res.Counts)
+			for _, k := range res.Kinds {
+				if k.Kind == ec2v1.PlatformKindKeyPair || k.Kind == ec2v1.PlatformKindPlacementGroup {
+					assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, k.Status, k.Kind)
+					assert.Equal(t, []string{ReasonReferenceCheckFailed}, k.IncompleteReasons, k.Kind)
+				}
+			}
+			keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs)
+			assert.Equal(t, 1, keys.OrphanPublicKeys)
+			groups, err := o.groups.ScanAccount(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Len(t, groups.Groups, 1)
+		})
+	}
+}
+
+// A key pair or placement group named by any version of a live launch
+// template, by name or group ID, is blocked: RunInstances may resolve any
+// version by $Default, $Latest or number. The template itself stays blocked.
+func TestCleanupPlatformBlocksOnLaunchTemplateReferences(t *testing.T) {
+	o := newOwners(t)
+	usedKey := o.importKey(t, testAccount, "template-key")
+	freeKey := o.importKey(t, testAccount, "free")
+	byName := o.createGroup(t, testAccount, "template-group")
+	byID := o.createGroup(t, testAccount, "by-id")
+	ltID := o.createTemplate(t, testAccount, "web")
+	_, err := o.templates.CreateLaunchTemplateVersion(t.Context(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(ltID),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			KeyName:   aws.String("template-key"),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String("template-group")},
+		},
+	}, testAccount)
+	require.NoError(t, err)
+	_, err = o.templates.CreateLaunchTemplateVersion(t.Context(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId:   aws.String(ltID),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{Placement: &ec2.LaunchTemplatePlacementRequest{GroupId: aws.String(byID)}},
+	}, testAccount)
+	require.NoError(t, err)
+
+	res := o.cleanup(t)
+	assert.ElementsMatch(t, []Item{
+		item(usedKey, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference),
+		item(freeKey, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(byName, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonLaunchTemplateReference),
+		item(byID, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonLaunchTemplateReference),
+		item(ltID, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown),
+	}, res.Items)
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{usedKey}, keys.LiveKeyPairIDs)
+	groups, err := o.groups.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Len(t, groups.Groups, 2)
+}
+
+// A live template version the owner cannot decode may name any key pair or
+// group, so both kinds are blocked and incomplete and nothing is deleted.
+func TestCleanupPlatformUndecodableTemplateVersionDeletesNothing(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "deploy")
+	groupID := o.createGroup(t, testAccount, "web")
+	ltID := o.createTemplate(t, testAccount, "web")
+	o.putTemplateRecord(t, testAccount+"."+ltID+".v2", "{not json")
+
+	res := o.cleanup(t)
+	assert.Contains(t, res.Items, item(keyID, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonReferenceCheckFailed))
+	assert.Contains(t, res.Items, item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonReferenceCheckFailed))
+	assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindKeyPair])
+	assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindPlacementGroup])
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs)
+}
+
+// A tag-index entry whose owner re-scan holds an unreadable record is not
+// cleared, because that record may be its owner; it is residue, not deleted.
+func TestCleanupPlatformUnresolvedOwnerKeepsTags(t *testing.T) {
+	for name, seed := range map[string]func(t *testing.T, o *owners) string{
+		"key pair": func(t *testing.T, o *owners) string {
+			o.putObject(t, "key-0badbadbadbadbad0.json", "{not json")
+			return "key-0stale000000000"
+		},
+		"placement group": func(t *testing.T, o *owners) string {
+			kv, err := o.js.KeyValue(t.Context(), ec2placementgroup.KVBucketPlacementGroups)
+			require.NoError(t, err)
+			_, err = kv.Put(t.Context(), testAccount+".broken", []byte("{not json"))
+			require.NoError(t, err)
+			return staleTagID
+		},
+		"launch template": func(t *testing.T, o *owners) string {
+			o.putTemplateRecord(t, testAccount+".lt-0broken000000000", "{not json")
+			return "lt-0stale000000000"
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			stale := seed(t, o)
+			o.tag(t, testAccount, stale)
+
+			res := o.cleanup(t)
+			assert.Contains(t, res.Items, item(stale, ec2v1.PlatformKindTagIndex, DispositionResidue, ReasonOwnerRecordsUnresolved))
+			assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindTagIndex])
+			tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, stale[:3])
+			require.NoError(t, err)
+			assert.Equal(t, []string{stale}, tagged)
+		})
+	}
+}
+
+// Settled launch-template residue the owner fails to remove is reported
+// failed from the re-scan, never deleted.
+func TestCleanupPlatformResidueRemovalFailureIsFailed(t *testing.T) {
+	o := newOwners(t)
+	o.putTemplateRecord(t, testAccount+".lt-0orphan000000000.v1", `{"version_number":1}`)
+
+	res := run(t, New(o.keys, o.groups, failingResidue{o.templates}, o.tags, o.readers()))
+	assert.Equal(t, []Count{{
+		Kind: ec2v1.PlatformKindLaunchTemplate, Class: ec2v1.PlatformClassVersionBodyWithoutHeader,
+		Disposition: DispositionFailed, Reason: ReasonOwnerDeleteFailed, Count: 1,
+	}}, res.Counts)
+	assert.Equal(t, ec2v1.PlatformStatusRecordsFound, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+}
+
+// failingResidue reports every settled orphan as a failed removal and removes nothing.
+type failingResidue struct {
+	*ec2launchtemplate.LaunchTemplateServiceImpl
+}
+
+func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string, _ time.Time) (*ec2launchtemplate.AccountCleanup, error) {
+	scan, err := f.ScanAccount(ctx, acct)
+	if err != nil {
+		return nil, err
+	}
+	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -0,0 +1,411 @@
+package accountcleanup
+
+import (
+	"context"
+	"log/slog"
+	"maps"
+	"slices"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+)
+
+// Each kind runs scan, owner cleanup, then re-scan. The first scan only bounds
+// how much nameless residue a run can claim to have removed.
+
+type keyPairRun struct {
+	kind *kindRun
+	// unchecked: the reference check failed, so nothing was attempted.
+	unchecked bool
+	before    *ec2key.AccountScan
+	cleaned   *ec2key.AccountCleanup
+	did       attempts
+}
+
+// cleanKeyPairs deletes only when the reference check completed; otherwise
+// every key pair and orphan public key is kept and the kind is incomplete.
+func (c *Cleanup) cleanKeyPairs(ctx context.Context, acct string, settledBefore time.Time, refs referenceCheck) *keyPairRun {
+	run := &keyPairRun{kind: c.startKind(ec2v1.PlatformKindKeyPair), did: newAttempts()}
+	if before, err := c.keys.ScanAccount(ctx, acct); err == nil {
+		run.before = before
+	}
+	if !refs.complete {
+		run.unchecked = true
+		run.kind.incomplete(ReasonReferenceCheckFailed)
+		return run
+	}
+	cleaned, err := c.keys.CleanupAccount(ctx, acct, settledBefore, refs.keyReason)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: key-pair cleanup failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return run
+	}
+	run.cleaned = cleaned
+	markAll(run.did.deleted, cleaned.DeletedKeyPairIDs)
+	markAll(run.did.failed, cleaned.FailedKeyPairIDs)
+	maps.Copy(run.did.kept, cleaned.Referenced)
+	return run
+}
+
+func (c *Cleanup) finishKeyPairs(ctx context.Context, acct string, run *keyPairRun, res *Result) ownedIDs {
+	defer c.finishKind(run.kind)
+	after, err := c.keys.ScanAccount(ctx, acct)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: key-pair re-scan failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		after = nil
+	}
+	present := map[string]bool{}
+	if after != nil {
+		markAll(present, after.LiveKeyPairIDs)
+		markAll(present, after.UnreadableKeyPairIDs)
+		if after.ReadFailures > 0 {
+			run.kind.incomplete(ec2v1.PlatformReasonRecordReadFailed)
+		}
+	}
+	var beforeLive, afterLive, deleted, failed []string
+	if run.before != nil {
+		beforeLive = run.before.LiveKeyPairIDs
+	}
+	if after != nil {
+		afterLive = after.LiveKeyPairIDs
+	}
+	if run.cleaned != nil {
+		deleted, failed = run.cleaned.DeletedKeyPairIDs, run.cleaned.FailedKeyPairIDs
+	}
+	if run.unchecked {
+		keepAll(run.did.kept, beforeLive, ReasonReferenceCheckFailed)
+		keepAll(run.did.kept, afterLive, ReasonReferenceCheckFailed)
+	}
+	for _, id := range sortedUnion(beforeLive, deleted, failed, afterLive) {
+		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)
+		res.addItem(run.kind, id, disposition, reason, 0)
+	}
+	if after == nil {
+		return ownedIDs{}
+	}
+	res.addCount(run.kind, ec2v1.PlatformClassUnreadableRecord, DispositionResidue, ReasonUnreadableRecord, len(after.UnreadableKeyPairIDs))
+	res.addCount(run.kind, ec2v1.PlatformClassPublicKeyUnresolved, DispositionResidue, ReasonOwnerRecordsUnresolved, after.UnresolvedPublicKeys)
+	var before, removed, failedN, unsettled int
+	kept := map[string]int{}
+	if run.before != nil {
+		before = run.before.OrphanPublicKeys
+	}
+	if run.cleaned != nil {
+		removed, failedN, unsettled = run.cleaned.OrphansRemoved, run.cleaned.OrphansFailed, run.cleaned.OrphansUnsettled
+		kept = run.cleaned.OrphansReferenced
+	} else if run.unchecked {
+		kept[ReasonReferenceCheckFailed] = after.OrphanPublicKeys
+	}
+	res.residueCounts(run.kind, ec2v1.PlatformClassPublicKeyWithoutMetadata, before, after.OrphanPublicKeys, removed, failedN, unsettled, kept)
+	return ownedIDs{ids: present, resolved: after.ReadFailures == 0 && len(after.UnreadableKeyPairIDs) == 0}
+}
+
+type placementGroupRun struct {
+	kind      *kindRun
+	unchecked bool
+	before    *ec2placementgroup.AccountScan
+	cleaned   *ec2placementgroup.AccountCleanup
+	did       attempts
+}
+
+// cleanPlacementGroups deletes only when the reference check completed.
+func (c *Cleanup) cleanPlacementGroups(ctx context.Context, acct string, refs referenceCheck) *placementGroupRun {
+	run := &placementGroupRun{kind: c.startKind(ec2v1.PlatformKindPlacementGroup), did: newAttempts()}
+	if before, err := c.groups.ScanAccount(ctx, acct); err == nil {
+		run.before = before
+	}
+	if !refs.complete {
+		run.unchecked = true
+		run.kind.incomplete(ReasonReferenceCheckFailed)
+		return run
+	}
+	cleaned, err := c.groups.CleanupAccount(ctx, acct, refs.groupReason)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: placement-group cleanup failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return run
+	}
+	run.cleaned = cleaned
+	markAll(run.did.deleted, cleaned.DeletedGroupIDs)
+	markAll(run.did.failed, cleaned.FailedGroupIDs)
+	for _, b := range cleaned.Blocked {
+		run.did.blocked[b.GroupID] = b.MemberCount
+	}
+	maps.Copy(run.did.kept, cleaned.Referenced)
+	return run
+}
+
+// finishPlacementGroups reports a group that still names instances as
+// blocked with its member count, from the re-scan when it can.
+func (c *Cleanup) finishPlacementGroups(ctx context.Context, acct string, run *placementGroupRun, res *Result) ownedIDs {
+	defer c.finishKind(run.kind)
+	after, err := c.groups.ScanAccount(ctx, acct)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: placement-group re-scan failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		after = nil
+	}
+	present := map[string]bool{}
+	members := map[string]int{}
+	var ids []string
+	if run.before != nil {
+		for _, g := range run.before.Groups {
+			ids = append(ids, g.GroupID)
+		}
+	}
+	if run.cleaned != nil {
+		ids = append(ids, run.cleaned.DeletedGroupIDs...)
+		ids = append(ids, run.cleaned.FailedGroupIDs...)
+		for _, b := range run.cleaned.Blocked {
+			ids = append(ids, b.GroupID)
+		}
+		for id := range run.cleaned.Referenced {
+			ids = append(ids, id)
+		}
+	}
+	if after != nil {
+		for _, g := range after.Groups {
+			present[g.GroupID] = true
+			members[g.GroupID] = g.MemberCount
+			ids = append(ids, g.GroupID)
+		}
+		if after.ReadFailures > 0 {
+			run.kind.incomplete(ec2v1.PlatformReasonRecordReadFailed)
+		}
+	}
+	if run.unchecked {
+		keepAll(run.did.kept, ids, ReasonReferenceCheckFailed)
+	}
+	for _, id := range sortedUnion(ids) {
+		if present[id] && (members[id] > 0 || run.did.blocked[id] > 0) {
+			count := members[id]
+			if count == 0 {
+				count = run.did.blocked[id]
+			}
+			res.addItem(run.kind, id, DispositionBlocked, ReasonInstanceMembers, count)
+			continue
+		}
+		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)
+		res.addItem(run.kind, id, disposition, reason, 0)
+	}
+	if after == nil {
+		return ownedIDs{}
+	}
+	res.addCount(run.kind, ec2v1.PlatformClassUnreadableRecord, DispositionResidue, ReasonUnreadableRecord, after.Unreadable)
+	return ownedIDs{ids: present, resolved: after.ReadFailures == 0 && after.Unreadable == 0}
+}
+
+type launchTemplateRun struct {
+	kind    *kindRun
+	before  *ec2launchtemplate.AccountScan
+	cleaned *ec2launchtemplate.AccountCleanup
+	did     attempts
+}
+
+func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time) *launchTemplateRun {
+	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts()}
+	if before, err := c.templates.ScanAccount(ctx, acct); err == nil {
+		run.before = before
+	}
+	cleaned, err := c.templates.CleanupAccountResidue(ctx, acct, settledBefore)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: launch-template residue cleanup failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return run
+	}
+	run.cleaned = cleaned
+	keepAll(run.did.kept, cleaned.KeptTemplateIDs, ReasonReferenceUnknown)
+	return run
+}
+
+func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *launchTemplateRun, res *Result) ownedIDs {
+	defer c.finishKind(run.kind)
+	after, err := c.templates.ScanAccount(ctx, acct)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: launch-template re-scan failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		after = nil
+	}
+	present := map[string]bool{}
+	if after != nil {
+		markAll(present, after.LiveTemplateIDs)
+		markAll(present, after.UnreadableTemplateIDs)
+		if after.ReadFailures > 0 {
+			run.kind.incomplete(ec2v1.PlatformReasonRecordReadFailed)
+		}
+	}
+	var beforeLive, afterLive, kept []string
+	if run.before != nil {
+		beforeLive = run.before.LiveTemplateIDs
+	}
+	if after != nil {
+		afterLive = after.LiveTemplateIDs
+	}
+	if run.cleaned != nil {
+		kept = run.cleaned.KeptTemplateIDs
+	}
+	// Live templates are never deleted here, so every one still present is
+	// blocked on the unknown EKS nodegroup reference, whenever it appeared.
+	for _, id := range afterLive {
+		run.did.kept[id] = ReasonReferenceUnknown
+	}
+	for _, id := range sortedUnion(beforeLive, kept, afterLive) {
+		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)
+		res.addItem(run.kind, id, disposition, reason, 0)
+	}
+	if after == nil {
+		return ownedIDs{}
+	}
+	res.addCount(run.kind, ec2v1.PlatformClassUnreadableRecord, DispositionResidue, ReasonUnreadableRecord,
+		len(after.UnreadableTemplateIDs)+after.UnreadableNameIndexEntries)
+	res.addCount(run.kind, ec2v1.PlatformClassUnrecognisedTemplateEntry, DispositionResidue, ReasonUnrecognisedRecord, after.UnrecognisedEntries)
+	var c0 ec2launchtemplate.AccountCleanup
+	if run.cleaned != nil {
+		c0 = *run.cleaned
+	}
+	var beforeBodies, beforeNames int
+	if run.before != nil {
+		beforeBodies, beforeNames = run.before.OrphanVersionBodies, run.before.OrphanNameIndexEntries
+	}
+	res.residueCounts(run.kind, ec2v1.PlatformClassVersionBodyWithoutHeader, beforeBodies, after.OrphanVersionBodies,
+		c0.OrphanBodiesRemoved, c0.OrphanBodiesFailed, c0.OrphanBodiesUnsettled, nil)
+	res.residueCounts(run.kind, ec2v1.PlatformClassNameIndexWithoutHeader, beforeNames, after.OrphanNameIndexEntries,
+		c0.OrphanNamesRemoved, c0.OrphanNamesFailed, c0.OrphanNamesUnsettled, nil)
+	return ownedIDs{ids: present, resolved: after.ReadFailures == 0 && len(after.UnreadableTemplateIDs) == 0 &&
+		after.UnreadableNameIndexEntries == 0 && after.UnrecognisedEntries == 0}
+}
+
+// cleanTagIndex clears entries whose owner re-scan resolved and holds no
+// record for the ID, then re-lists the index to prove each clear. Entries of
+// resources that still exist are theirs, not residue, and are left alone.
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+	if listed {
+		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+	}
+	c.finishKind(kind)
+	return kind.report
+}
+
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+	// A resource this run reports as not deleted owns its tags even when its
+	// kind's re-scan failed.
+	reported := map[string]bool{}
+	for _, it := range res.Items {
+		if it.Disposition != DispositionDeleted {
+			reported[it.ResourceID] = true
+		}
+	}
+	var cleared []string
+	for _, id := range tagged {
+		owner := owners[idPrefix(id)]
+		switch {
+		case owner.ids[id], reported[id]:
+			kind.found = true
+		case !owner.resolved:
+			kind.incomplete(ec2v1.PlatformReasonOwnerRecordsUnresolved)
+			res.addItem(kind, id, DispositionResidue, ReasonOwnerRecordsUnresolved, 0)
+		default:
+			if err := c.tags.DeleteAllTags(ctx, acct, id); err != nil {
+				slog.WarnContext(ctx, "Platform cleanup: tag-index clear failed", "resourceId", id, "err", err)
+				res.addItem(kind, id, DispositionFailed, ReasonOwnerDeleteFailed, 0)
+				continue
+			}
+			cleared = append(cleared, id)
+		}
+	}
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
+		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+	}
+	for _, id := range cleared {
+		disposition, reason := itemDisposition(id, attempts{deleted: map[string]bool{id: true}}, slices.Contains(after, id), err == nil)
+		res.addItem(kind, id, disposition, reason, 0)
+	}
+	if len(after) > 0 {
+		kind.found = true
+	}
+}
+
+func keepAll(set map[string]string, ids []string, reason string) {
+	for _, id := range ids {
+		set[id] = reason
+	}
+}
+
+// referenceCheck is the outcome of reading what instance records, spot
+// requests and launch-template versions name. complete is false when any
+// reader failed or is absent.
+type referenceCheck struct {
+	complete                  bool
+	instances, spot, template *References
+}
+
+// readReferences runs every reference check. A check that cannot complete is
+// never read as "no references".
+func (c *Cleanup) readReferences(ctx context.Context, acct string) referenceCheck {
+	r := c.refs
+	if r.Instances == nil || r.Spot == nil || r.Templates == nil {
+		slog.WarnContext(ctx, "Platform cleanup: reference reader not configured")
+		return referenceCheck{}
+	}
+	instances, err := r.Instances.AccountInstanceReferences(ctx, acct)
+	if err != nil || instances == nil {
+		slog.WarnContext(ctx, "Platform cleanup: instance reference check failed", "err", err)
+		return referenceCheck{}
+	}
+	launch, err := r.Spot.AccountLaunchReferences(ctx, acct)
+	if err != nil || launch == nil {
+		slog.WarnContext(ctx, "Platform cleanup: spot reference check failed", "err", err)
+		return referenceCheck{}
+	}
+	tmpl, err := r.Templates.AccountTemplateReferences(ctx, acct)
+	if err != nil || tmpl == nil {
+		slog.WarnContext(ctx, "Platform cleanup: launch-template reference check failed", "err", err)
+		return referenceCheck{}
+	}
+	inst := newReferences()
+	inst.merge(instances)
+	spot := newReferences()
+	spot.addKeys(launch.KeyNames...)
+	spot.addGroups(launch.PlacementGroupNames...)
+	template := newReferences()
+	template.addKeys(tmpl.KeyNames...)
+	template.addGroups(tmpl.PlacementGroupNames...)
+	template.addGroupIDs(tmpl.PlacementGroupIDs...)
+	return referenceCheck{complete: true, instances: inst, spot: spot, template: template}
+}
+
+func (r referenceCheck) keyReason(name string) string {
+	switch {
+	case r.instances.KeyNames[name]:
+		return ReasonInstanceReference
+	case r.spot.KeyNames[name]:
+		return ReasonSpotRequestReference
+	case r.template.KeyNames[name]:
+		return ReasonLaunchTemplateReference
+	}
+	return ""
+}
+
+func (r referenceCheck) groupReason(id, name string) string {
+	switch {
+	case r.instances.hasGroup(id, name):
+		return ReasonInstanceReference
+	case r.spot.hasGroup(id, name):
+		return ReasonSpotRequestReference
+	case r.template.hasGroup(id, name):
+		return ReasonLaunchTemplateReference
+	}
+	return ""
+}
+
+func markAll(set map[string]bool, ids []string) {
+	for _, id := range ids {
+		set[id] = true
+	}
+}

spinifex/domains/ec2/accountcleanup/references.go

@@ -0,0 +1,93 @@
+package accountcleanup
+
+import (
+	"context"
+
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// References are the key-pair and placement-group names, and placement-group
+// IDs, that other EC2 records still name. They select what to keep and are
+// never reported.
+type References struct {
+	KeyNames            map[string]bool
+	PlacementGroupNames map[string]bool
+	PlacementGroupIDs   map[string]bool
+}
+
+// InstanceReferenceReader reads the key-pair and placement-group names the
+// account's non-terminated instance records carry, stopped ones included. It
+// is complete or an error, never a partial answer. A later wiring supplies it.
+type InstanceReferenceReader interface {
+	AccountInstanceReferences(ctx context.Context, accountID string) (*References, error)
+}
+
+// SpotReferenceReader reads the names the account's open or active spot
+// requests would launch with, complete or an error.
+type SpotReferenceReader interface {
+	AccountLaunchReferences(ctx context.Context, accountID string) (*ec2spotinstance.LaunchReferences, error)
+}
+
+// TemplateReferenceReader reads the names and IDs every version of every
+// live launch template would launch with, complete or an error.
+type TemplateReferenceReader interface {
+	AccountTemplateReferences(ctx context.Context, accountID string) (*ec2launchtemplate.TemplateReferences, error)
+}
+
+var (
+	_ SpotReferenceReader     = (*ec2spotinstance.SpotInstanceServiceImpl)(nil)
+	_ TemplateReferenceReader = (*ec2launchtemplate.LaunchTemplateServiceImpl)(nil)
+)
+
+// ReferenceReaders are the checks run before key pairs and placement groups
+// are deleted. A nil reader counts as a failed check.
+type ReferenceReaders struct {
+	Instances InstanceReferenceReader
+	Spot      SpotReferenceReader
+	Templates TemplateReferenceReader
+}
+
+func newReferences() *References {
+	return &References{KeyNames: map[string]bool{}, PlacementGroupNames: map[string]bool{}, PlacementGroupIDs: map[string]bool{}}
+}
+
+func (r *References) addKeys(names ...string) {
+	for _, name := range names {
+		if name != "" {
+			r.KeyNames[name] = true
+		}
+	}
+}
+
+func (r *References) addGroups(names ...string) {
+	for _, name := range names {
+		if name != "" {
+			r.PlacementGroupNames[name] = true
+		}
+	}
+}
+
+func (r *References) addGroupIDs(ids ...string) {
+	for _, id := range ids {
+		if id != "" {
+			r.PlacementGroupIDs[id] = true
+		}
+	}
+}
+
+func (r *References) merge(o *References) {
+	for name := range o.KeyNames {
+		r.addKeys(name)
+	}
+	for name := range o.PlacementGroupNames {
+		r.addGroups(name)
+	}
+	for id := range o.PlacementGroupIDs {
+		r.addGroupIDs(id)
+	}
+}
+
+func (r *References) hasGroup(id, name string) bool {
+	return r.PlacementGroupIDs[id] || r.PlacementGroupNames[name]
+}

spinifex/domains/ec2/accountcleanup/types.go

@@ -0,0 +1,97 @@
+package accountcleanup
+
+import "time"
+
+// Request names the account to clean. CorrelationRef is an opaque caller
+// reference echoed for logs; it grants nothing and is not a fence.
+type Request struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// Result is what the owner stores hold after one run. Kinds reuse the ec2.v1
+// platform kind and status vocabulary; Items and Counts carry resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type Result struct {
+	AccountID      string       `json:"account_id"`
+	CorrelationRef string       `json:"correlation_ref,omitempty"`
+	Kinds          []KindReport `json:"kinds"`
+	Items          []Item       `json:"items,omitempty"`
+	Counts         []Count      `json:"counts,omitempty"`
+}
+
+// KindReport is one kind's status after its re-scan. Clean means the re-scan
+// read every store completely and found nothing; writers are not fenced, so
+// it is a snapshot, not evidence of absence.
+type KindReport struct {
+	Kind              string    `json:"kind"`
+	Status            string    `json:"status"`
+	IncompleteReasons []string  `json:"incomplete_reasons,omitempty"`
+	StartedAt         time.Time `json:"started_at"`
+	EndedAt           time.Time `json:"ended_at"`
+}
+
+// Item is one resource by its AWS resource ID. MemberCount is set only for a
+// placement group blocked by members; instance and request IDs are never reported.
+type Item struct {
+	ResourceID  string `json:"resource_id"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"member_count,omitempty"`
+}
+
+// Count is the number of records of one class with one disposition, for
+// records that have no public identity.
+type Count struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// Dispositions. Only a complete owner re-scan that no longer finds a record
+// makes it deleted.
+const (
+	DispositionDeleted = "deleted"
+	DispositionBlocked = "blocked"
+	DispositionFailed  = "failed"
+	DispositionResidue = "residue"
+)
+
+// Reasons are fixed categories, never store error text.
+const (
+	// ReasonInstanceMembers: a placement group still names instances.
+	ReasonInstanceMembers = "instance-members"
+	// ReasonOwnerDeleteFailed: the owner delete hit a store failure.
+	ReasonOwnerDeleteFailed = "owner-delete-failed"
+	// ReasonStillPresent: the owner reported a delete but the re-scan found it.
+	ReasonStillPresent = "still-present"
+	// ReasonAbsenceUnproven: the re-scan failed, so absence is not shown.
+	ReasonAbsenceUnproven = "absence-unproven"
+	// ReasonUnreadableRecord: a record the owner cannot read is never deleted.
+	ReasonUnreadableRecord = "unreadable-record"
+	// ReasonUnrecognisedRecord: an entry matching no owner record shape.
+	ReasonUnrecognisedRecord = "unrecognised-record"
+	// ReasonOwnerRecordsUnresolved: the record may belong to an unreadable one.
+	ReasonOwnerRecordsUnresolved = "owner-records-unresolved"
+	// ReasonRecentlyWritten: residue younger than the settle window, which may
+	// be a create still in flight.
+	ReasonRecentlyWritten = "recently-written"
+	// ReasonAppearedDuringCleanup: a record the run did not see until its re-scan.
+	ReasonAppearedDuringCleanup = "appeared-during-cleanup"
+	// ReasonInstanceReference: a non-terminated instance record names it.
+	ReasonInstanceReference = "instance-reference"
+	// ReasonSpotRequestReference: an open or active spot request names it.
+	ReasonSpotRequestReference = "spot-request-reference"
+	// ReasonLaunchTemplateReference: a version of a live launch template names it.
+	ReasonLaunchTemplateReference = "launch-template-reference"
+	// ReasonReferenceUnknown: a record EC2 cannot read may name it, so it is
+	// never deleted by account cleanup.
+	ReasonReferenceUnknown = "reference-unknown"
+	// ReasonReferenceCheckFailed: an instance, spot or launch-template reference
+	// check did not complete, so nothing it guards was deleted. Also a kind
+	// incomplete reason.
+	ReasonReferenceCheckFailed = "reference-check-failed"
+)

spinifex/domains/ec2/awsapi/instance/placement.go

@@ -227,7 +227,7 @@ func aggregateResults(ctx context.Context, results []nodeLaunchResult, minCount
 			return nil, clientErr
 		}
 		// A launch that was attempted and failed is not a capacity shortage; surface the real cause.
-		if failErr := launchFailure(results); failErr != nil {
+		if failErr := launchFailure(ctx, results); failErr != nil {
 			slog.ErrorContext(ctx, "distributeInstances: launch fell short of minCount", "minCount", minCount, "launched", totalLaunched, "err", failErr)
 			return nil, failErr
 		}
@@ -321,7 +321,7 @@ func distributeInstancesSpread(ctx context.Context, input *ec2.RunInstancesInput
 		if clientErr := extractClientError(results); clientErr != nil {
 			return nil, clientErr
 		}
-		if failErr := launchFailure(results); failErr != nil {
+		if failErr := launchFailure(ctx, results); failErr != nil {
 			slog.ErrorContext(ctx, "distributeInstancesSpread: launch fell short of minCount", "minCount", minCount, "launched", totalLaunched, "err", failErr)
 			return nil, failErr
 		}
@@ -414,7 +414,7 @@ func distributeInstancesCluster(ctx context.Context, input *ec2.RunInstancesInpu
 		if clientErr := extractClientError(results); clientErr != nil {
 			return nil, clientErr
 		}
-		return nil, results[0].Err
+		return nil, launchFailure(ctx, results)
 	}
 
 	reservation := results[0].Reservation
@@ -468,17 +468,26 @@ func extractClientError(results []nodeLaunchResult) error {
 	return nil
 }
 
-// launchFailure returns the first node launch error, or nil when no node errored.
-// A sub-minCount launch whose nodes actually errored is not a capacity shortage:
-// genuine capacity is caught pre-launch in distributeInstances, and a node-side
-// capacity race is surfaced by extractClientError. Surfacing the real error (RPC
-// timeout, network/OVN failure) stops it masquerading as
-// InsufficientInstanceCapacity — the mislabel that derails triage.
-func launchFailure(results []nodeLaunchResult) error {
+// launchFailure returns the first real node launch error so an RPC or OVN failure is
+// not mislabelled as capacity; a node with no launch subscriber for the type cannot
+// place it, so when only those failed the result is InsufficientInstanceCapacity.
+func launchFailure(ctx context.Context, results []nodeLaunchResult) error {
+	var noResponders error
 	for _, r := range results {
-		if r.Err != nil {
-			return r.Err
+		if r.Err == nil {
+			continue
 		}
+		if errors.Is(r.Err, nats.ErrNoResponders) {
+			if noResponders == nil {
+				noResponders = r.Err
+			}
+			continue
+		}
+		return r.Err
+	}
+	if noResponders != nil {
+		slog.WarnContext(ctx, "distributeInstances: no launch subscriber on node, reporting InsufficientInstanceCapacity", "err", noResponders)
+		return errors.New(awserrors.ErrorInsufficientInstanceCapacity)
 	}
 	return nil
 }

spinifex/domains/ec2/awsapi/instance/placement_test.go

@@ -335,6 +335,32 @@ func TestAggregateResults_TimeoutSurfacedNotMasked(t *testing.T) {
 	assert.ErrorIs(t, err, context.DeadlineExceeded)
 }
 
+// A node with no subscriber on its launch subject surfaces as capacity, never the raw NATS error.
+func TestAggregateResults_NoRespondersIsCapacity(t *testing.T) {
+	t.Parallel()
+	noResp := fmt.Errorf("launch on node-1: %w", fmt.Errorf("NATS request to ec2.RunInstances.t3.large.node-1: %w", nats.ErrNoResponders))
+	results := []nodeLaunchResult{
+		{NodeID: "node-1", Err: noResp},
+	}
+
+	_, err := aggregateResults(context.Background(), results, 1, nil, "")
+	require.EqualError(t, err, awserrors.ErrorInsufficientInstanceCapacity)
+}
+
+// A real launch failure on another node outranks a no-responders node.
+func TestAggregateResults_RealFailureOutranksNoResponders(t *testing.T) {
+	t.Parallel()
+	noResp := fmt.Errorf("launch on node-1: %w", nats.ErrNoResponders)
+	timeoutErr := fmt.Errorf("launch on node-2: %w", context.DeadlineExceeded)
+	results := []nodeLaunchResult{
+		{NodeID: "node-1", Err: noResp},
+		{NodeID: "node-2", Err: timeoutErr},
+	}
+
+	_, err := aggregateResults(context.Background(), results, 1, nil, "")
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+}
+
 // Short launch with no node errors (nodes returned fewer instances than assigned
 // without erroring) keeps the generic capacity code.
 func TestAggregateResults_ShortWithoutNodeErrors(t *testing.T) {
@@ -1120,6 +1146,40 @@ func TestDistributeInstancesCluster_InsufficientCapacityOnPinnedNode(t *testing.
 	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error())
 }
 
+// A pinned node with no launch subscriber reports capacity, not the raw NATS error.
+func TestDistributeInstancesCluster_NoLaunchSubscriberIsCapacity(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+
+	statusSub, err := nc.Subscribe("spinifex.node.status", func(msg *nats.Msg) {
+		data, _ := json.Marshal(types.NodeStatusResponse{Node: "node-1", InstanceTypes: []types.InstanceTypeCap{{Name: "t3.micro", Available: 10}}})
+		_ = nc.Publish(msg.Reply, data)
+	})
+	require.NoError(t, err)
+	defer statusSub.Unsubscribe()
+
+	reserveSub, err := nc.QueueSubscribe("ec2.ReserveClusterNode", "spinifex-workers", func(msg *nats.Msg) {
+		data, _ := json.Marshal(struct {
+			TargetNode string `json:"target_node"`
+		}{TargetNode: "node-1"})
+		_ = msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer reserveSub.Unsubscribe()
+
+	require.NoError(t, nc.Flush())
+
+	input := &ec2.RunInstancesInput{
+		ImageId:      aws.String("ami-test"),
+		InstanceType: aws.String("t3.micro"),
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(1),
+	}
+
+	_, err = distributeInstancesCluster(context.Background(), input, nc, "test-account", "my-cluster-group", 1)
+	require.EqualError(t, err, awserrors.ErrorInsufficientInstanceCapacity)
+}
+
 func TestDistributeInstancesCluster_PinnedNodeNotInCapacityResults(t *testing.T) {
 	t.Parallel()
 	// Pinned node has no capacity at all (not in fan-out results) → InsufficientInstanceCapacity

spinifex/domains/ec2/awsapi/instance/status_synthesis_test.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/cache"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
@@ -237,6 +239,44 @@ func TestSynthesis_HonoursInstanceIDFilter(t *testing.T) {
 	require.Equal(t, "i-asked", aws.StringValue(out.InstanceStatuses[0].InstanceId))
 }
 
+// Synthesis shares the node path's selection, so a platform-managed VM on a
+// stale node is hidden from the customer account it is recorded under.
+func TestSynthesis_HidesManagedSystemVMFromCustomer(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	answerWith(t, nc, "node-active")
+
+	managed := cachedVM("i-ekscp", "node-dead", "az-a", vm.StateRunning)
+	managed.ManagedBy = tags.ManagedByEKS
+	synth := StatusSynthesis{
+		Records:  fakeRecords{ready: true, vms: []*vm.VM{managed}},
+		Liveness: fakeLiveness{states: map[string]instancecache.NodeState{"node-dead": instancecache.NodeStale}},
+	}
+
+	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 1, synthAccount, "az-a", synth)
+	require.NoError(t, err)
+	require.Empty(t, out.InstanceStatuses)
+}
+
+func TestSynthesis_RootSeesManagedSystemVM(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	answerWith(t, nc, "node-active")
+
+	managed := cachedVM("i-lb", "node-dead", "az-a", vm.StateRunning)
+	managed.AccountID = awsidentifiers.GlobalAccountID
+	managed.ManagedBy = tags.ManagedByELBv2
+	synth := StatusSynthesis{
+		Records:  fakeRecords{ready: true, vms: []*vm.VM{managed}},
+		Liveness: fakeLiveness{states: map[string]instancecache.NodeState{"node-dead": instancecache.NodeStale}},
+	}
+
+	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 1, awsidentifiers.GlobalAccountID, "az-a", synth)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	require.Equal(t, "i-lb", aws.StringValue(out.InstanceStatuses[0].InstanceId))
+}
+
 // A cache that has not finished its first whole-set sync cannot support a
 // claim about what is missing, so it contributes nothing.
 func TestSynthesis_ColdCacheContributesNothing(t *testing.T) {

spinifex/domains/ec2/image/account_termination_current_behaviour_test.go

@@ -0,0 +1,264 @@
+package image_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
+	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The TestCurrentBehaviour_AccountTermination_ tests pin how volumes,
+// snapshots and images guard one another today, wired the way the daemon wires
+// them; a failure means a dependency rule changed.
+
+const (
+	ctAccountID = "123456789012"
+	ctBucket    = "ct-bucket"
+	ctAZ        = "ct-az"
+)
+
+type storageServices struct {
+	volumes   *ec2volume.VolumeServiceImpl
+	snapshots *ec2snapshot.SnapshotServiceImpl
+	images    *ec2image.ImageServiceImpl
+	provider  *ebsprovider.MemoryProvider
+	metadata  *ebsmetadata.Store
+}
+
+// newStorageServices shares one object store, one provider and one
+// volume-to-snapshot index between the three services. As in the daemon, the
+// image service is given no handle on that index.
+func newStorageServices(t *testing.T) storageServices {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	refs, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: ec2snapshot.KVBucketVolumeSnapshots})
+	require.NoError(t, err)
+
+	store := objectstore.NewMemoryObjectStore()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{SnapshotEnumeration: true})
+	cfg := &config.Config{AZ: ctAZ, WalDir: t.TempDir(), Predastore: config.PredastoreConfig{Bucket: ctBucket}}
+
+	volumes := ec2volume.NewVolumeServiceImplWithStore(cfg, store, nc, refs)
+	volumes.SetEBSProvider(provider)
+	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc, refs)
+	snapshots.SetEBSProvider(provider)
+	images := ec2image.NewImageServiceImplWithConfig(cfg, store, nil)
+	images.SetEBSProvider(provider)
+
+	return storageServices{
+		volumes: volumes, snapshots: snapshots, images: images,
+		provider: provider, metadata: ebsmetadata.NewStore(store, ctBucket),
+	}
+}
+
+func (s storageServices) seedVolume(t *testing.T, volumeID string) {
+	t.Helper()
+	ctx := context.Background()
+	created, err := s.provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned:     ebsprovider.NewVersioned(),
+		VolumeID:      volumeID,
+		CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	require.NoError(t, s.metadata.PutVolume(ctx, ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: ctAccountID, CapacityGiB: 1,
+		State: "available", ProviderHandle: created.Handle,
+	}))
+}
+
+func (s storageServices) createImage(t *testing.T, rootVolumeID, name string) string {
+	t.Helper()
+	out, err := s.images.CreateImageFromInstance(ec2image.CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-0123456789abcdef0"), Name: aws.String(name)},
+		RootVolumeID: rootVolumeID,
+	}, ctAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(out.ImageId)
+}
+
+func (s storageServices) deleteVolume(volumeID string) error {
+	_, err := s.volumes.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: aws.String(volumeID)}, ctAccountID)
+	return err
+}
+
+// A CreateSnapshot snapshot pins its source volume through the index, but a
+// CreateImage snapshot is never added to it, so its source volume deletes and
+// the image keeps naming a volume that no longer exists.
+func TestCurrentBehaviour_AccountTermination_CreateImageSnapshotDoesNotPinSourceVolume(t *testing.T) {
+	s := newStorageServices(t)
+	ctx := context.Background()
+
+	s.seedVolume(t, "vol-snapshotted")
+	_, err := s.snapshots.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-snapshotted")}, ctAccountID)
+	require.NoError(t, err)
+	assert.EqualError(t, s.deleteVolume("vol-snapshotted"), awserrors.ErrorVolumeInUse,
+		"a CreateSnapshot snapshot blocks deleting its source volume")
+
+	s.seedVolume(t, "vol-imaged")
+	imageID := s.createImage(t, "vol-imaged", "ct-image")
+	require.NoError(t, s.deleteVolume("vol-imaged"), "a CreateImage snapshot does not block deleting its source volume")
+
+	_, err = s.metadata.GetVolume(ctx, ctAccountID, "vol-imaged")
+	require.Error(t, err, "the source volume's document is gone")
+
+	images, err := s.images.DescribeImages(ctx, &ec2.DescribeImagesInput{ImageIds: []*string{aws.String(imageID)}}, ctAccountID)
+	require.NoError(t, err)
+	require.Len(t, images.Images, 1)
+	assert.Equal(t, ec2.ImageStateAvailable, aws.StringValue(images.Images[0].State), "the image still reports available")
+
+	source, err := s.images.GetAMISourceVolumeID(ctx, imageID)
+	require.NoError(t, err)
+	assert.Equal(t, "vol-imaged", source, "a launch from the image still resolves the deleted volume")
+}
+
+// DeleteSnapshot consults AMI documents as well as volumes cloned from the
+// snapshot, so the snapshot under a live image refuses to delete until the
+// image is deregistered.
+func TestAccountTermination_DeleteSnapshotRefusesARegisteredImage(t *testing.T) {
+	s := newStorageServices(t)
+	ctx := context.Background()
+
+	s.seedVolume(t, "vol-registered")
+	snap, err := s.snapshots.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-registered")}, ctAccountID)
+	require.NoError(t, err)
+	snapshotID := aws.StringValue(snap.SnapshotId)
+
+	registered, err := s.images.RegisterImage(ctx, &ec2.RegisterImageInput{
+		Name:           aws.String("ct-registered"),
+		RootDeviceName: aws.String("/dev/sda1"),
+		BlockDeviceMappings: []*ec2.BlockDeviceMapping{{
+			DeviceName: aws.String("/dev/sda1"),
+			Ebs:        &ec2.EbsBlockDevice{SnapshotId: aws.String(snapshotID)},
+		}},
+	}, ctAccountID)
+	require.NoError(t, err)
+
+	_, err = s.snapshots.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, ctAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotInUse, "the snapshot refuses to delete while an image is registered on it")
+
+	_, err = s.metadata.GetSnapshot(ctx, ctAccountID, snapshotID)
+	assert.NoError(t, err, "the snapshot document is still present")
+	_, held := s.provider.Snapshot(snapshotID)
+	assert.True(t, held, "the provider still holds the snapshot")
+
+	_, err = s.images.DeregisterImage(ctx, &ec2.DeregisterImageInput{ImageId: registered.ImageId}, ctAccountID)
+	require.NoError(t, err)
+
+	_, err = s.snapshots.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, ctAccountID)
+	require.NoError(t, err, "the snapshot deletes once the image is deregistered")
+}
+
+// deleteDuringClone runs a hook once, after the provider has accepted a clone
+// from a snapshot and before CreateVolume writes the clone's document.
+type deleteDuringClone struct {
+	ebsprovider.EBSProvider
+
+	hook  func()
+	fired bool
+}
+
+func (p *deleteDuringClone) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
+	created, err := p.EBSProvider.CreateVolume(ctx, req)
+	if err == nil && req.SourceSnapshotID != "" && !p.fired {
+		p.fired = true
+		p.hook()
+	}
+	return created, err
+}
+
+// DeleteSnapshot's clone check reads volume documents, and CreateVolume writes
+// the clone's document only after the provider has verified and cloned the
+// snapshot, so a delete in that window passes the check and strands the clone.
+func TestCurrentBehaviour_AccountTermination_DeleteSnapshotDuringCloneCreationStrandsTheClone(t *testing.T) {
+	s := newStorageServices(t)
+	ctx := context.Background()
+	createFrom := func(snapshotID string) (*ec2.Volume, error) {
+		return s.volumes.CreateVolume(ctx, &ec2.CreateVolumeInput{
+			AvailabilityZone: aws.String(ctAZ), SnapshotId: aws.String(snapshotID),
+		}, ctAccountID)
+	}
+	deleteSnapshot := func(snapshotID string) error {
+		_, err := s.snapshots.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, ctAccountID)
+		return err
+	}
+
+	s.seedVolume(t, "vol-settled")
+	settled, err := s.snapshots.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-settled")}, ctAccountID)
+	require.NoError(t, err)
+	_, err = createFrom(aws.StringValue(settled.SnapshotId))
+	require.NoError(t, err)
+	assert.True(t, awserrors.IsErrorCode(deleteSnapshot(aws.StringValue(settled.SnapshotId)), awserrors.ErrorInvalidSnapshotInUse),
+		"once the clone's document exists, the delete is refused")
+
+	s.seedVolume(t, "vol-raced")
+	raced, err := s.snapshots.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-raced")}, ctAccountID)
+	require.NoError(t, err)
+	snapshotID := aws.StringValue(raced.SnapshotId)
+
+	var raceErr error
+	s.volumes.SetEBSProvider(&deleteDuringClone{EBSProvider: s.provider, hook: func() { raceErr = deleteSnapshot(snapshotID) }})
+	clone, err := createFrom(snapshotID)
+	require.NoError(t, err, "CreateVolume reports success")
+	require.NoError(t, raceErr, "the delete inside the window passes the clone check")
+
+	doc, err := s.metadata.GetVolume(ctx, ctAccountID, aws.StringValue(clone.VolumeId))
+	require.NoError(t, err)
+	assert.Equal(t, snapshotID, doc.SnapshotID, "the clone's document names the deleted snapshot")
+	_, err = s.metadata.GetSnapshot(ctx, ctAccountID, snapshotID)
+	require.Error(t, err, "the snapshot's document is gone")
+	_, held := s.provider.Snapshot(snapshotID)
+	assert.False(t, held, "the provider no longer holds the snapshot the clone was made from")
+
+	_, err = createFrom(snapshotID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotNotFound, "a clone started after the delete is refused")
+}
+
+// CreateImage on an instance re-reads the image the instance launched from,
+// after the provider has taken the snapshot, so once that image is
+// deregistered the call fails and leaves a provider snapshot with no document.
+func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsItsLaunchImage(t *testing.T) {
+	s := newStorageServices(t)
+	ctx := context.Background()
+
+	s.seedVolume(t, "vol-launch-source")
+	launchImageID := s.createImage(t, "vol-launch-source", "ct-launch-image")
+	_, err := s.images.DeregisterImage(ctx, &ec2.DeregisterImageInput{ImageId: aws.String(launchImageID)}, ctAccountID)
+	require.NoError(t, err)
+
+	s.seedVolume(t, "vol-retained-root")
+	_, err = s.images.CreateImageFromInstance(ec2image.CreateImageParams{
+		Input:         &ec2.CreateImageInput{InstanceId: aws.String("i-0123456789abcdef0"), Name: aws.String("ct-after-deregister")},
+		RootVolumeID:  "vol-retained-root",
+		SourceImageID: launchImageID,
+	}, ctAccountID)
+	assert.EqualError(t, err, awserrors.ErrorServerInternal, "the instance's image can no longer be created")
+
+	listed, err := s.provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	var orphans []string
+	for _, ref := range listed.Snapshots {
+		if ref.SourceVolumeID == "vol-retained-root" {
+			orphans = append(orphans, ref.ID)
+		}
+	}
+	require.Len(t, orphans, 1, "the provider kept the snapshot it took before the image read failed")
+
+	docs, err := s.metadata.ListSnapshotsStrict(ctx, ctAccountID)
+	require.NoError(t, err)
+	for _, doc := range docs {
+		assert.NotEqual(t, orphans[0], doc.SnapshotID, "no document names that snapshot")
+	}
+}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -4507,6 +4507,30 @@ func TestDescribeInstanceStatus_AccountFilteringHidesOtherTenant(t *testing.T) {
 	assert.Empty(t, out.InstanceStatuses)
 }
 
+// A platform-managed VM recorded under a customer account stays out of that
+// customer's status listing, matching DescribeInstances.
+func TestDescribeInstanceStatus_HidesManagedSystemVMFromCustomer(t *testing.T) {
+	owner := "111122223333"
+	v := runningVM("i-ekscp", owner)
+	v.ManagedBy = tags.ManagedByEKS
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	out, err := svc.DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceStatuses, "managed system VM must not appear in customer status listing")
+}
+
+func TestDescribeInstanceStatus_RootSeesManagedSystemVM(t *testing.T) {
+	v := runningVM("i-lb", awsidentifiers.GlobalAccountID)
+	v.ManagedBy = tags.ManagedByELBv2
+	svc := instanceStatusService(t, "az-a", map[string]*vm.VM{v.ID: v})
+
+	out, err := svc.DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, awsidentifiers.GlobalAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.InstanceStatuses, 1)
+	assert.Equal(t, "i-lb", *out.InstanceStatuses[0].InstanceId)
+}
+
 func TestDescribeInstanceStatus_StoppedExcludedByDefault(t *testing.T) {
 	owner := "111122223333"
 	stopped := runningVM("i-stop", owner)

spinifex/domains/ec2/instance/status_selection.go

@@ -59,7 +59,7 @@ func ParseStatusSelection(input *ec2.DescribeInstanceStatusInput, accountID stri
 // status is built. Split out so a caller can test membership without paying
 // for the projection.
 func (s StatusSelection) Includes(v *vm.VM) bool {
-	if v == nil || !IsInstanceVisible(s.accountID, v.AccountID) {
+	if v == nil || !IsInstanceVisibleToCaller(s.accountID, v) {
 		return false
 	}
 	if len(s.instanceIDs) > 0 && !s.instanceIDs[v.ID] {

spinifex/domains/ec2/key/account_cleanup.go

@@ -0,0 +1,75 @@
+package key
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// AccountCleanup is what one CleanupAccount run did. It holds key-pair IDs and
+// counts only; it is the owner's account of its attempts, not proof of
+// absence, which only a later ScanAccount gives.
+type AccountCleanup struct {
+	// DeletedKeyPairIDs completed the same owner delete DeleteKeyPair runs.
+	DeletedKeyPairIDs []string
+	// FailedKeyPairIDs had a store failure part way; the metadata remains.
+	FailedKeyPairIDs []string
+	// Referenced maps a kept key-pair ID to the reason the caller's predicate
+	// gave for keeping it.
+	Referenced map[string]string
+	// Orphan public-key objects removed, that failed to delete, or left because
+	// they were written after settledBefore and may belong to an in-flight create.
+	OrphansRemoved, OrphansFailed, OrphansUnsettled int
+	// OrphansReferenced counts public-key objects kept, by predicate reason.
+	OrphansReferenced map[string]int
+}
+
+// CleanupAccount deletes every readable key pair of the account through the
+// owner delete, and removes public-key objects no metadata names once they
+// were last written before settledBefore. A key pair or public-key object
+// for whose name referenced returns a reason is kept: IMDS serves the public
+// key by name to any instance naming it. Unreadable metadata, and public keys that
+// may belong to it, are left for the owner scan to report. A listing failure
+// is returned as an error and nothing is deleted.
+func (s *KeyServiceImpl) CleanupAccount(ctx context.Context, accountID string, settledBefore time.Time, referenced func(keyName string) string) (*AccountCleanup, error) {
+	records, err := s.classifyAccount(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	if referenced == nil {
+		referenced = func(string) string { return "" }
+	}
+	out := &AccountCleanup{Referenced: map[string]string{}, OrphansReferenced: map[string]int{}}
+	for _, kp := range records.live {
+		if reason := referenced(kp.name); reason != "" {
+			out.Referenced[kp.id] = reason
+			continue
+		}
+		if err := s.deleteKeyPairRecords(ctx, accountID, kp.name, kp.id); err != nil {
+			out.FailedKeyPairIDs = append(out.FailedKeyPairIDs, kp.id)
+			continue
+		}
+		out.DeletedKeyPairIDs = append(out.DeletedKeyPairIDs, kp.id)
+	}
+	for _, orphan := range records.orphans {
+		if reason := referenced(orphan.name); reason != "" {
+			out.OrphansReferenced[reason]++
+			continue
+		}
+		if orphan.modified == nil || !orphan.modified.Before(settledBefore) {
+			out.OrphansUnsettled++
+			continue
+		}
+		if err := s.deleteObject(ctx, orphan.key); err != nil {
+			slog.WarnContext(ctx, "Key-pair account cleanup: orphan public key delete failed", "err", err)
+			out.OrphansFailed++
+			continue
+		}
+		out.OrphansRemoved++
+	}
+	slog.InfoContext(ctx, "Key-pair account cleanup completed", "accountID", accountID,
+		"deleted", len(out.DeletedKeyPairIDs), "failed", len(out.FailedKeyPairIDs), "referenced", len(out.Referenced),
+		"orphansRemoved", out.OrphansRemoved, "orphansFailed", out.OrphansFailed,
+		"orphansUnsettled", out.OrphansUnsettled)
+	return out, nil
+}

spinifex/domains/ec2/key/account_scan.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"log/slog"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -29,10 +30,49 @@ type AccountScan struct {
 	ReadFailures int
 }
 
+// liveKeyPair is a readable metadata record and the name it gives its key.
+type liveKeyPair struct {
+	id, name string
+}
+
+// orphanObject is a public-key object no metadata names, the key name it is
+// served under, and its last write time when the listing reported one.
+type orphanObject struct {
+	key, name string
+	modified  *time.Time
+}
+
+// accountRecords is the private classification ScanAccount and CleanupAccount
+// share. It holds object keys and names, so it never leaves this package.
+type accountRecords struct {
+	live         []liveKeyPair
+	unreadable   []string
+	orphans      []orphanObject
+	unresolved   int
+	readFailures int
+}
+
 // ScanAccount lists and classifies every object under keys/<accountID>/. A
 // listing failure is returned as an error; per-object read failures are
 // counted. It reads only objects named as metadata (key-*.json).
 func (s *KeyServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
+	records, err := s.classifyAccount(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	scan := &AccountScan{
+		UnreadableKeyPairIDs: records.unreadable,
+		OrphanPublicKeys:     len(records.orphans),
+		UnresolvedPublicKeys: records.unresolved,
+		ReadFailures:         records.readFailures,
+	}
+	for _, kp := range records.live {
+		scan.LiveKeyPairIDs = append(scan.LiveKeyPairIDs, kp.id)
+	}
+	return scan, nil
+}
+
+func (s *KeyServiceImpl) classifyAccount(ctx context.Context, accountID string) (*accountRecords, error) {
 	prefix := fmt.Sprintf("keys/%s/", accountID)
 	objects, _, err := objectstore.ListAll(ctx, s.store, &s3.ListObjectsV2Input{
 		Bucket: aws.String(s.bucketName),
@@ -42,9 +82,9 @@ func (s *KeyServiceImpl) ScanAccount(ctx context.Context, accountID string) (*Ac
 		return nil, fmt.Errorf("list key-pair objects: %w", err)
 	}
 
-	scan := &AccountScan{}
+	records := &accountRecords{}
 	liveNames := make(map[string]bool)
-	var rest []string
+	var rest []*s3.Object
 	unreadable := make(map[string]bool)
 	for _, obj := range objects {
 		if obj == nil || obj.Key == nil {
@@ -53,24 +93,24 @@ func (s *KeyServiceImpl) ScanAccount(ctx context.Context, accountID string) (*Ac
 		name := strings.TrimPrefix(*obj.Key, prefix)
 		id, isJSON := strings.CutSuffix(name, ".json")
 		if !isJSON || !strings.HasPrefix(id, "key-") {
-			rest = append(rest, name)
+			rest = append(rest, obj)
 			continue
 		}
 		metadata, found, readErr := s.readScanMetadata(ctx, *obj.Key)
 		switch {
 		case readErr != nil:
-			scan.ReadFailures++
+			records.readFailures++
 			unreadable[name] = true
 		case !found:
 			// Removed between the listing and the read.
 		case metadata == nil || aws.StringValue(metadata.KeyPairId) != id || aws.StringValue(metadata.KeyName) == "":
 			unreadable[name] = true
 		default:
-			scan.LiveKeyPairIDs = append(scan.LiveKeyPairIDs, id)
+			records.live = append(records.live, liveKeyPair{id: id, name: *metadata.KeyName})
 			liveNames[*metadata.KeyName] = true
 		}
 		if unreadable[name] {
-			rest = append(rest, name)
+			rest = append(rest, obj)
 		}
 	}
 
@@ -81,19 +121,20 @@ func (s *KeyServiceImpl) ScanAccount(ctx context.Context, accountID string) (*Ac
 			delete(unreadable, name)
 		}
 	}
-	for _, name := range rest {
+	for _, obj := range rest {
+		name := strings.TrimPrefix(*obj.Key, prefix)
 		switch {
 		case liveNames[name]:
 			// The public key of a live key pair.
 		case unreadable[name]:
-			scan.UnreadableKeyPairIDs = append(scan.UnreadableKeyPairIDs, strings.TrimSuffix(name, ".json"))
+			records.unreadable = append(records.unreadable, strings.TrimSuffix(name, ".json"))
 		case len(unreadable) > 0:
-			scan.UnresolvedPublicKeys++
+			records.unresolved++
 		default:
-			scan.OrphanPublicKeys++
+			records.orphans = append(records.orphans, orphanObject{key: *obj.Key, name: name, modified: obj.LastModified})
 		}
 	}
-	return scan, nil
+	return records, nil
 }
 
 // readScanMetadata reads and decodes one metadata object. A decode failure is

spinifex/domains/ec2/key/account_scan_test.go

@@ -33,15 +33,12 @@ func TestScanAccountClassifiesEveryObjectUnderTheAccountPrefix(t *testing.T) {
 	live := importKey(t, svc, ctAccount, "live")
 	dotted := importKey(t, svc, ctAccount, "web.json")
 
-	importKey(t, svc, ctAccount, "deleted")
-	store.failDelete = func(key string) bool { return key == prefix+"deleted" }
-	_, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deleted")}, ctAccount)
-	require.NoError(t, err)
-	store.failDelete = nil
+	// A public key whose metadata is gone, as an interrupted delete leaves it.
+	putObject(t, store, prefix+"deleted", ctPubKey+"\n")
 
 	store.failPut = func(key string) bool { return strings.HasSuffix(key, ".json") }
 	store.failDelete = func(string) bool { return true }
-	_, err = svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{KeyName: aws.String("half")}, ctAccount)
+	_, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{KeyName: aws.String("half")}, ctAccount)
 	require.Error(t, err)
 	store.failPut, store.failDelete = nil, nil
 
@@ -50,7 +47,7 @@ func TestScanAccountClassifiesEveryObjectUnderTheAccountPrefix(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.ElementsMatch(t, []string{aws.StringValue(live.KeyPairId), aws.StringValue(dotted.KeyPairId)}, scan.LiveKeyPairIDs)
-	assert.Equal(t, 2, scan.OrphanPublicKeys, "the failed-delete and failed-create public keys")
+	assert.Equal(t, 2, scan.OrphanPublicKeys, "the interrupted-delete and failed-create public keys")
 	assert.Empty(t, scan.UnreadableKeyPairIDs)
 	assert.Zero(t, scan.UnresolvedPublicKeys)
 	assert.Zero(t, scan.ReadFailures)

spinifex/domains/ec2/key/account_termination_current_behaviour_test.go

@@ -7,6 +7,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -19,8 +20,8 @@ import (
 )
 
 // The TestCurrentBehaviour_AccountTermination_ tests pin how key pairs are
-// stored and what a delete or Describe leaves behind today. They describe
-// current behaviour, not the target design; a failure means it changed.
+// stored and what Describe leaves behind today; a failure means it changed.
+// The TestAccountTermination_ tests pin the corrected owner delete.
 
 const (
 	ctBucket  = "test-bucket"
@@ -152,35 +153,97 @@ func TestCurrentBehaviour_AccountTermination_KeyPairStorageLayout(t *testing.T)
 	}
 }
 
-// Pins current behaviour that differs from AWS: a failed public-key delete is
-// logged, the metadata delete proceeds and the call reports success. The
-// leftover object still blocks the name and passes launch validation.
-func TestCurrentBehaviour_AccountTermination_DeleteKeyPairLeavesPublicKeyWhenItsDeleteFails(t *testing.T) {
+// A key pair is deleted public key first and metadata last, so a failed step
+// returns an error and leaves the key pair described and found by name; a
+// retried delete then removes both objects and frees the name.
+func TestAccountTermination_DeleteKeyPairKeepsTheKeyPairWhenAStepFails(t *testing.T) {
+	prefix := "keys/" + ctAccount + "/"
+	tests := []struct {
+		name   string
+		failOn func(keyPairID string) string
+		left   func(keyPairID string) []string
+	}{
+		{"public key delete fails", func(string) string { return prefix + "deploy" },
+			func(id string) []string { return sortedStrings(prefix+"deploy", prefix+id+".json") }},
+		{"metadata delete fails", func(id string) string { return prefix + id + ".json" },
+			func(id string) []string { return []string{prefix + id + ".json"} }},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			store := newFaultyStore()
+			svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+			id := aws.StringValue(importKey(t, svc, ctAccount, "deploy").KeyPairId)
+			failing := tc.failOn(id)
+			store.failDelete = func(key string) bool { return key == failing }
+
+			_, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+			assert.Equal(t, tc.left(id), accountObjects(t, store, ctAccount))
+			assert.Equal(t, []string{"deploy"}, describedKeyNames(t, svc, ctAccount))
+
+			store.failDelete = nil
+			out, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+			require.NoError(t, err)
+			assert.Equal(t, id, aws.StringValue(out.KeyPairId))
+			assert.Empty(t, accountObjects(t, store, ctAccount))
+			assert.ErrorContains(t, svc.ValidateKeyPairExists(context.Background(), ctAccount, "deploy"), awserrors.ErrorInvalidKeyPairNotFound)
+
+			_, err = svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+			assert.NoError(t, err, "the name is free again")
+		})
+	}
+}
+
+// A public key left with no metadata is that name's key: a delete by name
+// removes it and reports success without a key pair ID, freeing the name.
+func TestAccountTermination_DeleteKeyPairByNameRemovesALeftoverPublicKey(t *testing.T) {
 	store := newFaultyStore()
 	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
-	imported := importKey(t, svc, ctAccount, "deploy")
-	pubKeyPath := "keys/" + ctAccount + "/deploy"
-	store.failDelete = func(key string) bool { return key == pubKeyPath }
+	putObject(t, store, "keys/"+ctAccount+"/orphan", ctPubKey+"\n")
 
-	out, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+	out, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("orphan")}, ctAccount)
 	require.NoError(t, err)
 	assert.True(t, aws.BoolValue(out.Return))
-	assert.Equal(t, aws.StringValue(imported.KeyPairId), aws.StringValue(out.KeyPairId))
+	assert.Nil(t, out.KeyPairId)
+	assert.Empty(t, accountObjects(t, store, ctAccount))
+	_, err = svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{KeyName: aws.String("orphan")}, ctAccount)
+	assert.NoError(t, err)
+}
 
-	assert.Equal(t, []string{pubKeyPath}, accountObjects(t, store, ctAccount), "only the public key object remains")
-	assert.Empty(t, describedKeyNames(t, svc, ctAccount))
-	assert.NoError(t, svc.ValidateKeyPairExists(context.Background(), ctAccount, "deploy"))
+// A by-name delete that cannot read some metadata cannot prove the name is
+// absent, so it fails rather than reporting an idempotent success.
+func TestAccountTermination_DeleteKeyPairByNameFailsWhenMetadataIsUnreadable(t *testing.T) {
+	store := newFaultyStore()
+	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+	id := aws.StringValue(importKey(t, svc, ctAccount, "deploy").KeyPairId)
+	store.failGet = func(key string) bool { return strings.HasSuffix(key, id+".json") }
 
-	// A retried delete finds no metadata, reports success and removes nothing.
-	store.failDelete = nil
-	_, err = svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
-	require.NoError(t, err)
-	assert.Equal(t, []string{pubKeyPath}, accountObjects(t, store, ctAccount))
-
-	_, err = svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+	_, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
 	require.Error(t, err)
-	code, _ := awserrors.ResolveErrorCode(err)
-	assert.Equal(t, awserrors.ErrorInvalidKeyPairDuplicate, code)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Len(t, accountObjects(t, store, ctAccount), 2, "nothing was deleted")
+}
+
+// Undecodable metadata may be the named key pair, so a by-name delete fails
+// and leaves both the metadata and the same-name public key in place.
+func TestAccountTermination_DeleteKeyPairByNameFailsOnUndecodableMetadata(t *testing.T) {
+	store := newFaultyStore()
+	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+	broken := "keys/" + ctAccount + "/key-0badbadbadbadbad0.json"
+	pub := "keys/" + ctAccount + "/deploy"
+	putObject(t, store, broken, "{not json")
+	putObject(t, store, pub, ctPubKey+"\n")
+
+	_, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Equal(t, sortedStrings(broken, pub), accountObjects(t, store, ctAccount))
+}
+
+func sortedStrings(s ...string) []string {
+	slices.Sort(s)
+	return s
 }
 
 // A create whose metadata write fails and whose cleanup delete also fails
@@ -231,3 +294,79 @@ func TestCurrentBehaviour_AccountTermination_DescribeKeyPairsHidesUnreadableReco
 		})
 	}
 }
+
+// Account cleanup deletes each readable key pair through the owner delete and
+// removes a settled public key no metadata names. A younger orphan may be a
+// create in flight and is left; unreadable metadata is never deleted.
+func TestAccountCleanup_DeletesKeyPairsAndSettledOrphans(t *testing.T) {
+	store := newFaultyStore()
+	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+	importKey(t, svc, ctAccount, "deploy")
+	importKey(t, svc, "999988887777", "theirs")
+	putObject(t, store, "keys/"+ctAccount+"/old-orphan", ctPubKey)
+	broken := "keys/" + ctAccount + "/key-0badbadbadbadbad0.json"
+	putObject(t, store, broken, "{not json")
+
+	out, err := svc.CleanupAccount(context.Background(), ctAccount, time.Now().Add(-time.Hour), nil)
+	require.NoError(t, err)
+	assert.Len(t, out.DeletedKeyPairIDs, 1)
+	assert.Zero(t, out.OrphansRemoved, "orphans are left while a public key may belong to unreadable metadata")
+
+	store.Clear()
+	importKey(t, svc, ctAccount, "deploy")
+	putObject(t, store, "keys/"+ctAccount+"/old-orphan", ctPubKey)
+	out, err = svc.CleanupAccount(context.Background(), ctAccount, time.Now().Add(-time.Hour), nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, out.OrphansUnsettled, "an orphan written inside the settle window is kept")
+	assert.Equal(t, []string{"keys/" + ctAccount + "/old-orphan"}, accountObjects(t, store, ctAccount))
+
+	out, err = svc.CleanupAccount(context.Background(), ctAccount, time.Now().Add(time.Second), nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, out.OrphansRemoved)
+	assert.Empty(t, accountObjects(t, store, ctAccount))
+}
+
+// A key pair, and an orphan public key, whose name the reference check names
+// are kept with the given reason: IMDS serves the public key by name.
+func TestAccountCleanup_KeepsReferencedKeyPairsAndPublicKeys(t *testing.T) {
+	store := newFaultyStore()
+	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+	kp := importKey(t, svc, ctAccount, "in-use")
+	importKey(t, svc, ctAccount, "free")
+	putObject(t, store, "keys/"+ctAccount+"/served-orphan", ctPubKey)
+	referenced := func(name string) string {
+		if name == "in-use" || name == "served-orphan" {
+			return "instance-reference"
+		}
+		return ""
+	}
+
+	out, err := svc.CleanupAccount(context.Background(), ctAccount, time.Now().Add(time.Second), referenced)
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{aws.StringValue(kp.KeyPairId): "instance-reference"}, out.Referenced)
+	assert.Len(t, out.DeletedKeyPairIDs, 1)
+	assert.Equal(t, map[string]int{"instance-reference": 1}, out.OrphansReferenced)
+	assert.Zero(t, out.OrphansRemoved)
+	assert.Equal(t, []string{"in-use"}, describedKeyNames(t, svc, ctAccount))
+	assert.Contains(t, accountObjects(t, store, ctAccount), "keys/"+ctAccount+"/served-orphan")
+}
+
+// A key pair whose delete fails part way is reported failed with its metadata
+// kept; the next run completes it.
+func TestAccountCleanup_InterruptedKeyPairDeleteCompletesOnRerun(t *testing.T) {
+	store := newFaultyStore()
+	svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
+	kp := importKey(t, svc, ctAccount, "deploy")
+	store.failDelete = func(key string) bool { return strings.HasSuffix(key, ".json") }
+
+	out, err := svc.CleanupAccount(context.Background(), ctAccount, time.Now(), nil)
+	require.NoError(t, err)
+	assert.Equal(t, []string{aws.StringValue(kp.KeyPairId)}, out.FailedKeyPairIDs)
+	assert.Equal(t, []string{"deploy"}, describedKeyNames(t, svc, ctAccount))
+
+	store.failDelete = nil
+	out, err = svc.CleanupAccount(context.Background(), ctAccount, time.Now(), nil)
+	require.NoError(t, err)
+	assert.Equal(t, []string{aws.StringValue(kp.KeyPairId)}, out.DeletedKeyPairIDs)
+	assert.Empty(t, accountObjects(t, store, ctAccount))
+}

spinifex/domains/ec2/key/service_impl.go

@@ -500,8 +500,12 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
 			Key:    obj.Key,
 		})
 		if err != nil {
-			slog.Debug("Failed to get metadata file", "key", *obj.Key, "err", err)
-			continue
+			if objectstore.IsNoSuchKeyError(err) {
+				continue
+			}
+			// Unread metadata may be the key pair asked for, so absence is unproven.
+			slog.ErrorContext(ctx, "Failed to get metadata file", "key", *obj.Key, "err", err)
+			return "", fmt.Errorf("read metadata %s: %w", *obj.Key, err)
 		}
 
 		body, err := io.ReadAll(getResult.Body)
@@ -509,14 +513,15 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
 			slog.Debug("Failed to close metadata body", "key", *obj.Key, "err", err)
 		}
 		if err != nil {
-			slog.Debug("Failed to read metadata body", "key", *obj.Key, "err", err)
-			continue
+			slog.ErrorContext(ctx, "Failed to read metadata body", "key", *obj.Key, "err", err)
+			return "", fmt.Errorf("read metadata %s: %w", *obj.Key, err)
 		}
 
 		metadata, err := decodeKeyPairMetadata(body)
 		if err != nil {
-			slog.Debug("Failed to unmarshal metadata", "key", *obj.Key, "err", err)
-			continue
+			// Undecodable metadata may be this name's key pair, so absence is unproven.
+			slog.ErrorContext(ctx, "Undecodable key-pair metadata", "key", *obj.Key, "err", err)
+			return "", fmt.Errorf("decode metadata %s: %w", *obj.Key, err)
 		}
 
 		// Check if this metadata matches the keyName
@@ -611,7 +616,7 @@ func (s *KeyServiceImpl) describedPublicKey(accountID, keyName string) (string,
 	return fmt.Sprintf("%s %s\n", bytes.TrimSpace(ssh.MarshalAuthorizedKey(publicKey)), keyName), nil
 }
 
-// DeleteKeyPair removes a key pair (both public key and metadata from S3).
+// DeleteKeyPair removes a key pair: its public key, metadata and tag-index entry.
 func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKeyPairInput, accountID string) (*ec2.DeleteKeyPairOutput, error) {
 	if input == nil {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -641,7 +646,7 @@ func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKey
 				return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true)}, nil
 			}
 			slog.ErrorContext(ctx, "Failed to get keyName from keyPairId", "keyPairId", keyPairID, "err", err)
-			return nil, err
+			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	} else if input.KeyName != nil && *input.KeyName != "" {
 		// KeyName provided - validate and find the keyPairId
@@ -655,50 +660,70 @@ func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKey
 
 		keyPairID, err = s.findKeyPairIdFromKeyName(ctx, accountID, keyName)
 		if err != nil {
-			// AWS DeleteKeyPair is idempotent — return success for non-existent keys
+			// AWS DeleteKeyPair is idempotent. A public key left with no metadata
+			// is this name's key, so it goes too and the name becomes reusable.
 			if err.Error() == awserrors.ErrorInvalidKeyPairNotFound {
+				if rmErr := s.deleteObject(ctx, publicKeyPath(accountID, keyName)); rmErr != nil {
+					slog.ErrorContext(ctx, "DeleteKeyPair: failed to remove leftover public key", "keyName", keyName, "err", rmErr)
+					return nil, errors.New(awserrors.ErrorServerInternal)
+				}
 				slog.DebugContext(ctx, "DeleteKeyPair: key pair not found, returning success (idempotent)", "keyName", keyName)
 				return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true)}, nil
 			}
 			slog.ErrorContext(ctx, "Failed to find keyPairId from keyName", "keyName", keyName, "err", err)
-			return nil, err
+			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	} else {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	slog.InfoContext(ctx, "Deleting key pair", "keyName", keyName, "keyPairId", keyPairID)
-
-	// Delete public key
-	publicKeyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName)
-	_, err = s.store.DeleteObject(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucketName),
-		Key:    aws.String(publicKeyPath),
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to delete public key", "path", publicKeyPath, "err", err)
-		// Continue to try deleting metadata even if public key deletion fails
-	}
-
-	// Delete metadata file (stored with keyPairID)
-	metadataPath := fmt.Sprintf("keys/%s/%s.json", accountID, keyPairID)
-	_, err = s.store.DeleteObject(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucketName),
-		Key:    aws.String(metadataPath),
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to delete metadata", "path", metadataPath, "err", err)
+	if err := s.deleteKeyPairRecords(ctx, accountID, keyName, keyPairID); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	s.clearRecordTags(ctx, accountID, keyPairID)
-
 	slog.InfoContext(ctx, "Key pair deleted successfully", "keyName", keyName, "keyPairId", keyPairID)
 
 	// AWS names the key only when one was actually deleted.
 	return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true), KeyPairId: aws.String(keyPairID)}, nil
 }
 
+// deleteKeyPairRecords removes the public key, then the metadata, then the
+// tag-index entry. Metadata goes last: while it exists the key pair is still
+// described and found by name or ID, so a retried delete finishes the job.
+func (s *KeyServiceImpl) deleteKeyPairRecords(ctx context.Context, accountID, keyName, keyPairID string) error {
+	if err := s.deleteObject(ctx, publicKeyPath(accountID, keyName)); err != nil {
+		slog.ErrorContext(ctx, "Failed to delete public key", "keyName", keyName, "keyPairId", keyPairID, "err", err)
+		return fmt.Errorf("delete public key: %w", err)
+	}
+	if err := s.deleteObject(ctx, metadataPath(accountID, keyPairID)); err != nil {
+		slog.ErrorContext(ctx, "Failed to delete key pair metadata", "keyPairId", keyPairID, "err", err)
+		return fmt.Errorf("delete metadata: %w", err)
+	}
+	s.clearRecordTags(ctx, accountID, keyPairID)
+	return nil
+}
+
+// deleteObject deletes one object; an absent object is already deleted.
+func (s *KeyServiceImpl) deleteObject(ctx context.Context, key string) error {
+	_, err := s.store.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(s.bucketName),
+		Key:    aws.String(key),
+	})
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		return err
+	}
+	return nil
+}
+
+func publicKeyPath(accountID, keyName string) string {
+	return fmt.Sprintf("keys/%s/%s", accountID, keyName)
+}
+
+func metadataPath(accountID, keyPairID string) string {
+	return fmt.Sprintf("keys/%s/%s.json", accountID, keyPairID)
+}
+
 // DescribeKeyPairs lists available key pairs by reading metadata files from S3
 // describeKeyPairsValidFilters defines the set of filter names accepted by DescribeKeyPairs.
 var describeKeyPairsValidFilters = map[string]bool{
@@ -769,7 +794,7 @@ func (s *KeyServiceImpl) DescribeKeyPairs(ctx context.Context, input *ec2.Descri
 			Key:    obj.Key,
 		})
 		if err != nil {
-			slog.DebugContext(ctx, "Failed to get metadata file", "key", *obj.Key, "err", err)
+			slog.WarnContext(ctx, "DescribeKeyPairs: skipping metadata the store cannot read", "key", *obj.Key, "err", err)
 			continue
 		}
 
@@ -778,13 +803,14 @@ func (s *KeyServiceImpl) DescribeKeyPairs(ctx context.Context, input *ec2.Descri
 			slog.DebugContext(ctx, "Failed to close metadata body", "key", *obj.Key, "err", err)
 		}
 		if err != nil {
-			slog.DebugContext(ctx, "Failed to read metadata body", "key", *obj.Key, "err", err)
+			slog.WarnContext(ctx, "DescribeKeyPairs: skipping metadata whose body cannot be read", "key", *obj.Key, "err", err)
 			continue
 		}
 
+		// The decoder's message is not logged: it can quote the stored bytes.
 		metadata, err := decodeKeyPairMetadata(body)
 		if err != nil {
-			slog.DebugContext(ctx, "Failed to unmarshal metadata", "key", *obj.Key, "err", err)
+			slog.WarnContext(ctx, "DescribeKeyPairs: skipping undecodable metadata", "key", *obj.Key)
 			continue
 		}
 

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -0,0 +1,132 @@
+package launchtemplate
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// AccountCleanup is what one CleanupAccountResidue run did. It holds template
+// IDs and counts only; it is the owner's account of its attempts, not proof of
+// absence, which only a later ScanAccount gives.
+type AccountCleanup struct {
+	// KeptTemplateIDs are live templates; account cleanup never deletes them.
+	KeptTemplateIDs []string
+	// Version bodies and name-index entries with no header: removed, failed,
+	// or left because they were written after settledBefore and may belong to
+	// a create still in flight.
+	OrphanBodiesRemoved, OrphanBodiesFailed, OrphanBodiesUnsettled int
+	OrphanNamesRemoved, OrphanNamesFailed, OrphanNamesUnsettled    int
+}
+
+// CleanupAccountResidue removes the account's version bodies and name-index
+// entries whose header is absent, once written before settledBefore: without
+// a header no ID, name or launch can resolve them. Live templates are listed,
+// not deleted, because not every record that may name one is visible to EC2.
+// Unreadable headers, their bodies and unrecognised entries are left for the
+// owner scan to report. A listing failure is returned and nothing is removed.
+func (s *LaunchTemplateServiceImpl) CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*AccountCleanup, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list launch-template keys: %w", err)
+	}
+
+	prefix := accountID + "."
+	present := make(map[string]bool)
+	out := &AccountCleanup{}
+	var bodies, names []string
+	for _, key := range keys {
+		rest, ok := strings.CutPrefix(key, prefix)
+		if !ok {
+			continue
+		}
+		switch {
+		case isHeaderKey(key, prefix):
+			entry, err := s.kv.Get(ctx, key)
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				continue
+			}
+			// An unreadable header still owns its bodies and name.
+			present[rest] = true
+			if err != nil {
+				continue
+			}
+			var header LaunchTemplateHeader
+			if json.Unmarshal(entry.Value(), &header) == nil && header.LaunchTemplateId == rest {
+				out.KeptTemplateIDs = append(out.KeptTemplateIDs, rest)
+			}
+		case strings.HasPrefix(rest, "name."):
+			names = append(names, key)
+		case versionBodyTemplateID(rest) != "":
+			bodies = append(bodies, key)
+		}
+	}
+
+	for _, key := range bodies {
+		if present[versionBodyTemplateID(strings.TrimPrefix(key, prefix))] {
+			continue
+		}
+		tallyOrphan(s.removeSettled(ctx, key, settledBefore, nil), &out.OrphanBodiesRemoved, &out.OrphanBodiesFailed, &out.OrphanBodiesUnsettled)
+	}
+	for _, key := range names {
+		// A name still naming a present header is that template's, not residue.
+		owned := func(value []byte) bool { return present[string(value)] }
+		tallyOrphan(s.removeSettled(ctx, key, settledBefore, owned), &out.OrphanNamesRemoved, &out.OrphanNamesFailed, &out.OrphanNamesUnsettled)
+	}
+	slog.InfoContext(ctx, "Launch-template account residue cleanup completed", "accountID", accountID,
+		"kept", len(out.KeptTemplateIDs), "orphanBodiesRemoved", out.OrphanBodiesRemoved, "orphanNamesRemoved", out.OrphanNamesRemoved)
+	return out, nil
+}
+
+// orphanOutcome is what removeSettled did with one headerless record.
+type orphanOutcome int
+
+const (
+	orphanGone orphanOutcome = iota
+	orphanRemoved
+	orphanFailed
+	orphanUnsettled
+	orphanOwned
+)
+
+// removeSettled deletes key at the revision read when it was written before
+// settledBefore and owned does not claim its value. The value is never decoded
+// or logged: a version body holds template data and user data.
+func (s *LaunchTemplateServiceImpl) removeSettled(ctx context.Context, key string, settledBefore time.Time, owned func([]byte) bool) orphanOutcome {
+	entry, err := s.kv.Get(ctx, key)
+	switch {
+	case errors.Is(err, jetstream.ErrKeyNotFound):
+		return orphanGone
+	case err != nil:
+		slog.WarnContext(ctx, "Launch-template account cleanup: orphan read failed", "err", err)
+		return orphanFailed
+	case owned != nil && owned(entry.Value()):
+		return orphanOwned
+	case !entry.Created().Before(settledBefore):
+		return orphanUnsettled
+	}
+	if err := s.kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
+		slog.WarnContext(ctx, "Launch-template account cleanup: orphan delete failed", "err", err)
+		return orphanFailed
+	}
+	return orphanRemoved
+}
+
+func tallyOrphan(outcome orphanOutcome, removed, failed, unsettled *int) {
+	switch outcome {
+	case orphanRemoved:
+		*removed++
+	case orphanFailed:
+		*failed++
+	case orphanUnsettled:
+		*unsettled++
+	case orphanGone, orphanOwned:
+	}
+}

spinifex/domains/ec2/launchtemplate/account_scan_test.go

@@ -6,7 +6,6 @@ package launchtemplate
 import (
 	"context"
 	"encoding/base64"
-	"strings"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -43,17 +42,17 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	svc := setupTestService(t)
 	live := createWithUserData(t, svc, "live")
 
+	// A header removed alone, as a delete interrupted after it leaves the rest.
 	deleted := createWithUserData(t, svc, "deleted")
 	inner := svc.kv
-	deletedHeader, deletedName := kvutil.AccountKey(testAccountID, deleted), nameKey(testAccountID, "deleted")
-	svc.kv = &faultyKV{KeyValue: inner, failDelete: func(key string) bool {
-		return strings.HasPrefix(key, deletedHeader+".v") || key == deletedName
-	}}
-	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(deleted)}, testAccountID)
-	require.NoError(t, err)
+	require.NoError(t, inner.Delete(t.Context(), kvutil.AccountKey(testAccountID, deleted)))
 
-	svc.kv = &faultyKV{KeyValue: inner, failPut: func(key string) bool { return isHeaderKey(key, testAccountID+".") }}
-	_, err = svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+	// A create whose header write and clean-up both fail.
+	svc.kv = &faultyKV{KeyValue: inner,
+		failPut:    func(key string) bool { return isHeaderKey(key, testAccountID+".") },
+		failDelete: func(string) bool { return true },
+	}
+	_, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String("interrupted"),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(ctUserData)))},
 	}, testAccountID)

spinifex/domains/ec2/launchtemplate/account_termination_current_behaviour_test.go

@@ -9,7 +9,9 @@ import (
 	"encoding/json"
 	"errors"
 	"slices"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -21,8 +23,8 @@ import (
 )
 
 // The TestCurrentBehaviour_AccountTermination_ tests pin how launch templates
-// are stored and what Describe and Delete leave behind today. They describe
-// current behaviour, not the target design; a failure means it changed.
+// are stored and what Describe leaves behind today; a failure means it
+// changed. The TestAccountTermination_ tests pin the corrected owner delete.
 
 var errInjected = errors.New("injected store failure")
 
@@ -31,10 +33,31 @@ var errInjected = errors.New("injected store failure")
 type faultyKV struct {
 	jetstream.KeyValue
 
-	failGet    func(key string) bool
-	failPut    func(key string) bool
-	failDelete func(key string) bool
-	failKeys   func() bool
+	failGet      func(key string) bool
+	failPut      func(key string) bool
+	failDelete   func(key string) bool
+	failKeys     func() bool
+	failUpdate   func(key string) bool
+	beforeDelete func(key string)
+	beforeCreate func(key string)
+	beforeUpdate func(key string)
+}
+
+func (k *faultyKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if k.beforeCreate != nil {
+		k.beforeCreate(key)
+	}
+	return k.KeyValue.Create(ctx, key, value, opts...)
+}
+
+func (k *faultyKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
+	if k.beforeUpdate != nil {
+		k.beforeUpdate(key)
+	}
+	if k.failUpdate != nil && k.failUpdate(key) {
+		return 0, errInjected
+	}
+	return k.KeyValue.Update(ctx, key, value, revision)
 }
 
 func (k *faultyKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
@@ -52,6 +75,9 @@ func (k *faultyKV) Put(ctx context.Context, key string, value []byte) (uint64, e
 }
 
 func (k *faultyKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	if k.beforeDelete != nil {
+		k.beforeDelete(key)
+	}
 	if k.failDelete != nil && k.failDelete(key) {
 		return errInjected
 	}
@@ -138,40 +164,44 @@ func TestCurrentBehaviour_AccountTermination_LaunchTemplateStorageLayout(t *test
 	assert.Equal(t, ctUserData, storedUserData(t, svc.kv, header+".v2"), "a version copied from v1 keeps its user data")
 }
 
-// Pins current behaviour that differs from AWS: the header delete alone
-// decides success, and a failed name-index or version-body cleanup leaves
-// those records, user data included, with no header to find them by.
-func TestCurrentBehaviour_AccountTermination_DeleteLaunchTemplateLeavesRecordsWhenCleanupFails(t *testing.T) {
+// The header is deleted last, so a failed body, listing, name-index or header
+// delete returns an error with the template still described, and a retried
+// delete by ID removes every record, user data included.
+func TestAccountTermination_DeleteLaunchTemplateKeepsTheTemplateWhenAStepFails(t *testing.T) {
 	tests := []struct {
-		name     string
-		inject   func(kv *faultyKV, header, nameIdx string)
-		residual func(header, nameIdx string) []string
+		name   string
+		inject func(kv *faultyKV, header, nameIdx string)
+		left   func(header, nameIdx string) []string
 	}{
 		{
 			name: "version body deletes fail",
 			inject: func(kv *faultyKV, header, _ string) {
 				kv.failDelete = func(key string) bool { return key == header+".v1" || key == header+".v2" }
 			},
-			residual: func(header, _ string) []string { return []string{header + ".v1", header + ".v2"} },
+			left: func(header, nameIdx string) []string {
+				return []string{header, header + ".v1", header + ".v2", nameIdx}
+			},
+		},
+		{
+			name:   "version listing fails",
+			inject: func(kv *faultyKV, _, _ string) { kv.failKeys = func() bool { return true } },
+			left: func(header, nameIdx string) []string {
+				return []string{header, header + ".v1", header + ".v2", nameIdx}
+			},
 		},
 		{
 			name: "name index delete fails",
 			inject: func(kv *faultyKV, _, nameIdx string) {
 				kv.failDelete = func(key string) bool { return key == nameIdx }
 			},
-			residual: func(_, nameIdx string) []string { return []string{nameIdx} },
+			left: func(header, nameIdx string) []string { return []string{header, nameIdx} },
 		},
 		{
-			name: "version listing fails after the header delete",
+			name: "header delete fails",
 			inject: func(kv *faultyKV, header, _ string) {
-				headerGone := false
-				kv.failDelete = func(key string) bool {
-					headerGone = headerGone || key == header
-					return false
-				}
-				kv.failKeys = func() bool { return headerGone }
+				kv.failDelete = func(key string) bool { return key == header }
 			},
-			residual: func(header, _ string) []string { return []string{header + ".v1", header + ".v2"} },
+			left: func(header, _ string) []string { return []string{header} },
 		},
 	}
 	for _, tc := range tests {
@@ -184,56 +214,85 @@ func TestCurrentBehaviour_AccountTermination_DeleteLaunchTemplateLeavesRecordsWh
 			tc.inject(faulty, header, nameIdx)
 			svc.kv = faulty
 
+			_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+
+			svc.kv = inner
+			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvutil.VersionKey)...), storedKeys(t, inner))
+			assert.Equal(t, []string{id}, describedTemplateIDs(t, svc, testAccountID))
+
 			out, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 			require.NoError(t, err)
 			assert.Equal(t, id, aws.StringValue(out.LaunchTemplate.LaunchTemplateId))
-
-			svc.kv = inner
-			want := append(tc.residual(header, nameIdx), kvutil.VersionKey)
-			assert.Equal(t, sortedKeys(want...), storedKeys(t, inner))
+			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
 			assert.Empty(t, describedTemplateIDs(t, svc, testAccountID))
-			if slices.Contains(want, header+".v1") {
-				assert.Equal(t, ctUserData, storedUserData(t, inner, header+".v1"))
-			}
 		})
 	}
 }
 
-// A create interrupted before its header write leaves the name claim and
-// version 1 behind. A retry under the same name reclaims the name, and the
-// first attempt's body is never reachable again.
-func TestCurrentBehaviour_AccountTermination_InterruptedCreateLeavesAnOrphanBody(t *testing.T) {
+// A header rewritten and a version added while the delete runs send it back
+// through the bodies at the new revision, and the late body is removed too.
+func TestAccountTermination_DeleteLaunchTemplateRestartsWhenTheHeaderChanges(t *testing.T) {
 	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	svc.kv = &faultyKV{KeyValue: inner, failPut: func(key string) bool { return isHeaderKey(key, testAccountID+".") }}
-
-	_, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
-		LaunchTemplateName: aws.String("web"),
-		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
-			UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(ctUserData))),
-		},
-	}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-
-	svc.kv = inner
-	keys := storedKeys(t, inner)
-	require.Len(t, keys, 3, "name index, version 1 and the bucket version key")
-	nameEntry, err := inner.Get(t.Context(), nameKey(testAccountID, "web"))
+	header := testAccountID + "." + id
+	headerEntry, err := inner.Get(t.Context(), header)
 	require.NoError(t, err)
-	orphanBody := testAccountID + "." + string(nameEntry.Value()) + ".v1"
-	assert.Contains(t, keys, orphanBody)
-	assert.Equal(t, ctUserData, storedUserData(t, inner, orphanBody))
-	assert.Empty(t, describedTemplateIDs(t, svc, testAccountID))
 
-	retried := createTemplate(t, svc, "web", "t3.micro")
-	assert.NotEqual(t, string(nameEntry.Value()), aws.StringValue(retried.LaunchTemplateId))
-	assert.Contains(t, storedKeys(t, inner), orphanBody)
-	assert.Equal(t, []string{aws.StringValue(retried.LaunchTemplateId)}, describedTemplateIDs(t, svc, testAccountID))
+	raced := false
+	svc.kv = &faultyKV{KeyValue: inner, beforeDelete: func(key string) {
+		if key != header || raced {
+			return
+		}
+		raced = true
+		_, err := inner.Put(t.Context(), header, headerEntry.Value())
+		require.NoError(t, err)
+		_, err = inner.Put(t.Context(), header+".v3", []byte(`{"version_number":3}`))
+		require.NoError(t, err)
+	}}
+
+	_, err = svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, raced)
+	assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
 }
 
-// Describe skips a header it cannot read or decode, and still succeeds. An
-// undecodable header also cannot be removed through DeleteLaunchTemplate.
+// A create that fails after claiming the name removes what it wrote, so no
+// body is orphaned and a retry under the same name succeeds.
+func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
+	for _, failing := range []string{"version", "header"} {
+		t.Run(failing+" write fails", func(t *testing.T) {
+			svc := setupTestService(t)
+			inner := svc.kv
+			svc.kv = &faultyKV{KeyValue: inner, failPut: func(key string) bool {
+				if failing == "header" {
+					return isHeaderKey(key, testAccountID+".")
+				}
+				return versionBodyTemplateID(strings.TrimPrefix(key, testAccountID+".")) != ""
+			}}
+
+			_, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+				LaunchTemplateName: aws.String("web"),
+				LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+					UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(ctUserData))),
+				},
+			}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+
+			svc.kv = inner
+			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+			retried := createTemplate(t, svc, "web", "t3.micro")
+			assert.Equal(t, []string{aws.StringValue(retried.LaunchTemplateId)}, describedTemplateIDs(t, svc, testAccountID))
+		})
+	}
+}
+
+// Describe skips a header it cannot read or decode, and still succeeds. Such
+// a header cannot be removed through DeleteLaunchTemplate either, and a read
+// failure is an internal error, never a missing template.
 func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreadableHeaders(t *testing.T) {
 	const brokenID = "lt-0badbadbadbadbad0"
 	brokenHeader := kvutil.AccountKey(testAccountID, brokenID)
@@ -252,7 +311,7 @@ func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreada
 			_, err = svc.kv.Put(t.Context(), brokenHeader, data)
 			require.NoError(t, err)
 			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == brokenHeader }}
-		}, awserrors.ErrorInvalidLaunchTemplateIdNotFound},
+		}, awserrors.ErrorServerInternal},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -271,3 +330,124 @@ func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreada
 		})
 	}
 }
+
+// Account cleanup never deletes a live template: every key of a readable
+// template stays and its ID is reported kept. Headerless bodies and names are
+// removed once settled; a record younger than the settle window, and an
+// unreadable header with its bodies and name, are kept.
+func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
+	svc := setupTestService(t)
+	live := createWithUserData(t, svc, "web")
+	liveKeys := accountKeys(t, svc.kv, testAccountID)
+	other := "999988887777"
+	_, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("theirs"), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ImageId: aws.String("ami-123")},
+	}, other)
+	require.NoError(t, err)
+	orphanBody := versionKey(testAccountID, "lt-0orphan000000000", 1)
+	orphanName := nameKey(testAccountID, "gone")
+	brokenHeader := kvutil.AccountKey(testAccountID, "lt-0broken000000000")
+	brokenBody := versionKey(testAccountID, "lt-0broken000000000", 1)
+	brokenName := nameKey(testAccountID, "broken")
+	for key, value := range map[string]string{
+		orphanBody: `{"version_number":1}`, orphanName: "lt-0orphan000000000",
+		brokenHeader: "{not json", brokenBody: `{"version_number":1}`, brokenName: "lt-0broken000000000",
+	} {
+		_, err := svc.kv.Put(t.Context(), key, []byte(value))
+		require.NoError(t, err)
+	}
+	keep := sortedKeys(append(liveKeys, brokenHeader, brokenBody, brokenName)...)
+
+	out, err := svc.CleanupAccountResidue(t.Context(), testAccountID, time.Now().Add(-time.Hour))
+	require.NoError(t, err)
+	assert.Equal(t, []string{live}, out.KeptTemplateIDs)
+	assert.Equal(t, 1, out.OrphanBodiesUnsettled)
+	assert.Equal(t, 1, out.OrphanNamesUnsettled)
+	assert.Equal(t, sortedKeys(append(keep, orphanBody, orphanName)...), accountKeys(t, svc.kv, testAccountID))
+
+	out, err = svc.CleanupAccountResidue(t.Context(), testAccountID, time.Now().Add(time.Second))
+	require.NoError(t, err)
+	assert.Equal(t, []string{live}, out.KeptTemplateIDs)
+	assert.Equal(t, 1, out.OrphanBodiesRemoved)
+	assert.Equal(t, 1, out.OrphanNamesRemoved)
+	assert.Equal(t, keep, accountKeys(t, svc.kv, testAccountID))
+	assert.Equal(t, []string{live}, describedTemplateIDs(t, svc, testAccountID))
+	assert.Len(t, describedTemplateIDs(t, svc, other), 1)
+}
+
+// A public delete interrupted at its header leaves the template live, so the
+// account cleanup keeps it rather than finishing it; the public retry does.
+func TestAccountCleanup_InterruptedTemplateDeleteIsLeftToThePublicRetry(t *testing.T) {
+	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
+	header := kvutil.AccountKey(testAccountID, id)
+	kv := svc.kv
+	svc.kv = &faultyKV{KeyValue: kv, failDelete: func(key string) bool { return key == header }}
+	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+	require.Error(t, err)
+	svc.kv = kv
+	require.Equal(t, []string{header}, accountKeys(t, kv, testAccountID), "only the header, the delete's anchor, remains")
+
+	out, err := svc.CleanupAccountResidue(t.Context(), testAccountID, time.Now().Add(time.Second))
+	require.NoError(t, err)
+	assert.Equal(t, []string{id}, out.KeptTemplateIDs)
+	assert.Equal(t, []string{header}, accountKeys(t, kv, testAccountID))
+
+	_, err = svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, accountKeys(t, kv, testAccountID))
+}
+
+// A listing failure is returned and nothing is deleted.
+func TestAccountCleanup_TemplateListingFailureDeletesNothing(t *testing.T) {
+	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
+	kv := svc.kv
+	svc.kv = &faultyKV{KeyValue: kv, failKeys: func() bool { return true }}
+
+	_, err := svc.CleanupAccountResidue(t.Context(), testAccountID, time.Now())
+	require.Error(t, err)
+	svc.kv = kv
+	assert.Equal(t, []string{id}, describedTemplateIDs(t, svc, testAccountID))
+}
+
+func accountKeys(t *testing.T, kv jetstream.KeyValue, account string) []string {
+	t.Helper()
+	var mine []string
+	for _, key := range storedKeys(t, kv) {
+		if strings.HasPrefix(key, account+".") {
+			mine = append(mine, key)
+		}
+	}
+	return mine
+}
+
+// A settled orphan the store cannot read or delete is counted failed, never
+// removed, and is still there for the next run.
+func TestAccountCleanup_OrphanReadOrDeleteFailureIsCountedFailed(t *testing.T) {
+	for name, fault := range map[string]func(kv *faultyKV, key string){
+		"read":   func(kv *faultyKV, key string) { kv.failGet = func(k string) bool { return k == key } },
+		"delete": func(kv *faultyKV, key string) { kv.failDelete = func(k string) bool { return k == key } },
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc := setupTestService(t)
+			body := versionKey(testAccountID, "lt-0orphan000000000", 1)
+			nameIdx := nameKey(testAccountID, "gone")
+			for key, value := range map[string]string{body: `{"version_number":1}`, nameIdx: "lt-0orphan000000000"} {
+				_, err := svc.kv.Put(t.Context(), key, []byte(value))
+				require.NoError(t, err)
+			}
+			inner := svc.kv
+			faulty := &faultyKV{KeyValue: inner}
+			fault(faulty, body)
+			svc.kv = faulty
+
+			out, err := svc.CleanupAccountResidue(t.Context(), testAccountID, time.Now().Add(time.Second))
+			require.NoError(t, err)
+			assert.Equal(t, 1, out.OrphanBodiesFailed)
+			assert.Zero(t, out.OrphanBodiesRemoved)
+			assert.Equal(t, 1, out.OrphanNamesRemoved, "a fault on one record does not stop the rest")
+			assert.Equal(t, []string{body}, accountKeys(t, inner, testAccountID))
+		})
+	}
+}

spinifex/domains/ec2/launchtemplate/service_impl.go

@@ -52,6 +52,9 @@ type LaunchTemplateServiceImpl struct {
 	config   *config.Config
 	natsConn *nats.Conn
 	kv       jetstream.KeyValue
+
+	// Optional: nil leaves the central tag index untouched by delete.
+	centralTags CentralTagStore
 }
 
 // NewLaunchTemplateServiceImplWithNATS creates a launch template service with NATS JetStream.
@@ -103,7 +106,12 @@ func versionPrefix(accountID, ltID string) string {
 func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
 	entry, err := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID))
 	if err != nil {
-		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
+		}
+		// A header the store cannot return now is not proven absent.
+		slog.ErrorContext(ctx, "Failed to read launch template header", "launchTemplateId", ltID, "err", err)
+		return nil, nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	var h LaunchTemplateHeader
 	if err := json.Unmarshal(entry.Value(), &h); err != nil {
@@ -117,12 +125,19 @@ func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID
 func (s *LaunchTemplateServiceImpl) getHeaderByName(ctx context.Context, accountID, name string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
 	entry, err := s.kv.Get(ctx, nameKey(accountID, name))
 	if err != nil {
-		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException)
+		}
+		slog.ErrorContext(ctx, "Failed to read launch template name index", "err", err)
+		return nil, nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	ltID := string(entry.Value())
 	h, hentry, err := s.getHeaderByID(ctx, accountID, ltID)
 	if err != nil {
-		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException)
+		if err.Error() == awserrors.ErrorInvalidLaunchTemplateIdNotFound {
+			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException)
+		}
+		return nil, nil, err
 	}
 	return h, hentry, nil
 }
@@ -291,6 +306,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in
 		Data:               data,
 	}
 	if err := s.putVersion(ctx, accountID, ltID, &rec); err != nil {
+		s.abandonCreate(ctx, accountID, name, ltID, false)
 		return nil, err
 	}
 
@@ -304,6 +320,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in
 		Tags:                 awstags.Extract(input.TagSpecifications, launchTemplateTagResourceType),
 	}
 	if err := s.putHeader(ctx, accountID, &header); err != nil {
+		s.abandonCreate(ctx, accountID, name, ltID, true)
 		return nil, err
 	}
 
@@ -314,6 +331,39 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in
 	}, nil
 }
 
+// abandonCreate removes what a failed create wrote, newest first. A failure
+// is logged; what remains has no header, and the account cleanup removes it
+// once it is older than an in-flight create could be.
+func (s *LaunchTemplateServiceImpl) abandonCreate(ctx context.Context, accountID, name, ltID string, wroteVersion bool) {
+	if wroteVersion {
+		if err := s.kv.Delete(ctx, versionKey(accountID, ltID, 1)); err != nil {
+			slog.ErrorContext(ctx, "CreateLaunchTemplate: failed to remove version 1 after a failed create", "launchTemplateId", ltID, "err", err)
+		}
+	}
+	if err := s.releaseName(ctx, accountID, name, ltID); err != nil {
+		slog.ErrorContext(ctx, "CreateLaunchTemplate: failed to release the name after a failed create", "launchTemplateId", ltID, "err", err)
+	}
+}
+
+// releaseName removes the name index only while it still names ltID, at the
+// revision read, so a name another template now holds is never released.
+func (s *LaunchTemplateServiceImpl) releaseName(ctx context.Context, accountID, name, ltID string) error {
+	key := nameKey(accountID, name)
+	entry, err := s.kv.Get(ctx, key)
+	switch {
+	case errors.Is(err, jetstream.ErrKeyNotFound):
+		return nil
+	case err != nil:
+		return fmt.Errorf("read name index: %w", err)
+	case string(entry.Value()) != ltID:
+		return nil
+	}
+	if err := s.kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
+		return fmt.Errorf("delete name index: %w", err)
+	}
+	return nil
+}
+
 // errNameInUse is AWS's error for a launch template name already taken.
 func errNameInUse() error {
 	return awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException, "Launch template name already in use.")
@@ -377,7 +427,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont
 	if input.LaunchTemplateData == nil {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
-	header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName)
+	header, entry, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName)
 	if err != nil {
 		return nil, err
 	}
@@ -429,6 +479,13 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont
 			slog.ErrorContext(ctx, "CreateLaunchTemplateVersion: version-key create failed", "next", next, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
+		if err := s.commitVersion(ctx, accountID, header.LaunchTemplateId, entry); err != nil {
+			s.abandonVersion(ctx, accountID, header.LaunchTemplateId, next)
+			if errors.Is(err, errTemplateGone) {
+				return nil, templateNotFound(input.LaunchTemplateId, input.LaunchTemplateName)
+			}
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
 		slog.InfoContext(ctx, "CreateLaunchTemplateVersion completed", "launchTemplateId", header.LaunchTemplateId, "version", next, "accountID", accountID)
 		return &ec2.CreateLaunchTemplateVersionOutput{
 			LaunchTemplateVersion: versionToEC2(header, &rec),
@@ -439,6 +496,53 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont
 	return nil, errors.New(awserrors.ErrorServerInternal)
 }
 
+// errTemplateGone: the header was deleted while a version was being added.
+var errTemplateGone = errors.New("launch template deleted")
+
+// commitVersion rewrites the header's current value under a revision guard
+// after a new body is written. A delete that removed the header at any earlier
+// revision makes it fail, so the body is never left without its template.
+func (s *LaunchTemplateServiceImpl) commitVersion(ctx context.Context, accountID, ltID string, entry jetstream.KeyValueEntry) error {
+	key := kvutil.AccountKey(accountID, ltID)
+	for range maxVersionRetries {
+		_, err := s.kv.Update(ctx, key, entry.Value(), entry.Revision())
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, jetstream.ErrKeyRevisionMismatch) && !errors.Is(err, jetstream.ErrKeyExists) {
+			slog.ErrorContext(ctx, "CreateLaunchTemplateVersion: header commit failed", "launchTemplateId", ltID, "err", err)
+			return err
+		}
+		next, gerr := s.kv.Get(ctx, key)
+		if errors.Is(gerr, jetstream.ErrKeyNotFound) {
+			return errTemplateGone
+		}
+		if gerr != nil {
+			return gerr
+		}
+		entry = next
+	}
+	return errors.New("header commit retries exhausted")
+}
+
+// abandonVersion removes a body whose header commit failed. A failure is
+// logged and leaves a headerless body, which no ID, name or launch resolves.
+func (s *LaunchTemplateServiceImpl) abandonVersion(ctx context.Context, accountID, ltID string, n int64) {
+	if err := s.kv.Delete(ctx, versionKey(accountID, ltID, n)); err != nil {
+		slog.ErrorContext(ctx, "CreateLaunchTemplateVersion: failed to remove an uncommitted version", "launchTemplateId", ltID, "version", n, "err", err)
+	}
+}
+
+// templateNotFound is the NotFound error for the template the request named.
+func templateNotFound(ltID, name *string) error {
+	if id := aws.StringValue(ltID); id != "" {
+		return awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateIdNotFound,
+			"The specified launch template, with template ID %s, does not exist.", id)
+	}
+	return awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException,
+		"The specified launch template, with template name %s, does not exist.", aws.StringValue(name))
+}
+
 // --- ModifyLaunchTemplate ---
 
 // ModifyLaunchTemplate implements the EC2 action. ClientToken is accepted but not stored: this only
@@ -481,8 +585,11 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in
 
 // --- DeleteLaunchTemplate ---
 
+// DeleteLaunchTemplate deletes a template and every version. A store failure
+// returns an error with the header still in place, so the template stays
+// visible and a retried delete finishes it.
 func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, input *ec2.DeleteLaunchTemplateInput, accountID string) (*ec2.DeleteLaunchTemplateOutput, error) {
-	header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName)
+	header, entry, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName)
 	if err != nil {
 		return nil, err
 	}
@@ -493,28 +600,92 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, in
 	}
 	out := headerToEC2(header, latest)
 
-	// Delete the header first: the template immediately vanishes from every
-	// describe. Version bodies and the name index are best-effort cleanup.
-	if err := s.kv.Delete(ctx, kvutil.AccountKey(accountID, header.LaunchTemplateId)); err != nil {
+	if err := s.deleteTemplate(ctx, accountID, header, entry); err != nil {
+		slog.ErrorContext(ctx, "DeleteLaunchTemplate failed", "launchTemplateId", header.LaunchTemplateId, "accountID", accountID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if err := s.kv.Delete(ctx, nameKey(accountID, header.LaunchTemplateName)); err != nil {
-		slog.WarnContext(ctx, "DeleteLaunchTemplate: name index cleanup failed", "name", header.LaunchTemplateName, "err", err)
-	}
-	nums, err := s.listVersionNumbers(ctx, accountID, header.LaunchTemplateId)
-	if err != nil {
-		slog.WarnContext(ctx, "DeleteLaunchTemplate: version enumeration failed, bodies left in place", "launchTemplateId", header.LaunchTemplateId, "err", err)
-	}
-	for _, n := range nums {
-		if err := s.kv.Delete(ctx, versionKey(accountID, header.LaunchTemplateId, n)); err != nil {
-			slog.WarnContext(ctx, "DeleteLaunchTemplate: version cleanup failed", "launchTemplateId", header.LaunchTemplateId, "version", n, "err", err)
-		}
-	}
 
 	slog.InfoContext(ctx, "DeleteLaunchTemplate completed", "launchTemplateId", header.LaunchTemplateId, "accountID", accountID)
 	return &ec2.DeleteLaunchTemplateOutput{LaunchTemplate: out}, nil
 }
 
+// maxHeaderDeleteAttempts bounds restarts when the header changes mid-delete.
+const maxHeaderDeleteAttempts = 3
+
+// deleteTemplate removes every version body, then the name index, then the
+// header at the revision read, and last any body a concurrent version create
+// added meanwhile. The header goes last so every earlier failure is retryable
+// through the public delete; a header changed in between restarts the sweep.
+func (s *LaunchTemplateServiceImpl) deleteTemplate(ctx context.Context, accountID string, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry) error {
+	ltID := header.LaunchTemplateId
+	headerKey := kvutil.AccountKey(accountID, ltID)
+	for attempt := 1; ; attempt++ {
+		if err := s.deleteVersionBodies(ctx, accountID, ltID); err != nil {
+			return err
+		}
+		if err := s.releaseName(ctx, accountID, header.LaunchTemplateName, ltID); err != nil {
+			return err
+		}
+		err := s.kv.Delete(ctx, headerKey, jetstream.LastRevision(entry.Revision()))
+		if err == nil {
+			break
+		}
+		if (!errors.Is(err, jetstream.ErrKeyRevisionMismatch) && !errors.Is(err, jetstream.ErrKeyExists)) || attempt >= maxHeaderDeleteAttempts {
+			return fmt.Errorf("delete header: %w", err)
+		}
+		next, gerr := s.kv.Get(ctx, headerKey)
+		if errors.Is(gerr, jetstream.ErrKeyNotFound) {
+			break
+		}
+		if gerr != nil {
+			return fmt.Errorf("re-read header: %w", gerr)
+		}
+		entry = next
+	}
+	s.clearRecordTags(ctx, accountID, ltID)
+	// A version create in flight fails its header commit and removes its own
+	// body; this sweep catches one that committed nothing before stopping.
+	return s.deleteVersionBodies(ctx, accountID, ltID)
+}
+
+// deleteVersionBodies deletes every version body of ltID, stopping at the
+// first failure so the caller can keep the header for a retry.
+func (s *LaunchTemplateServiceImpl) deleteVersionBodies(ctx context.Context, accountID, ltID string) error {
+	nums, err := s.listVersionNumbers(ctx, accountID, ltID)
+	if err != nil {
+		return fmt.Errorf("list versions: %w", err)
+	}
+	for _, n := range nums {
+		if err := s.kv.Delete(ctx, versionKey(accountID, ltID, n)); err != nil {
+			return fmt.Errorf("delete version %d: %w", n, err)
+		}
+	}
+	return nil
+}
+
+// CentralTagStore clears a deleted template's entry from the central EC2 tag
+// index. Implemented by domains/ec2/tags.TagsServiceImpl.
+type CentralTagStore interface {
+	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
+}
+
+// SetCentralTagStore injects the central tag index that delete clears.
+func (s *LaunchTemplateServiceImpl) SetCentralTagStore(st CentralTagStore) {
+	s.centralTags = st
+}
+
+// clearRecordTags drops a deleted template's tag-index entry. It runs only
+// after the header delete committed; a failure is logged, as a retried delete
+// would find nothing, and the entry is left as tag residue.
+func (s *LaunchTemplateServiceImpl) clearRecordTags(ctx context.Context, accountID, ltID string) {
+	if s.centralTags == nil {
+		return
+	}
+	if err := s.centralTags.DeleteAllTags(ctx, accountID, ltID); err != nil {
+		slog.ErrorContext(ctx, "central tag store clear failed", "launchTemplateId", ltID, "err", err)
+	}
+}
+
 // --- DeleteLaunchTemplateVersions ---
 
 func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Context, input *ec2.DeleteLaunchTemplateVersionsInput, accountID string) (*ec2.DeleteLaunchTemplateVersionsOutput, error) {

spinifex/domains/ec2/launchtemplate/template_lifecycle_test.go

@@ -0,0 +1,235 @@
+package launchtemplate
+
+//test:in-package — the interleavings are forced by wrapping the service's
+// unexported KV handle.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// raceDelete runs a whole DeleteLaunchTemplate on the unwrapped bucket, so the
+// hook that calls it is not re-entered.
+func raceDelete(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
+	t.Helper()
+	other := &LaunchTemplateServiceImpl{kv: svc.kv.(*faultyKV).KeyValue}
+	_, err := other.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+	require.NoError(t, err)
+}
+
+func newVersionInput(id string) *ec2.CreateLaunchTemplateVersionInput {
+	return &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId:   aws.String(id),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{InstanceType: aws.String("t3.small")},
+	}
+}
+
+// A version create paused before it writes its body or before it commits the
+// header, while a whole delete runs, fails NotFound and leaves nothing behind.
+func TestAccountTermination_VersionCreateRacingADeleteLeavesNoBody(t *testing.T) {
+	for _, pause := range []string{"before-body", "before-commit"} {
+		t.Run(pause, func(t *testing.T) {
+			svc := setupTestService(t)
+			id := createWithUserData(t, svc, "web")
+			header := kvutil.AccountKey(testAccountID, id)
+			raced := false
+			race := func(key string, at string) {
+				if raced || pause != at {
+					return
+				}
+				if (at == "before-body" && key == versionKey(testAccountID, id, 3)) || (at == "before-commit" && key == header) {
+					raced = true
+					raceDelete(t, svc, id)
+				}
+			}
+			svc.kv = &faultyKV{
+				KeyValue:     svc.kv,
+				beforeCreate: func(key string) { race(key, "before-body") },
+				beforeUpdate: func(key string) { race(key, "before-commit") },
+			}
+
+			_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
+			require.True(t, raced)
+			require.Error(t, err)
+			code, _ := awserrors.ResolveErrorCode(err)
+			assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, code)
+			assert.Empty(t, accountKeys(t, svc.kv, testAccountID), "no headerless body is left")
+		})
+	}
+}
+
+// A version that commits before the delete removes the header moves the
+// header revision, so the delete restarts and removes the new body too.
+func TestAccountTermination_DeleteRacingACommittedVersionRemovesIt(t *testing.T) {
+	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
+	inner := svc.kv
+	header := kvutil.AccountKey(testAccountID, id)
+	raced := false
+	svc.kv = &faultyKV{KeyValue: inner, beforeDelete: func(key string) {
+		if key != header || raced {
+			return
+		}
+		raced = true
+		other := &LaunchTemplateServiceImpl{kv: inner}
+		_, err := other.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
+		require.NoError(t, err)
+	}}
+
+	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, raced)
+	assert.Empty(t, accountKeys(t, inner, testAccountID))
+}
+
+// A header commit that fails removes the body it guarded and returns an
+// error; the template keeps exactly the versions it had.
+func TestAccountTermination_FailedVersionCommitRemovesTheBody(t *testing.T) {
+	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
+	before := accountKeys(t, svc.kv, testAccountID)
+	header := kvutil.AccountKey(testAccountID, id)
+	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
+
+	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Equal(t, before, accountKeys(t, svc.kv, testAccountID))
+}
+
+// A version create racing a default-version change commits at the new header
+// revision and keeps the change.
+func TestAccountTermination_VersionCommitKeepsAConcurrentHeaderChange(t *testing.T) {
+	svc := setupTestService(t)
+	id := createWithUserData(t, svc, "web")
+	inner := svc.kv
+	header := kvutil.AccountKey(testAccountID, id)
+	raced := false
+	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
+		if key != header || raced {
+			return
+		}
+		raced = true
+		other := &LaunchTemplateServiceImpl{kv: inner}
+		_, err := other.ModifyLaunchTemplate(context.Background(), &ec2.ModifyLaunchTemplateInput{
+			LaunchTemplateId: aws.String(id), DefaultVersion: aws.String("2"),
+		}, testAccountID)
+		require.NoError(t, err)
+	}}
+
+	out, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
+	require.NoError(t, err)
+	assert.True(t, raced)
+	assert.Equal(t, int64(3), aws.Int64Value(out.LaunchTemplateVersion.VersionNumber))
+	h, _, err := svc.getHeaderByID(context.Background(), testAccountID, id)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), h.DefaultVersionNumber)
+}
+
+// Template data carrying a key name and a placement group reaches the
+// RunInstances input for $Default, $Latest and an explicit version.
+func TestCurrentBehaviour_AccountTermination_TemplateKeyAndGroupReachRunInstances(t *testing.T) {
+	svc := setupTestService(t)
+	out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("web"),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			ImageId: aws.String("ami-123"), KeyName: aws.String("v1-key"),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String("v1-group")},
+		},
+	}, testAccountID)
+	require.NoError(t, err)
+	id := aws.StringValue(out.LaunchTemplate.LaunchTemplateId)
+	_, err = svc.CreateLaunchTemplateVersion(context.Background(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(id),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			ImageId: aws.String("ami-123"), KeyName: aws.String("v2-key"),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String("v2-group")},
+		},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	for version, want := range map[string]string{"$Default": "v1", "$Latest": "v2", "2": "v2", "1": "v1"} {
+		in := &ec2.RunInstancesInput{LaunchTemplate: &ec2.LaunchTemplateSpecification{LaunchTemplateId: aws.String(id), Version: aws.String(version)}}
+		require.NoError(t, ExpandRunInstances(context.Background(), svc, in, testAccountID), version)
+		assert.Equal(t, want+"-key", aws.StringValue(in.KeyName), version)
+		require.NotNil(t, in.Placement, version)
+		assert.Equal(t, want+"-group", aws.StringValue(in.Placement.GroupName), version)
+	}
+}
+
+// The reference reader returns what every version of every live template
+// names, and nothing from headerless bodies or other accounts.
+func TestAccountTemplateReferences_ReadsEveryVersionOfLiveTemplates(t *testing.T) {
+	svc := setupTestService(t)
+	out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("web"),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{KeyName: aws.String("default-key")},
+	}, testAccountID)
+	require.NoError(t, err)
+	id := aws.StringValue(out.LaunchTemplate.LaunchTemplateId)
+	_, err = svc.CreateLaunchTemplateVersion(context.Background(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(id),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			KeyName:   aws.String("later-key"),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String("later-group"), GroupId: aws.String("pg-0123456789abcdef0")},
+		},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
+		LaunchTemplateName: aws.String("theirs"), LaunchTemplateData: &ec2.RequestLaunchTemplateData{KeyName: aws.String("their-key")},
+	}, "999988887777")
+	require.NoError(t, err)
+	_, err = svc.kv.Put(t.Context(), versionKey(testAccountID, "lt-0orphan000000000", 1), []byte(`{"version_number":1,"data":{"KeyName":"orphan-key"}}`))
+	require.NoError(t, err)
+
+	refs, err := svc.AccountTemplateReferences(t.Context(), testAccountID)
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"default-key", "later-key"}, refs.KeyNames)
+	assert.Equal(t, []string{"later-group"}, refs.PlacementGroupNames)
+	assert.Equal(t, []string{"pg-0123456789abcdef0"}, refs.PlacementGroupIDs)
+}
+
+// A header or version the reader cannot read or decode, or a listing
+// failure, is an error: the unread record may name the resource.
+func TestAccountTemplateReferences_FailsClosed(t *testing.T) {
+	tests := map[string]func(t *testing.T, svc *LaunchTemplateServiceImpl, id string){
+		"listing fails": func(t *testing.T, svc *LaunchTemplateServiceImpl, _ string) {
+			svc.kv = &faultyKV{KeyValue: svc.kv, failKeys: func() bool { return true }}
+		},
+		"version unreadable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
+			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == versionKey(testAccountID, id, 2) }}
+		},
+		"version undecodable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
+			_, err := svc.kv.Put(t.Context(), versionKey(testAccountID, id, 2), []byte("{not json"))
+			require.NoError(t, err)
+		},
+		"version without data": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
+			_, err := svc.kv.Put(t.Context(), versionKey(testAccountID, id, 2), []byte(`{"version_number":2}`))
+			require.NoError(t, err)
+		},
+		"header unreadable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
+			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvutil.AccountKey(testAccountID, id) }}
+		},
+		"header undecodable": func(t *testing.T, svc *LaunchTemplateServiceImpl, _ string) {
+			_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
+			require.NoError(t, err)
+		},
+	}
+	for name, seed := range tests {
+		t.Run(name, func(t *testing.T) {
+			svc := setupTestService(t)
+			id := createWithUserData(t, svc, "web")
+			seed(t, svc, id)
+			refs, err := svc.AccountTemplateReferences(t.Context(), testAccountID)
+			require.Error(t, err)
+			assert.Nil(t, refs)
+		})
+	}
+}

spinifex/domains/ec2/launchtemplate/template_references.go

@@ -0,0 +1,95 @@
+package launchtemplate
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// TemplateReferences are the key-pair names, placement-group names and
+// placement-group IDs that the account's live launch-template versions carry.
+// They stay inside EC2; callers must not report them.
+type TemplateReferences struct {
+	KeyNames            []string
+	PlacementGroupNames []string
+	PlacementGroupIDs   []string
+}
+
+// AccountTemplateReferences reads every version body of every live template
+// in the account, since RunInstances may resolve any of them by $Default,
+// $Latest or number. It is complete or an error: an unlistable bucket, or a
+// header or body it cannot read or decode, is an error, because the unread
+// record may name the resource. Headerless bodies resolve from no launch.
+func (s *LaunchTemplateServiceImpl) AccountTemplateReferences(ctx context.Context, accountID string) (*TemplateReferences, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list launch-template keys: %w", err)
+	}
+	prefix := accountID + "."
+	var live []string
+	for _, key := range keys {
+		if !isHeaderKey(key, prefix) {
+			continue
+		}
+		entry, err := s.kv.Get(ctx, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, errors.New("launch-template header unreadable")
+		}
+		var header LaunchTemplateHeader
+		if json.Unmarshal(entry.Value(), &header) != nil || prefix+header.LaunchTemplateId != key {
+			return nil, errors.New("launch-template header undecodable")
+		}
+		live = append(live, header.LaunchTemplateId)
+	}
+
+	out := &TemplateReferences{}
+	for _, ltID := range live {
+		bodyPrefix := versionPrefix(accountID, ltID)
+		for _, key := range keys {
+			if !strings.HasPrefix(key, bodyPrefix) {
+				continue
+			}
+			if err := s.addVersionReferences(ctx, key, out); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return out, nil
+}
+
+// addVersionReferences adds what one version body would launch with. The
+// body is never logged: it holds template data and user data.
+func (s *LaunchTemplateServiceImpl) addVersionReferences(ctx context.Context, key string, out *TemplateReferences) error {
+	entry, err := s.kv.Get(ctx, key)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return nil
+	}
+	if err != nil {
+		return errors.New("launch-template version unreadable")
+	}
+	var rec LaunchTemplateVersionRec
+	if json.Unmarshal(entry.Value(), &rec) != nil || rec.Data == nil {
+		return errors.New("launch-template version undecodable")
+	}
+	if name := aws.StringValue(rec.Data.KeyName); name != "" {
+		out.KeyNames = append(out.KeyNames, name)
+	}
+	if p := rec.Data.Placement; p != nil {
+		if name := aws.StringValue(p.GroupName); name != "" {
+			out.PlacementGroupNames = append(out.PlacementGroupNames, name)
+		}
+		if id := aws.StringValue(p.GroupId); id != "" {
+			out.PlacementGroupIDs = append(out.PlacementGroupIDs, id)
+		}
+	}
+	return nil
+}

spinifex/domains/ec2/placementgroup/account_cleanup.go

@@ -0,0 +1,85 @@
+package placementgroup
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// BlockedGroup is a group the owner refused to delete because its member map
+// still names instances. Only the count is kept; member IDs never leave.
+type BlockedGroup struct {
+	GroupID     string
+	MemberCount int
+}
+
+// AccountCleanup is what one CleanupAccount run did, by group ID. It is the
+// owner's account of its attempts, not proof of absence, which only a later
+// ScanAccount gives.
+type AccountCleanup struct {
+	// DeletedGroupIDs were removed by the owner delete or were already gone.
+	DeletedGroupIDs []string
+	// FailedGroupIDs hit a store failure; the record remains.
+	FailedGroupIDs []string
+	// Blocked groups still hold instances and are never forced.
+	Blocked []BlockedGroup
+	// Referenced maps a kept group ID to the reason the caller's predicate gave.
+	Referenced map[string]string
+}
+
+// CleanupAccount deletes every readable, empty placement group of the account
+// through the same revision-guarded delete DeletePlacementGroup runs. A group
+// holding instances is reported blocked, and one for whose ID or name
+// referenced returns a reason is kept; unreadable records are left for the owner scan to report. A
+// listing failure is returned and nothing is deleted.
+func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*AccountCleanup, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list placement-group keys: %w", err)
+	}
+
+	prefix := accountID + "."
+	out := &AccountCleanup{Referenced: map[string]string{}}
+	for _, key := range keys {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		entry, err := s.kv.Get(ctx, key)
+		if err != nil {
+			continue
+		}
+		var record PlacementGroupRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil || record.GroupId == "" {
+			continue
+		}
+		if referenced != nil {
+			reason := referenced(record.GroupId, record.GroupName)
+			if reason == "" {
+				reason = referenced(record.GroupId, strings.TrimPrefix(key, prefix))
+			}
+			if reason != "" {
+				out.Referenced[record.GroupId] = reason
+				continue
+			}
+		}
+		members, err := s.deleteGroup(ctx, accountID, key)
+		switch {
+		case err == nil, errors.Is(err, errPGAbsent):
+			out.DeletedGroupIDs = append(out.DeletedGroupIDs, record.GroupId)
+		case errors.Is(err, errPGInUse):
+			out.Blocked = append(out.Blocked, BlockedGroup{GroupID: record.GroupId, MemberCount: members})
+		default:
+			slog.WarnContext(ctx, "Placement-group account cleanup: delete failed", "groupId", record.GroupId, "err", err)
+			out.FailedGroupIDs = append(out.FailedGroupIDs, record.GroupId)
+		}
+	}
+	slog.InfoContext(ctx, "Placement-group account cleanup completed", "accountID", accountID,
+		"deleted", len(out.DeletedGroupIDs), "failed", len(out.FailedGroupIDs), "blocked", len(out.Blocked), "referenced", len(out.Referenced))
+	return out, nil
+}

spinifex/domains/ec2/placementgroup/account_termination_current_behaviour_test.go

@@ -21,16 +21,18 @@ import (
 )
 
 // The TestCurrentBehaviour_AccountTermination_ tests pin how placement groups
-// are stored and what Describe and Delete do with them today. They describe
-// current behaviour, not the target design; a failure means it changed.
+// are stored and what Describe and Delete do with them today; a failure means
+// it changed. The TestAccountTermination_ tests pin the revision-guarded delete.
 
-// hookedKV fails reads of failGet and runs afterGet once, after the first read
-// of key returns, standing in for a writer on another node.
+// hookedKV fails reads of failGet and runs afterGet once, after read number
+// skip+1 of key returns, standing in for a writer on another node.
 type hookedKV struct {
 	jetstream.KeyValue
 
 	failGet  string
 	key      string
+	skip     int
+	reads    int
 	once     sync.Once
 	afterGet func()
 }
@@ -41,7 +43,9 @@ func (k *hookedKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry
 	}
 	entry, err := k.KeyValue.Get(ctx, key)
 	if key == k.key && k.afterGet != nil {
-		k.once.Do(k.afterGet)
+		if k.reads++; k.reads > k.skip {
+			k.once.Do(k.afterGet)
+		}
 	}
 	return entry, err
 }
@@ -92,8 +96,9 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 	assert.Empty(t, record.NodeInstances)
 }
 
-// Describe skips a record it cannot read or decode, and still succeeds. An
-// undecodable record also cannot be removed through DeletePlacementGroup.
+// Describe skips a record it cannot read or decode, and still succeeds. Such
+// a record cannot be removed through DeletePlacementGroup either, and a read
+// failure is an internal error, never an unknown group.
 func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreadableRecords(t *testing.T) {
 	brokenKey := kvutil.AccountKey(testAccountID, "broken")
 	tests := []struct {
@@ -108,7 +113,7 @@ func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreada
 		{"record the store cannot read", func(t *testing.T, svc *PlacementGroupServiceImpl) {
 			createTestGroup(t, svc, "broken", ec2.PlacementStrategySpread)
 			svc.kv = &hookedKV{KeyValue: svc.kv, failGet: brokenKey}
-		}, awserrors.ErrorInvalidPlacementGroupUnknown},
+		}, awserrors.ErrorServerInternal},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -128,10 +133,10 @@ func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreada
 	}
 }
 
-// Pins current behaviour that differs from AWS: the delete reads the record,
-// checks membership, then deletes without a revision guard. A launch that
-// records an instance between the two is deleted with the group.
-func TestCurrentBehaviour_AccountTermination_DeletePlacementGroupDropsAConcurrentMember(t *testing.T) {
+// The delete commits only at the revision whose membership it checked. A
+// launch that records an instance between the check and the delete wins: the
+// delete re-reads, refuses with InUse, and the member stays.
+func TestAccountTermination_DeletePlacementGroupLosesToAConcurrentMember(t *testing.T) {
 	svc := setupTestService(t)
 	createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
 	key := kvutil.AccountKey(testAccountID, "cluster-a")
@@ -151,17 +156,162 @@ func TestCurrentBehaviour_AccountTermination_DeletePlacementGroupDropsAConcurren
 	}}
 
 	_, err := svc.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{GroupName: aws.String("cluster-a")}, testAccountID)
-	require.NoError(t, err, "the delete succeeds although the group gained a member")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidPlacementGroupInUse, err.Error())
 	require.NoError(t, reserveErr)
 	require.NoError(t, finalizeErr, "the interleaved launch committed its member")
 
-	_, err = launcher.kv.Get(t.Context(), key)
-	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
-	assert.Empty(t, describedGroupNames(t, launcher, testAccountID))
+	entry, err := launcher.kv.Get(t.Context(), key)
+	require.NoError(t, err)
+	var record PlacementGroupRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &record))
+	assert.Equal(t, map[string][]string{"node-1": {"i-0123456789abcdef0"}}, record.NodeInstances)
 
-	// The member's later shutdown finds no group and reports success.
+	// Once the member leaves, the same delete succeeds.
 	_, err = launcher.RemoveInstance(context.Background(), &RemoveInstanceInput{
 		GroupName: "cluster-a", NodeName: "node-1", InstanceID: "i-0123456789abcdef0",
 	}, testAccountID)
-	assert.NoError(t, err)
+	require.NoError(t, err)
+	_, err = launcher.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{GroupName: aws.String("cluster-a")}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, describedGroupNames(t, launcher, testAccountID))
+}
+
+// A reservation placeholder is a launch in flight, not a member, so the
+// delete may win; the launch's finalize then finds no group and fails, which
+// makes the launcher roll its instances back rather than lose them.
+func TestAccountTermination_DeletePlacementGroupBeforeFinalizeFailsTheLaunch(t *testing.T) {
+	svc := setupTestService(t)
+	createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
+	reserved, err := svc.ReserveClusterNode(context.Background(), &ReserveClusterNodeInput{
+		GroupName: "cluster-a", EligibleNodes: []string{"node-1"},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{GroupName: aws.String("cluster-a")}, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.FinalizeClusterInstances(context.Background(), &FinalizeClusterInstancesInput{
+		GroupName: "cluster-a", NodeInstances: map[string][]string{reserved.TargetNode: {"i-0123456789abcdef0"}},
+	}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidPlacementGroupUnknown, err.Error())
+	assert.Empty(t, describedGroupNames(t, svc, testAccountID))
+}
+
+// addMember reserves a node in a cluster group and records one instance on it.
+func addMember(t *testing.T, svc *PlacementGroupServiceImpl, group, instanceID string) error {
+	t.Helper()
+	reserved, err := svc.ReserveClusterNode(context.Background(), &ReserveClusterNodeInput{
+		GroupName: group, EligibleNodes: []string{"node-1"},
+	}, testAccountID)
+	if err != nil {
+		return err
+	}
+	_, err = svc.FinalizeClusterInstances(context.Background(), &FinalizeClusterInstancesInput{
+		GroupName: group, NodeInstances: map[string][]string{reserved.TargetNode: {instanceID}},
+	}, testAccountID)
+	return err
+}
+
+// Account cleanup deletes empty groups through the revision-guarded delete,
+// reports a group holding instances as blocked with its member count only,
+// and leaves records it cannot read for the owner scan to report.
+func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
+	svc := setupTestService(t)
+	empty := createTestGroup(t, svc, "empty", ec2.PlacementStrategySpread)
+	held := createTestGroup(t, svc, "held", ec2.PlacementStrategyCluster)
+	require.NoError(t, addMember(t, svc, "held", "i-0123456789abcdef0"))
+	_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "broken"), []byte("{not json"))
+	require.NoError(t, err)
+	other := "999988887777"
+	_, err = svc.CreatePlacementGroup(context.Background(), &ec2.CreatePlacementGroupInput{
+		GroupName: aws.String("theirs"), Strategy: aws.String(ec2.PlacementStrategySpread),
+	}, other)
+	require.NoError(t, err)
+
+	out, err := svc.CleanupAccount(t.Context(), testAccountID, nil)
+	require.NoError(t, err)
+	assert.Equal(t, []string{aws.StringValue(empty.GroupId)}, out.DeletedGroupIDs)
+	assert.Equal(t, []BlockedGroup{{GroupID: aws.StringValue(held.GroupId), MemberCount: 1}}, out.Blocked)
+	assert.Empty(t, out.FailedGroupIDs)
+
+	assert.Equal(t, []string{"held"}, describedGroupNames(t, svc, testAccountID))
+	assert.Equal(t, []string{"theirs"}, describedGroupNames(t, svc, other))
+	_, err = svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, "broken"))
+	require.NoError(t, err, "an unreadable record is never deleted")
+}
+
+// A launch that records a member between the cleanup's membership check and
+// its delete wins: the cleanup reports the group blocked and the member stays.
+func TestAccountCleanup_ConcurrentMemberBlocksTheDelete(t *testing.T) {
+	svc := setupTestService(t)
+	group := createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
+	key := kvutil.AccountKey(testAccountID, "cluster-a")
+	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
+	var memberErr error
+	// Read 1 is the cleanup's listing read; read 2 is the delete's check.
+	svc.kv = &hookedKV{KeyValue: svc.kv, key: key, skip: 1, afterGet: func() {
+		memberErr = addMember(t, launcher, "cluster-a", "i-0123456789abcdef0")
+	}}
+
+	out, err := svc.CleanupAccount(t.Context(), testAccountID, nil)
+	require.NoError(t, err)
+	require.NoError(t, memberErr, "the interleaved launch committed its member")
+	assert.Equal(t, []BlockedGroup{{GroupID: aws.StringValue(group.GroupId), MemberCount: 1}}, out.Blocked)
+	assert.Empty(t, out.DeletedGroupIDs)
+	assert.Equal(t, map[string][]string{"node-1": {"i-0123456789abcdef0"}}, readRecord(t, launcher, "cluster-a").NodeInstances)
+}
+
+// A group the caller's reference check names is kept with the given reason,
+// whether it is matched by its name or its group ID.
+func TestAccountCleanup_KeepsReferencedGroups(t *testing.T) {
+	svc := setupTestService(t)
+	named := createTestGroup(t, svc, "named", ec2.PlacementStrategySpread)
+	byID := createTestGroup(t, svc, "by-id", ec2.PlacementStrategySpread)
+	free := createTestGroup(t, svc, "free", ec2.PlacementStrategySpread)
+	referenced := func(id, name string) string {
+		switch {
+		case name == "named":
+			return "spot-request-reference"
+		case id == aws.StringValue(byID.GroupId):
+			return "launch-template-reference"
+		}
+		return ""
+	}
+
+	out, err := svc.CleanupAccount(t.Context(), testAccountID, referenced)
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		aws.StringValue(named.GroupId): "spot-request-reference",
+		aws.StringValue(byID.GroupId):  "launch-template-reference",
+	}, out.Referenced)
+	assert.Equal(t, []string{aws.StringValue(free.GroupId)}, out.DeletedGroupIDs)
+	assert.ElementsMatch(t, []string{"named", "by-id"}, describedGroupNames(t, svc, testAccountID))
+}
+
+// A listing failure is returned and nothing is deleted.
+func TestAccountCleanup_ListingFailureDeletesNothing(t *testing.T) {
+	svc := setupTestService(t)
+	createTestGroup(t, svc, "empty", ec2.PlacementStrategySpread)
+	kv := svc.kv
+	svc.kv = &listFailingKV{KeyValue: kv}
+
+	_, err := svc.CleanupAccount(t.Context(), testAccountID, nil)
+	require.Error(t, err)
+	svc.kv = kv
+	assert.Equal(t, []string{"empty"}, describedGroupNames(t, svc, testAccountID))
+}
+
+// listFailingKV fails every key listing.
+type listFailingKV struct {
+	jetstream.KeyValue
+}
+
+func (k *listFailingKV) ListKeys(context.Context, ...jetstream.WatchOpt) (jetstream.KeyLister, error) {
+	return nil, errors.New("injected listing failure")
+}
+
+func (k *listFailingKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
+	return nil, errors.New("injected listing failure")
 }

spinifex/domains/ec2/placementgroup/service_impl.go

@@ -52,6 +52,32 @@ type PlacementGroupServiceImpl struct {
 	natsConn *nats.Conn
 	kv       jetstream.KeyValue
 	region   string
+
+	// Optional: nil leaves the central tag index untouched by delete.
+	centralTags CentralTagStore
+}
+
+// CentralTagStore clears a deleted group's entry from the central EC2 tag
+// index. Implemented by domains/ec2/tags.TagsServiceImpl.
+type CentralTagStore interface {
+	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
+}
+
+// SetCentralTagStore injects the central tag index that delete clears.
+func (s *PlacementGroupServiceImpl) SetCentralTagStore(st CentralTagStore) {
+	s.centralTags = st
+}
+
+// clearRecordTags drops a deleted group's tag-index entry. It runs only after
+// the record delete committed; a failure is logged, as the group is gone and a
+// retried delete would find nothing, and the entry is left as tag residue.
+func (s *PlacementGroupServiceImpl) clearRecordTags(ctx context.Context, accountID, groupID string) {
+	if s.centralTags == nil || groupID == "" {
+		return
+	}
+	if err := s.centralTags.DeleteAllTags(ctx, accountID, groupID); err != nil {
+		slog.ErrorContext(ctx, "central tag store clear failed", "groupId", groupID, "err", err)
+	}
 }
 
 // NewPlacementGroupServiceImplWithNATS creates a placement group service with NATS JetStream.
@@ -142,37 +168,19 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in
 	}, nil
 }
 
-// DeletePlacementGroup deletes a placement group.
+// DeletePlacementGroup deletes a placement group that holds no instances.
 func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, input *ec2.DeletePlacementGroupInput, accountID string) (*ec2.DeletePlacementGroupOutput, error) {
 	if input.GroupName == nil || *input.GroupName == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
 	groupName := *input.GroupName
-	key := kvutil.AccountKey(accountID, groupName)
-
-	entry, err := s.kv.Get(ctx, key)
-	if err != nil {
-		// Lower case, unlike DescribePlacementGroups' message for the same code.
-		return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
-	}
-
-	var record PlacementGroupRecord
-	if err := json.Unmarshal(entry.Value(), &record); err != nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// Check for running instances
-	instanceCount := 0
-	for _, ids := range record.NodeInstances {
-		instanceCount += len(ids)
-	}
-	if instanceCount > 0 {
-		return nil, errors.New(awserrors.ErrorInvalidPlacementGroupInUse)
-	}
-
-	if err := s.kv.Delete(ctx, key); err != nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
+	if _, err := s.deleteGroup(ctx, accountID, kvutil.AccountKey(accountID, groupName)); err != nil {
+		if errors.Is(err, errPGAbsent) {
+			// Lower case, unlike DescribePlacementGroups' message for the same code.
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
+		}
+		return nil, mapCASError(ctx, "DeletePlacementGroup", groupName, accountID, err)
 	}
 
 	slog.InfoContext(ctx, "DeletePlacementGroup completed", "groupName", groupName, "accountID", accountID)
@@ -180,6 +188,26 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 	return &ec2.DeletePlacementGroupOutput{}, nil
 }
 
+// deleteGroup removes the record at key only while it names no instance, at
+// the revision that check read, so a reservation committed in between sends
+// the delete back to re-check. Its tag-index entry is cleared afterwards.
+// The returned count is the members seen by the last check.
+func (s *PlacementGroupServiceImpl) deleteGroup(ctx context.Context, accountID, key string) (int, error) {
+	members := 0
+	record, err := kvutil.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
+		members = memberCount(record.NodeInstances)
+		if members > 0 {
+			return errPGInUse
+		}
+		return nil
+	})
+	if err != nil {
+		return members, err
+	}
+	s.clearRecordTags(ctx, accountID, record.GroupId)
+	return 0, nil
+}
+
 var describePlacementGroupsValidFilters = map[string]bool{
 	"group-id":     true,
 	"strategy":     true,
@@ -324,11 +352,12 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
 const maxCASRetries = 5
 
 // errPGAbsent and errPGContended mark the two kvutil outcomes that map to
-// specific AWS errors rather than a generic internal failure. Neither
-// escapes this file.
+// specific AWS errors rather than a generic internal failure, and neither
+// escapes this file. errPGInUse is the AWS error itself.
 var (
 	errPGAbsent    = errors.New("placementgroup: record absent")
 	errPGContended = errors.New("placementgroup: CAS retries exhausted")
+	errPGInUse     = errors.New(awserrors.ErrorInvalidPlacementGroupInUse)
 )
 
 // mapCASError turns a kvutil failure into the AWS error the caller returns.

spinifex/domains/ec2/snapshot/service_impl.go

@@ -662,10 +662,10 @@ func snapshotMatchesFilters(cfg ebsmetadata.Snapshot, filters map[string][]strin
 	return awsfilters.MatchesTags(filters, cfg.Tags)
 }
 
-// snapshotInUseByVolumes checks if any volume was created from the given
-// snapshot. A clone records its source in its ebsmetadata document; missing one
-// here would let a delete strip the chunks a live clone still reads.
-func (s *SnapshotServiceImpl) snapshotInUseByVolumes(ctx context.Context, snapshotID string) (bool, error) {
+// snapshotInUseByVolumes reports whether any volume was created from the given
+// snapshot, and the ID of one owned by accountID if any. A clone records its
+// source in its ebsmetadata document; missing one would strip chunks it reads.
+func (s *SnapshotServiceImpl) snapshotInUseByVolumes(ctx context.Context, snapshotID, accountID string) (bool, string, error) {
 	// Whole-cluster, and not scoping this to the snapshot's own account is the
 	// point: launching from a system AMI writes a root volume owned by the
 	// launching tenant whose SnapshotID is the system snapshot, so a clone
@@ -675,10 +675,34 @@ func (s *SnapshotServiceImpl) snapshotInUseByVolumes(ctx context.Context, snapsh
 	// the snapshot's chunks, and this answer gates deleting them.
 	volumes, err := s.metadata.ListAllVolumesStrict(ctx)
 	if err != nil {
-		return false, fmt.Errorf("snapshotInUseByVolumes: failed to list volumes: %w", err)
+		return false, "", fmt.Errorf("snapshotInUseByVolumes: failed to list volumes: %w", err)
 	}
+	inUse := false
 	for _, volume := range volumes {
-		if volume.SnapshotID == snapshotID {
+		if volume.SnapshotID != snapshotID {
+			continue
+		}
+		if volume.TenantID == accountID {
+			return true, volume.VolumeID, nil
+		}
+		inUse = true
+	}
+	return inUse, "", nil
+}
+
+// amiInUseBySnapshot checks if any registered AMI names the given snapshot as
+// its root device. AWS refuses DeleteSnapshot while a registered AMI
+// references the snapshot, the same way it refuses one backing a volume.
+func (s *SnapshotServiceImpl) amiInUseBySnapshot(ctx context.Context, snapshotID string) (bool, error) {
+	// Whole-cluster and strict for the same reason as the volume check above:
+	// a system AMI can be registered by an account other than the snapshot's
+	// owner, and an AMI this cannot read is not evidence that none exists.
+	amis, err := s.metadata.ListAMIsStrict(ctx)
+	if err != nil {
+		return false, fmt.Errorf("amiInUseBySnapshot: failed to list AMIs: %w", err)
+	}
+	for _, ami := range amis {
+		if ami.SnapshotID == snapshotID {
 			return true, nil
 		}
 	}
@@ -715,13 +739,29 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	}
 
 	// Check if any volumes were created from this snapshot
-	inUse, err := s.snapshotInUseByVolumes(ctx, snapshotID)
+	inUse, volumeID, err := s.snapshotInUseByVolumes(ctx, snapshotID, accountID)
 	if err != nil {
 		slog.ErrorContext(ctx, "DeleteSnapshot failed to check snapshot usage", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	if inUse {
-		slog.InfoContext(ctx, "DeleteSnapshot blocked: snapshot in use by volume", "snapshotId", snapshotID)
+		slog.InfoContext(ctx, "DeleteSnapshot blocked: snapshot in use by volume", "snapshotId", snapshotID, "volumeId", volumeID)
+		// The registered message blames AMIs. Name the volume only when the
+		// caller owns it: the scan is whole-cluster.
+		if volumeID == "" {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidSnapshotInUse, "The snapshot '%s' is in use by a volume.", snapshotID)
+		}
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidSnapshotInUse, "The snapshot '%s' is in use by volume '%s'.", snapshotID, volumeID)
+	}
+
+	// Check if any registered AMI names this snapshot as its root device
+	amiInUse, err := s.amiInUseBySnapshot(ctx, snapshotID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DeleteSnapshot failed to check AMI usage", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if amiInUse {
+		slog.InfoContext(ctx, "DeleteSnapshot blocked: snapshot in use by registered AMI", "snapshotId", snapshotID)
 		return nil, errors.New(awserrors.ErrorInvalidSnapshotInUse)
 	}
 	if s.provider == nil {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -205,9 +205,10 @@ func TestSnapshotInUseByVolumes_EncryptedVolume(t *testing.T) {
 		VolumeID: "vol-fromsnap", CapacityGiB: 50, SnapshotID: "snap-src", Encrypted: true,
 	})
 
-	inUse, err := svc.snapshotInUseByVolumes(context.Background(), "snap-src")
+	inUse, volumeID, err := svc.snapshotInUseByVolumes(context.Background(), "snap-src", testAccountID)
 	require.NoError(t, err)
-	assert.True(t, inUse, "encrypted volume's SnapshotID must be visible through the envelope")
+	assert.True(t, inUse)
+	assert.Equal(t, "vol-fromsnap", volumeID, "encrypted volume's SnapshotID must be visible through the envelope")
 }
 
 // TestCreateSnapshot_VolumeNotFound tests creating a snapshot from non-existent volume.
@@ -459,6 +460,18 @@ func TestDeleteSnapshot_WrongAccount(t *testing.T) {
 	assert.Len(t, result.Snapshots, 1)
 }
 
+// clientErrorDetail returns the code and message the gateway renders for err:
+// a message attached at the call site, else the code's registered EC2 wording.
+func clientErrorDetail(t *testing.T, err error) (string, string) {
+	t.Helper()
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	if message == "" {
+		message = awserrors.LookupErrorMessage("ec2", code).Message
+	}
+	return code, message
+}
+
 // TestDeleteSnapshot_InUseByVolume tests that deleting a snapshot fails when a volume was created from it.
 func TestDeleteSnapshot_InUseByVolume(t *testing.T) {
 	svc, store := setupTestSnapshotService(t)
@@ -480,7 +493,9 @@ func TestDeleteSnapshot_InUseByVolume(t *testing.T) {
 		SnapshotId: snap.SnapshotId,
 	}, testAccountID)
 	require.Error(t, err)
-	assert.Contains(t, err.Error(), awserrors.ErrorInvalidSnapshotInUse)
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidSnapshotInUse, code)
+	assert.Equal(t, "The snapshot '"+*snap.SnapshotId+"' is in use by volume 'vol-cloned'.", message)
 
 	// Verify snapshot still exists
 	result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{
@@ -490,6 +505,139 @@ func TestDeleteSnapshot_InUseByVolume(t *testing.T) {
 	assert.Len(t, result.Snapshots, 1)
 }
 
+// TestDeleteSnapshot_InUseByAnotherAccountsVolume tests that the refusal
+// never names a volume the caller does not own, since the dependency scan is
+// whole-cluster and an endpoint never confirms another account's IDs.
+func TestDeleteSnapshot_InUseByAnotherAccountsVolume(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+
+	createTestVolume(t, svc, store, "vol-source", 50)
+	snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
+		VolumeId: aws.String("vol-source"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	seedVolumeDocument(t, store, ebsmetadata.Volume{
+		VolumeID: "vol-foreign", TenantID: otherAccountID, CapacityGiB: 50, SnapshotID: *snap.SnapshotId,
+	})
+
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	require.Error(t, err)
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidSnapshotInUse, code)
+	assert.NotContains(t, message, "vol-foreign")
+	assert.Equal(t, "The snapshot '"+*snap.SnapshotId+"' is in use by a volume.", message)
+
+	// A volume the caller owns is named even when a foreign one also matches.
+	seedVolumeDocument(t, store, ebsmetadata.Volume{
+		VolumeID: "vol-own", CapacityGiB: 50, SnapshotID: *snap.SnapshotId,
+	})
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	_, message = clientErrorDetail(t, err)
+	assert.Equal(t, "The snapshot '"+*snap.SnapshotId+"' is in use by volume 'vol-own'.", message)
+}
+
+// TestDeleteSnapshot_InUseByRegisteredAMI tests that deleting a snapshot
+// fails while an AMI's own CreateImage snapshot still names it, matching AWS.
+func TestDeleteSnapshot_InUseByRegisteredAMI(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+
+	createTestVolume(t, svc, store, "vol-source", 50)
+	snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
+		VolumeId: aws.String("vol-source"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	require.NoError(t, svc.metadata.PutAMI(context.Background(), ebsmetadata.AMI{
+		ImageID: "ami-registered", Name: "ct-ami", SnapshotID: *snap.SnapshotId,
+	}))
+
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	require.Error(t, err)
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidSnapshotInUse, code)
+	assert.Equal(t, "The snapshot that you are trying to delete is in use by one or more AMIs.", message)
+
+	// The provider and control-plane document both survive the refusal.
+	result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{
+		SnapshotIds: []*string{snap.SnapshotId},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, result.Snapshots, 1)
+
+	require.NoError(t, svc.metadata.DeleteAMI(context.Background(), "ami-registered"))
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	require.NoError(t, err, "once the AMI is gone, the delete succeeds")
+}
+
+// TestDeleteSnapshot_InUseByRegisteredAMI_AnotherAccount tests that the AMI
+// check is whole-cluster: a system AMI registered by a different account than
+// the snapshot's owner still blocks the delete.
+func TestDeleteSnapshot_InUseByRegisteredAMI_AnotherAccount(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+
+	createTestVolume(t, svc, store, "vol-source", 50)
+	snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
+		VolumeId: aws.String("vol-source"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	require.NoError(t, svc.metadata.PutAMI(context.Background(), ebsmetadata.AMI{
+		ImageID: "ami-other-account", Name: "ct-system-ami",
+		ImageOwnerAlias: otherAccountID, SnapshotID: *snap.SnapshotId,
+	}))
+
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorInvalidSnapshotInUse)
+}
+
+// TestDeleteSnapshot_AMIReadFailureFailsClosed tests that an AMI document
+// this cannot decode aborts the delete rather than being treated as absent:
+// a partial AMI listing is not evidence that no AMI names the snapshot.
+func TestDeleteSnapshot_AMIReadFailureFailsClosed(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+
+	createTestVolume(t, svc, store, "vol-source", 50)
+	snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
+		VolumeId: aws.String("vol-source"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	key, err := ebsmetadata.AMIKey("ami-corrupt")
+	require.NoError(t, err)
+	_, err = store.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String("test-bucket"),
+		Key:    aws.String(key),
+		Body:   strings.NewReader("not-json"),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
+		SnapshotId: snap.SnapshotId,
+	}, testAccountID)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorServerInternal)
+
+	// Nothing was deleted: the snapshot survives in both the provider and the
+	// control-plane document.
+	result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{
+		SnapshotIds: []*string{snap.SnapshotId},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, result.Snapshots, 1)
+}
+
 // TestDeleteSnapshot_NotFound tests deleting a non-existent snapshot.
 func TestDeleteSnapshot_NotFound(t *testing.T) {
 	svc, _ := setupTestSnapshotService(t)
@@ -1074,7 +1222,7 @@ func TestDeleteSnapshot_Provider_BlockedByCloneRecordedInMetadata(t *testing.T)
 	}))
 
 	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, testAccountID)
-	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotInUse)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidSnapshotInUse), "got %v", err)
 
 	// Once the clone is gone the snapshot is deletable again.
 	require.NoError(t, svc.metadata.DeleteVolume(ctx, testAccountID, "vol-clone"))

spinifex/domains/ec2/snapshot/snapshot_partition_test.go

@@ -270,7 +270,7 @@ func TestDeleteSnapshot_BlockedByACloneInAnotherAccount(t *testing.T) {
 
 	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: snap.SnapshotId}, awsidentifiers.GlobalAccountID)
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidSnapshotInUse, err.Error())
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidSnapshotInUse), "got %v", err)
 }
 
 // The irreversible case. The teardown reaper lists with OwnerIds set to the

spinifex/domains/ec2/spotinstance/launch_references.go

@@ -0,0 +1,59 @@
+package spotinstance
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// LaunchReferences are the key-pair and placement-group names that an
+// account's open or active spot requests would launch with. They stay inside
+// EC2; callers must not report them.
+type LaunchReferences struct {
+	KeyNames            []string
+	PlacementGroupNames []string
+}
+
+// AccountLaunchReferences reads every open or active spot request of the
+// account. It is complete or an error: an unlistable bucket or an unreadable
+// request is an error, because the unread request may name the resource.
+func (s *SpotInstanceServiceImpl) AccountLaunchReferences(ctx context.Context, accountID string) (*LaunchReferences, error) {
+	keys, err := kvutil.Keys(ctx, s.activeKV)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list spot requests: %w", err)
+	}
+	prefix := accountID + "."
+	out := &LaunchReferences{}
+	for _, key := range keys {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		record, err := s.readRecord(ctx, s.activeKV, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil || record.Request == nil {
+			return nil, errors.New("spot request unreadable")
+		}
+		switch aws.StringValue(record.Request.State) {
+		case ec2.SpotInstanceStateClosed, ec2.SpotInstanceStateCancelled, ec2.SpotInstanceStateFailed:
+			continue
+		}
+		spec := launchSpec(record.Request)
+		if name := aws.StringValue(spec.KeyName); name != "" {
+			out.KeyNames = append(out.KeyNames, name)
+		}
+		if spec.Placement != nil {
+			if name := aws.StringValue(spec.Placement.GroupName); name != "" {
+				out.PlacementGroupNames = append(out.PlacementGroupNames, name)
+			}
+		}
+	}
+	return out, nil
+}

spinifex/domains/ec2/spotinstance/launch_references_test.go

@@ -0,0 +1,49 @@
+package spotinstance
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Open and active requests contribute their key and group names; cancelled
+// requests and other accounts' requests do not.
+func TestAccountLaunchReferences_ReadsOpenAndActiveRequests(t *testing.T) {
+	svc := setupTestService(t)
+	active := makeSIR("sir-active01", "i-0000000000000001")
+	active.LaunchSpecification.Placement = &ec2.SpotPlacement{GroupName: aws.String("pg-spread")}
+	open := makeSIR("sir-open0001", "")
+	open.State = aws.String(ec2.SpotInstanceStateOpen)
+	open.LaunchSpecification.KeyName = aws.String("open-key")
+	putSIRs(t, svc, active, open)
+	cancelled := makeSIR("sir-cancel01", "i-0000000000000002")
+	cancelled.LaunchSpecification.KeyName = aws.String("cancelled-key")
+	putSIRs(t, svc, cancelled)
+	_, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{
+		SpotInstanceRequestIds: []*string{aws.String("sir-cancel01")},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.PutSpotInstanceRequests(t.Context(), &PutSpotRequestsInput{Requests: []*ec2.SpotInstanceRequest{makeSIR("sir-other001", "")}}, "999988887777")
+	require.NoError(t, err)
+
+	refs, err := svc.AccountLaunchReferences(t.Context(), testAccountID)
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"my-key", "open-key"}, refs.KeyNames)
+	assert.Equal(t, []string{"pg-spread"}, refs.PlacementGroupNames)
+}
+
+// An unreadable request may name anything, so the check fails rather than
+// answer without it.
+func TestAccountLaunchReferences_UnreadableRequestFailsTheCheck(t *testing.T) {
+	svc := setupTestService(t)
+	putSIRs(t, svc, makeSIR("sir-active01", "i-0000000000000001"))
+	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
+	require.NoError(t, err)
+
+	_, err = svc.AccountLaunchReferences(t.Context(), testAccountID)
+	require.Error(t, err)
+}

spinifex/domains/ec2/volume/account_termination_current_behaviour_test.go

@@ -0,0 +1,79 @@
+package volume_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
+	ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The TestCurrentBehaviour_AccountTermination_ tests pin what terminate does
+// to a DeleteOnTermination volume today; a failure means it changed.
+
+// A snapshotted DeleteOnTermination volume loses its attachment before the
+// delete refuses, so it survives as an unattached volume that the leak reaper,
+// which keys on the attachment, never marks.
+func TestCurrentBehaviour_AccountTermination_SnapshottedDeleteOnTerminationVolumeIsDetachedAndUnmarked(t *testing.T) {
+	const (
+		accountID  = "123456789012"
+		bucket     = "ct-bucket"
+		volumeID   = "vol-dot-snapshotted"
+		instanceID = "i-0123456789abcdef0"
+	)
+	ctx := context.Background()
+	_, nc, js := testutil.StartTestJetStream(t)
+	refs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: ec2snapshot.KVBucketVolumeSnapshots})
+	require.NoError(t, err)
+
+	store := objectstore.NewMemoryObjectStore()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	cfg := &config.Config{WalDir: t.TempDir(), Predastore: config.PredastoreConfig{Bucket: bucket}}
+	volumes := ec2volume.NewVolumeServiceImplWithStore(cfg, store, nc, refs)
+	volumes.SetEBSProvider(provider)
+	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc, refs)
+	snapshots.SetEBSProvider(provider)
+	metadata := ebsmetadata.NewStore(store, bucket)
+
+	created, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
+		CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	require.NoError(t, metadata.PutVolume(ctx, ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: accountID, CapacityGiB: 1,
+		State: "available", ProviderHandle: created.Handle, DeleteOnTermination: true,
+	}))
+	_, err = snapshots.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String(volumeID)}, accountID)
+	require.NoError(t, err)
+	require.NoError(t, volumes.UpdateVolumeState(accountID, volumeID, "in-use", instanceID, "/dev/sda1"))
+
+	err = volumes.DeleteVolumeOnTerminate(ctx, volumeID, accountID)
+	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "the snapshot index refuses the terminate delete")
+
+	survivor, err := metadata.GetVolume(ctx, accountID, volumeID)
+	require.NoError(t, err, "the volume survives its instance")
+	assert.Empty(t, survivor.AttachedInstance, "the attachment was cleared before the refusal")
+	assert.Equal(t, "available", survivor.State)
+
+	reaper := volumes.NewVolumeLeakReaper(func() (map[string]bool, error) {
+		return map[string]bool{instanceID: true}, nil
+	})
+	marked, err := reaper.Sweep(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, marked, "with no attachment left, the leak reaper cannot attribute the volume to its instance")
+
+	_, err = volumes.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: aws.String(volumeID)}, accountID)
+	assert.EqualError(t, err, awserrors.ErrorVolumeInUse, "an ordinary delete is refused for the same reason")
+}

spinifex/domains/ec2/volume/delete_volume_unreadable_test.go

@@ -0,0 +1,211 @@
+package volume
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// getFailingStore fails the read of one key with an error that is not a
+// missing key, standing in for a document that is present but unreadable
+// (e.g. a storage-layer reconstruction failure).
+type getFailingStore struct {
+	*objectstore.MemoryObjectStore
+
+	failKey string
+}
+
+func (s *getFailingStore) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	if aws.StringValue(input.Key) == s.failKey {
+		return nil, errors.New("reconstruction failed: shard sizes do not match")
+	}
+	return s.MemoryObjectStore.GetObject(ctx, input)
+}
+
+// countingFailAfterStore fails every read of one key from its (succeedCalls+1)th
+// call onward, deterministically reproducing a document that answers one read
+// and then goes unreadable on the next, without timing.
+type countingFailAfterStore struct {
+	*objectstore.MemoryObjectStore
+
+	failKey      string
+	succeedCalls int
+	calls        int
+}
+
+func (s *countingFailAfterStore) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	if aws.StringValue(input.Key) == s.failKey {
+		s.calls++
+		if s.calls > s.succeedCalls {
+			return nil, errors.New("reconstruction failed: shard sizes do not match")
+		}
+	}
+	return s.MemoryObjectStore.GetObject(ctx, input)
+}
+
+// TestDeleteVolume_UnreadableDocumentIsInternalError pins the fix: a volume
+// document that fails to read for a reason other than absence must answer
+// InternalError, never InvalidVolume.NotFound. Reporting NotFound here is
+// what let terminate cleanup and the legacy reaper count a volume whose
+// state is unknown as already deleted.
+func TestDeleteVolume_UnreadableDocumentIsInternalError(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+
+	volumeID := "vol-unreadable0000"
+	createVolumeInStoreWithMeta(t, svc, memory, volumeID, ebsmetadata.Volume{
+		VolumeID: volumeID, CapacityGiB: 10, State: "available",
+	})
+
+	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, volumeID)}
+	svc.store = failing
+	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String(volumeID),
+	}, testVolAccountID)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, awserrors.ValidErrorCodeFromError(err),
+		"an unreadable document must answer InternalError, not NotFound")
+	assert.NotContains(t, err.Error(), awserrors.ErrorInvalidVolumeNotFound)
+
+	// Nothing was deleted: the document is still there (read through the
+	// underlying store, bypassing the fault), and so is the provider volume.
+	_, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
+	require.NoError(t, getErr, "the control-plane document must survive a failed delete")
+
+	provVol, provErr := svc.provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
+	})
+	require.NoError(t, provErr)
+	require.NotNil(t, provVol, "the provider's data must survive a failed delete")
+}
+
+// TestDeleteVolume_AbsentDocumentIsStillNotFound guards the other side of the
+// fix: absence (NoSuchKey) must keep returning InvalidVolume.NotFound, not be
+// swept up into the new InternalError branch.
+func TestDeleteVolume_AbsentDocumentIsStillNotFound(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-absent00000001"),
+	}, testVolAccountID)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
+}
+
+// TestDeleteVolume_CrossAccountDocumentIsStillNotFound guards the cross-tenant
+// branch: a document that exists but belongs to another account must keep
+// returning InvalidVolume.NotFound, exactly as before the fix.
+func TestDeleteVolume_CrossAccountDocumentIsStillNotFound(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-othertenant000", ebsmetadata.Volume{
+		VolumeID: "vol-othertenant000", CapacityGiB: 10, State: "available", TenantID: "222222222222",
+	})
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-othertenant000"),
+	}, testVolAccountID)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
+}
+
+// TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess reproduces the
+// production shape from mulga-salhr: the attachment-clear read inside
+// UpdateVolumeState succeeds, but the document goes unreadable before
+// DeleteVolume's own read of the same key. That read failure must come back
+// as a non-NotFound error, which is exactly the signal
+// daemon/vm_adapters.go's DeleteVolumes and instance/service_impl.go's
+// deleteInstanceVolumes use to tell a real failure from an already-gone
+// volume (!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err)).
+func TestDeleteVolumeOnTerminate_DeleteReadFailureIsNotSuccess(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+
+	volumeID := "vol-stopped-root2"
+	createVolumeInStoreWithMeta(t, svc, memory, volumeID, ebsmetadata.Volume{
+		VolumeID:         volumeID,
+		CapacityGiB:      10,
+		State:            "available",
+		AttachedInstance: "i-stopped",
+	})
+
+	// The first read of the document (UpdateVolumeState's attachment clear)
+	// succeeds; every read from the second onward (DeleteVolume's own read)
+	// fails, deterministically, with no reliance on timing.
+	failing := &countingFailAfterStore{MemoryObjectStore: memory, failKey: metadataKey(t, volumeID), succeedCalls: 1}
+	svc.store = failing
+	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
+
+	err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
+
+	require.Error(t, err, "a delete-read failure after a successful attachment clear must not be reported as success")
+	assert.False(t, awserrors.IsNotFound(err),
+		"terminate cleanup treats IsNotFound as already-gone; this must not qualify")
+	assert.False(t, objectstore.IsNoSuchKeyError(err),
+		"terminate cleanup treats IsNoSuchKeyError as already-gone; this must not qualify")
+
+	// The attachment was cleared (terminate implies detach), but the document
+	// and the provider volume must both still exist: nothing was deleted.
+	_, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
+	require.NoError(t, getErr, "the control-plane document must survive a failed delete")
+
+	provVol, provErr := svc.provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
+	})
+	require.NoError(t, provErr)
+	require.NotNil(t, provVol, "the provider's data must survive a failed delete")
+}
+
+// TestForceDetachVolume_UnreadableDocumentIsInternalError: an attached volume
+// whose document is present but unreadable must answer InternalError and keep
+// its attachment; absence and another account's volume stay NotFound.
+func TestForceDetachVolume_UnreadableDocumentIsInternalError(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+
+	volumeID := "vol-forcedetach000"
+	createVolumeInStoreWithMeta(t, svc, memory, volumeID, ebsmetadata.Volume{
+		VolumeID: volumeID, CapacityGiB: 10, State: "in-use", AttachedInstance: "i-0123456789abcdef0",
+	})
+
+	failing := &getFailingStore{MemoryObjectStore: memory, failKey: metadataKey(t, volumeID)}
+	svc.store = failing
+	svc.metadata = ebsmetadata.NewStore(failing, "test-bucket")
+
+	_, err := svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId: aws.String(volumeID),
+	}, testVolAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, awserrors.ValidErrorCodeFromError(err))
+
+	meta, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
+	require.NoError(t, getErr)
+	assert.Equal(t, "in-use", meta.State, "a failed force-detach must not clear the attachment")
+	assert.Equal(t, "i-0123456789abcdef0", meta.AttachedInstance)
+
+	_, err = svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId: aws.String("vol-doesnotexist00"),
+	}, testVolAccountID)
+	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, awserrors.ValidErrorCodeFromError(err))
+
+	_, err = svc.ForceDetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId: aws.String(volumeID),
+	}, "999999999999")
+	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, awserrors.ValidErrorCodeFromError(err))
+}

spinifex/domains/ec2/volume/service_impl.go

@@ -289,6 +289,7 @@ var describeVolumesValidFilters = map[string]bool{
 	"attachment.status":      true,
 	"attachment.device":      true,
 	"availability-zone":      true,
+	"snapshot-id":            true,
 }
 
 // DescribeVolumes lists EBS volumes by reading config.json files from S3.
@@ -401,6 +402,10 @@ func volumeMatchesFilters(vol *ec2.Volume, filters map[string][]string) bool {
 			if vol.AvailabilityZone != nil {
 				field = *vol.AvailabilityZone
 			}
+		case "snapshot-id":
+			if vol.SnapshotId != nil {
+				field = *vol.SnapshotId
+			}
 		default:
 			return false
 		}
@@ -1056,7 +1061,10 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
 
 	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
 	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "get volume metadata %s: %v", volumeID, err)
 	}
 	if meta.TenantID != accountID {
 		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
@@ -1091,10 +1099,15 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV
 		return nil, err
 	}
 
-	// AWS-faithful: an absent volume returns InvalidVolume.NotFound.
+	// AWS-faithful: an absent volume returns InvalidVolume.NotFound. Any other
+	// read failure means the document's state is unknown, so it must not be
+	// reported or treated as already deleted.
 	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
 	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "get volume metadata %s: %v", volumeID, err)
 	}
 	if meta.TenantID != accountID {
 		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)

spinifex/domains/ec2/volume/service_impl_test.go

@@ -2081,6 +2081,40 @@ func TestDescribeVolumes_FilterByAZ(t *testing.T) {
 	assert.Equal(t, "vol-az1", *out.Volumes[0].VolumeId)
 }
 
+func TestDescribeVolumes_FilterBySnapshotId(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-fromsnap", ebsmetadata.Volume{
+		VolumeID: "vol-fromsnap", CapacityGiB: 10, State: "available",
+		SnapshotID: "snap-src", TenantID: "000000000001",
+	})
+	createVolumeInStoreWithMeta(t, svc, store, "vol-othersnap", ebsmetadata.Volume{
+		VolumeID: "vol-othersnap", CapacityGiB: 10, State: "available",
+		SnapshotID: "snap-other", TenantID: "000000000001",
+	})
+	createVolumeInStoreWithMeta(t, svc, store, "vol-blank", ebsmetadata.Volume{
+		VolumeID: "vol-blank", CapacityGiB: 10, State: "available", TenantID: "000000000001",
+	})
+
+	describe := func(values ...string) []string {
+		t.Helper()
+		out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
+			Filters: []*ec2.Filter{{Name: aws.String("snapshot-id"), Values: aws.StringSlice(values)}},
+		}, "000000000001")
+		require.NoError(t, err)
+		ids := make([]string, 0, len(out.Volumes))
+		for _, v := range out.Volumes {
+			ids = append(ids, *v.VolumeId)
+		}
+		return ids
+	}
+
+	assert.Equal(t, []string{"vol-fromsnap"}, describe("snap-src"))
+	assert.ElementsMatch(t, []string{"vol-fromsnap", "vol-othersnap"}, describe("snap-src", "snap-other"))
+	assert.Empty(t, describe("snap-unknown"))
+}
+
 func TestDescribeVolumes_FilterMultipleValues_OR(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)

spinifex/domains/network/reconcile/lock.go

@@ -41,12 +41,12 @@ func AcquireLeader(ctx context.Context, nc *nats.Conn, bucket, holder string) (f
 	})
 	if err != nil {
 		slog.ErrorContext(ctx, "reconcile/lock: lease config invalid",
-			"holder", holder, "bucket", bucket, "err", err)
+			"node", holder, "bucket", bucket, "err", err)
 		return nil, false
 	}
 	if !lease.TryAcquire(ctx) {
 		slog.InfoContext(ctx, "reconcile/lock: not leader, skipping reconcile",
-			"holder", holder, "bucket", bucket)
+			"node", holder, "bucket", bucket)
 		return nil, false
 	}
 	return func() { lease.Release(ctx) }, true

spinifex/domains/network/reconcile/lock_test.go

@@ -1,7 +1,10 @@
 package reconcile
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"log/slog"
 	"testing"
 	"time"
 
@@ -103,3 +106,34 @@ func TestAcquireLeader_ReleaseDoesNotDeleteSuccessorKey(t *testing.T) {
 	require.NoError(t, err, "stale release deleted the successor's leader key")
 	assert.Equal(t, "node-2", string(entry.Value()), "successor's key must survive a stale release")
 }
+
+// A node that loses the election logs itself as node: "holder=node-2" on
+// node-2's log reads as though node-2 holds the lock.
+func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
+	require.True(t, ok)
+	defer release()
+
+	var buf bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+
+	_, ok = AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
+	require.False(t, ok)
+
+	var line map[string]any
+	dec := json.NewDecoder(&buf)
+	for dec.More() {
+		var rec map[string]any
+		require.NoError(t, dec.Decode(&rec))
+		if rec["msg"] == "reconcile/lock: not leader, skipping reconcile" {
+			line = rec
+		}
+	}
+	require.NotNil(t, line, "a losing acquire must log that it is skipping")
+	assert.Equal(t, "node-2", line["node"])
+	assert.NotEqual(t, "node-2", line["holder"], "the local node is not the holder")
+}

spinifex/foundation/state/kvlease/lease.go

@@ -18,7 +18,10 @@ import (
 // context and so cannot inherit the caller's deadline.
 const releaseTimeout = 5 * time.Second
 
-var errNotHolder = errors.New("kvlease: another node holds the lease")
+// heldError reports the node that owns the key when a claim loses to it.
+type heldError struct{ holder string }
+
+func (e *heldError) Error() string { return "kvlease: lease held by " + e.holder }
 
 // BucketFunc returns the KV bucket holding the lease key. It is injected because
 // each subsystem initialises its own bucket, some with migrations attached.
@@ -83,10 +86,11 @@ func (l *Lease) Held() bool {
 	return l.held
 }
 
-// logArgs prefixes the lease identity onto a log line's own fields.
+// logArgs prefixes the lease identity onto a log line's own fields. The local
+// node logs as "node"; "holder" is reserved for whoever owns the key.
 func (l *Lease) logArgs(extra ...any) []any {
 	args := make([]any, 0, 4+len(l.cfg.Attrs)+len(extra))
-	args = append(args, "lease", l.cfg.Name, "holder", l.cfg.Holder)
+	args = append(args, "lease", l.cfg.Name, "node", l.cfg.Holder)
 	args = append(args, l.cfg.Attrs...)
 	return append(args, extra...)
 }
@@ -104,6 +108,9 @@ func (l *Lease) TryAcquire(ctx context.Context) bool {
 	}
 	rev, err := l.claim(ctx, kv)
 	if err != nil {
+		if held, ok := errors.AsType[*heldError](err); ok {
+			slog.DebugContext(ctx, "kvlease: held by another node", l.logArgs("holder", held.holder)...)
+		}
 		return false
 	}
 
@@ -142,7 +149,7 @@ func (l *Lease) claim(ctx context.Context, kv jetstream.KeyValue) (uint64, error
 		return 0, gerr
 	}
 	if string(entry.Value()) != l.cfg.Holder {
-		return 0, errNotHolder
+		return 0, &heldError{holder: string(entry.Value())}
 	}
 	return kv.Update(ctx, l.cfg.Key, []byte(l.cfg.Holder), entry.Revision())
 }

spinifex/foundation/state/kvlease/lease_test.go

@@ -3,6 +3,7 @@ package kvlease_test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"log/slog"
 	"sync/atomic"
@@ -388,3 +389,49 @@ func TestRelease_AfterRenewalLossWarns(t *testing.T) {
 	a.Release(t.Context())
 	assert.Contains(t, buf.String(), "already lost before release")
 }
+
+// logLines captures default-logger output as decoded JSON records.
+func logLines(t *testing.T) func() []map[string]any {
+	t.Helper()
+	var buf bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return func() []map[string]any {
+		var out []map[string]any
+		dec := json.NewDecoder(&buf)
+		for dec.More() {
+			var rec map[string]any
+			require.NoError(t, dec.Decode(&rec))
+			out = append(out, rec)
+		}
+		return out
+	}
+}
+
+// "holder" must name whoever owns the key. The local node logs as "node", so a
+// loser's log does not read as though it won.
+func TestLogFields_SeparateLocalNodeFromCurrentHolder(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	read := logLines(t)
+
+	a := newTestLease(t, nc, "node-a", nil)
+	require.True(t, a.TryAcquire(t.Context()))
+	t.Cleanup(func() { a.Release(t.Context()) })
+	b := newTestLease(t, nc, "node-b", nil)
+	require.False(t, b.TryAcquire(t.Context()))
+
+	var elected, refused map[string]any
+	for _, rec := range read() {
+		switch {
+		case rec["msg"] == "kvlease: elected" && rec["node"] == "node-a":
+			elected = rec
+		case rec["node"] == "node-b":
+			refused = rec
+		}
+	}
+	require.NotNil(t, elected, "the elected line must carry the local node as node")
+	assert.NotContains(t, elected, "holder")
+	require.NotNil(t, refused, "a refused acquire must log the local node as node")
+	assert.Equal(t, "node-a", refused["holder"], "a refused acquire must report the node that holds the key")
+}

spinifex/foundation/state/kvstore/bucket.go

@@ -378,7 +378,8 @@ func (b *Bucket) WatchFrom(ctx context.Context, filter string, revision uint64)
 	return w, nil
 }
 
-// open picks the kvutil helper matching the configured TTL and replica count.
+// open attaches to the bucket or creates it through kvutil, which sets the
+// cluster's replica count on every bucket, TTL-backed or not.
 func (b *Bucket) open(ctx context.Context) (jetstream.KeyValue, error) {
 	switch {
 	case b.cfg.AttachOnly:

spinifex/foundation/state/kvstore/cluster_test.go

@@ -0,0 +1,95 @@
+package kvstore_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// startJetStreamCluster starts n routed JetStream servers and returns a handle
+// connected to the first, so a create can actually place n replicas.
+func startJetStreamCluster(t *testing.T, n int) jetstream.JetStream {
+	t.Helper()
+	ports := make([]int, n)
+	routes := make([]string, n)
+	for i := range ports {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		ports[i] = l.Addr().(*net.TCPAddr).Port
+		require.NoError(t, l.Close())
+		routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
+	}
+	servers := make([]*server.Server, n)
+	for i := range servers {
+		ns, err := server.NewServer(&server.Options{
+			ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: t.TempDir(),
+			Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
+			Cluster: server.ClusterOpts{Name: "kvstore", Host: "127.0.0.1", Port: ports[i]},
+			Routes:  server.RoutesFromStr(strings.Join(routes, ",")),
+		})
+		require.NoError(t, err)
+		ns.Start()
+		servers[i] = ns
+	}
+	t.Cleanup(func() {
+		for _, ns := range servers {
+			ns.Shutdown()
+			ns.WaitForShutdown()
+		}
+	})
+
+	nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	return js
+}
+
+// TestBucket_TTLBucketIsReplicatedAcrossTheCluster pins TTL and replica count
+// together: a TTL bucket created at one replica loses its markers with one node,
+// and the usage dedupe bucket is exactly that kind of bucket.
+func TestBucket_TTLBucketIsReplicatedAcrossTheCluster(t *testing.T) {
+	const nodes = 3
+	js := startJetStreamCluster(t, nodes)
+	clustersize.RedeclareForTest(t, nodes)
+
+	bucket := kvstore.NewBucket(js, kvstore.Config{
+		Name:    "kvstore-ttl-replicated",
+		History: 1,
+		TTL:     90 * time.Minute,
+	})
+
+	// Wait for a meta leader before the one open, so a retried open cannot find a
+	// one-replica bucket from a timed-out create and raise it into a false pass.
+	var lastErr error
+	ready := assert.Eventually(t, func() bool {
+		ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+		defer cancel()
+		_, lastErr = js.AccountInfo(ctx)
+		return lastErr == nil
+	}, 30*time.Second, 50*time.Millisecond)
+	require.True(t, ready, "JetStream cluster never elected a meta leader: %v", lastErr)
+
+	kv, err := bucket.KV(t.Context())
+	require.NoError(t, err)
+
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, 90*time.Minute, status.TTL(), "Config.TTL must reach the created bucket")
+	bucketStatus, ok := status.(*jetstream.KeyValueBucketStatus)
+	require.True(t, ok, "status is %T", status)
+	assert.Equal(t, nodes, bucketStatus.StreamInfo().Config.Replicas,
+		"a TTL bucket must be created at the cluster's replica count")
+}

spinifex/gateway/admin.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"io"
@@ -54,6 +55,15 @@ func adminPathMethod(path string) (string, bool) {
 	return rest, true
 }
 
+// ctxAdminRequestID carries the /admin request ID to handlers that log under it.
+// It correlates log records only and authorises nothing.
+const ctxAdminRequestID contextKey = "admin.requestID"
+
+func adminRequestIDFrom(ctx context.Context) string {
+	id, _ := ctx.Value(ctxAdminRequestID).(string)
+	return id
+}
+
 // adminMaxBodyBytes caps the JSON request body. The largest legitimate request
 // is a few hundred bytes; the cap exists so an unauthenticated-shaped mistake
 // cannot make the gateway buffer megabytes.
@@ -81,7 +91,7 @@ func (gw *GatewayConfig) Admin_Request(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("X-Amzn-Requestid", requestID)
 
 	method := chi.URLParam(r, "method")
-	ctx := r.Context()
+	ctx := context.WithValue(r.Context(), ctxAdminRequestID, requestID)
 
 	// One handler for every verb rather than a POST route plus a catch-all:
 	// chi resolves the two to the same pattern, and the catch-all wins.

spinifex/gateway/admin_deleteaccount.go

@@ -121,7 +121,7 @@ func (gw *GatewayConfig) adminDeleteAccount(ctx context.Context, body []byte) (a
 		inventoryCtx, cancel := context.WithTimeout(ctx, accountDeletionInventoryTimeout)
 		defer cancel()
 
-		inventory, err := engine.Inventory(inventoryCtx, req.AccountID)
+		inventory, err := engine.DryRun(inventoryCtx, req.AccountID, adminRequestIDFrom(ctx))
 		if err != nil {
 			return nil, teardownError("DeleteAccount", req.AccountID, err)
 		}

spinifex/gateway/admin_deleteaccount_dryrun_test.go

@@ -0,0 +1,99 @@
+package gateway
+
+//test:in-package — the admin dispatcher's request-ID context and the
+// deletion handler it reaches are unexported.
+
+import (
+	"bytes"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const dryRunShadowMessage = "Account teardown dry-run EC2 platform shadow comparison"
+
+// deleteAccountGrant is the real IAM service with a policy granting
+// spinifex:DeleteAccount, so a request clears the admin gates.
+type deleteAccountGrant struct {
+	handlers_iam.IAMService
+}
+
+func (deleteAccountGrant) GetUserPolicies(string, string) ([]handlers_iam.PolicyDocument, error) {
+	return []handlers_iam.PolicyDocument{{
+		Version: "2012-10-17",
+		Statement: []handlers_iam.Statement{{
+			Effect: "Allow", Action: handlers_iam.StringOrArr{"spinifex:DeleteAccount"}, Resource: handlers_iam.StringOrArr{"*"},
+		}},
+	}}, nil
+}
+
+// syncBuffer is a log sink safe for the gateway's concurrent writers.
+type syncBuffer struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+func (b *syncBuffer) Write(p []byte) (int, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.Write(p)
+}
+
+func (b *syncBuffer) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.String()
+}
+
+func captureDefaultLog(t *testing.T) *syncBuffer {
+	t.Helper()
+	logs := &syncBuffer{}
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewJSONHandler(logs, nil)))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return logs
+}
+
+// The admin API dry run answers exactly as the plain inventory would, and its
+// shadow comparison is logged under the request ID the response carries.
+func TestAdminDeleteAccountDryRunLogsTheShadowUnderTheRequestID(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.IAMService = deleteAccountGrant{IAMService: gw.IAMService}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	logs := captureDefaultLog(t)
+
+	rec := authorizedAdminRequestForMethod(gw, "DeleteAccount", `{"accountId":"`+account.AccountID+`","dryRun":true}`)
+	require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
+	requestID := rec.Header().Get("X-Amzn-Requestid")
+	require.NotEmpty(t, requestID)
+
+	engine, err := accountteardown.NewClusterEngine(t.Context(), gw.NATSConn, gw.ExpectedNodes, gw.NodeIDs, gw.IAMService, gw.BucketStore)
+	require.NoError(t, err)
+	inventory, err := engine.Inventory(t.Context(), account.AccountID)
+	require.NoError(t, err)
+	want, err := json.Marshal(&DeleteAccountResponse{AccountID: account.AccountID, State: "DRY_RUN", DryRun: true, Inventory: inventory})
+	require.NoError(t, err)
+	assert.JSONEq(t, string(want), rec.Body.String())
+
+	var records []map[string]any
+	for line := range strings.SplitSeq(strings.TrimSpace(logs.String()), "\n") {
+		var record map[string]any
+		if json.Unmarshal([]byte(line), &record) == nil && record["msg"] == dryRunShadowMessage {
+			records = append(records, record)
+		}
+	}
+	require.Len(t, records, 1)
+	assert.Equal(t, requestID, records[0]["correlationRef"])
+	assert.Equal(t, account.AccountID, records[0]["accountID"])
+	// The stub cluster answers every ec2 subject with an empty body, which is
+	// no inventory for this account, so the comparison cannot be clean.
+	assert.Equal(t, "scan-incomplete", records[0]["outcome"])
+}

spinifex/gateway/auth_current_behaviour_test.go

@@ -0,0 +1,882 @@
+package gateway
+
+//test:in-package — this characterization drives unexported gateway wiring.
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts"
+)
+
+// The TestCurrentBehaviour_ tests in this file pin today's SigV4 authentication,
+// STS session, revocation, backing-store and identity-policy outcomes at the
+// AWSGW door. They drive the real router over real IAM and STS services on an
+// embedded JetStream, and describe current behaviour, not the target design.
+
+const (
+	cbRootAKID   = "AKIACBROOTKEYAAAAAAA"
+	cbRootSecret = "cb-root-secret-0123456789abcdefghijklmn"
+	cbUnknownKey = "AKIACBUNKNOWNKEYAAAA"
+	cbUnknownSes = "ASIACBUNKNOWNSESSION"
+	cbBadSecret  = "WRONG-secret-0123456789abcdefghijklmnop"
+)
+
+type cbCreds struct {
+	akid, secret, token string
+}
+
+// cbDecryptCounter is the real IAM service with DecryptSecret counted, so a test
+// can observe whether the door decrypted a secret before rejecting a request.
+type cbDecryptCounter struct {
+	*handlers_iam.IAMServiceImpl
+
+	decrypts atomic.Int32
+}
+
+func (c *cbDecryptCounter) DecryptSecret(ciphertext string) (string, error) {
+	c.decrypts.Add(1)
+	return c.IAMServiceImpl.DecryptSecret(ciphertext)
+}
+
+type cbEnvOpts struct {
+	// noRevoker leaves IAM without the STS session revoker, standing in for a
+	// revocation that failed at delete time.
+	noRevoker bool
+	// separateIAMConn gives IAM and STS their own NATS connection, so it can be
+	// closed while the gateway's own connection stays up.
+	separateIAMConn bool
+}
+
+type cbEnv struct {
+	nc        *nats.Conn // the gateway's connection
+	iamNC     *nats.Conn // IAM/STS connection; == nc unless separateIAMConn
+	js        jetstream.JetStream
+	iam       *handlers_iam.IAMServiceImpl
+	sts       *handlers_sts.STSServiceImpl
+	counter   *cbDecryptCounter
+	h         http.Handler
+	accountID string
+	alice     cbCreds
+	aliceID   string
+	addr      atomic.Uint32
+}
+
+func newCBEnv(t *testing.T, opts cbEnvOpts) *cbEnv {
+	t.Helper()
+	srv, nc, _ := testutil.StartTestJetStream(t)
+	e := &cbEnv{nc: nc, iamNC: nc, js: testutil.NewJetStream(t, nc)}
+	if opts.separateIAMConn {
+		iamNC, err := nats.Connect(srv.ClientURL())
+		require.NoError(t, err)
+		t.Cleanup(iamNC.Close)
+		e.iamNC = iamNC
+	}
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	e.iam, err = handlers_iam.NewIAMServiceImpl(t.Context(), e.iamNC, masterKey)
+	require.NoError(t, err)
+	iamJS := testutil.NewJetStream(t, e.iamNC)
+	e.sts, err = handlers_sts.NewSTSServiceImpl(t.Context(), e.iamNC, e.iam, masterKey, handlers_eks.NewOIDCJWKSReader(iamJS))
+	require.NoError(t, err)
+	if !opts.noRevoker {
+		e.iam.SetSessionRevoker(e.sts)
+	}
+
+	encrypted, err := handlers_iam.EncryptSecret(cbRootSecret, masterKey)
+	require.NoError(t, err)
+	require.NoError(t, e.iam.SeedBootstrap(&handlers_iam.BootstrapData{
+		AccessKeyID: cbRootAKID, EncryptedSecret: encrypted, AccountID: awsidentifiers.GlobalAccountID,
+	}))
+
+	acct, err := e.iam.CreateAccount("tenant")
+	require.NoError(t, err)
+	require.Equal(t, testDiscAccount, acct.AccountID, "web-identity fixtures assume the first tenant account")
+	e.accountID = acct.AccountID
+	e.alice, e.aliceID = e.newUser(t, "alice")
+
+	e.counter = &cbDecryptCounter{IAMServiceImpl: e.iam}
+	e.h = e.gateway()
+	return e
+}
+
+// gateway builds a router over the env's services with its own (cold) caches.
+func (e *cbEnv) gateway() http.Handler {
+	gw := &GatewayConfig{
+		DisableLogging: true,
+		NATSConn:       e.nc,
+		Region:         testRegion,
+		ExpectedNodes:  1,
+		IAMService:     e.counter,
+		STSService:     e.sts,
+	}
+	return gw.SetupRoutes()
+}
+
+func (e *cbEnv) newUser(t *testing.T, name string) (cbCreds, string) {
+	t.Helper()
+	u, err := e.iam.CreateUser(e.accountID, &iam.CreateUserInput{UserName: aws.String(name)})
+	require.NoError(t, err)
+	return e.newKey(t, name), aws.StringValue(u.User.UserId)
+}
+
+func (e *cbEnv) newKey(t *testing.T, user string) cbCreds {
+	t.Helper()
+	k, err := e.iam.CreateAccessKey(e.accountID, &iam.CreateAccessKeyInput{UserName: aws.String(user)})
+	require.NoError(t, err)
+	return cbCreds{akid: aws.StringValue(k.AccessKey.AccessKeyId), secret: aws.StringValue(k.AccessKey.SecretAccessKey)}
+}
+
+func (e *cbEnv) putUserPolicy(t *testing.T, user, doc string) {
+	t.Helper()
+	_, err := e.iam.PutUserPolicy(e.accountID, &iam.PutUserPolicyInput{
+		UserName: aws.String(user), PolicyName: aws.String("p"), PolicyDocument: aws.String(doc),
+	})
+	require.NoError(t, err)
+}
+
+// request builds a signed (or, for a zero cbCreds, unsigned) query-protocol POST.
+// Each request gets a distinct source address so the auth-failure lockout,
+// which is per source IP, never couples one case to another.
+func (e *cbEnv) request(t *testing.T, h http.Handler, c cbCreds, service string, form url.Values) *httptest.ResponseRecorder {
+	t.Helper()
+	body := []byte(form.Encode())
+	req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))
+	req.Host = "localhost:9999"
+	n := e.addr.Add(1)
+	req.RemoteAddr = fmt.Sprintf("10.%d.%d.%d:40000", (n>>16)&0xff, (n>>8)&0xff, n&0xff)
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	if c.token != "" {
+		req.Header.Set("X-Amz-Security-Token", c.token)
+	}
+	if c.akid != "" {
+		signTestRequestAs(t, req, body, c.akid, c.secret, service, time.Now().UTC())
+	}
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	return rec
+}
+
+func (e *cbEnv) callerIdentity(t *testing.T, c cbCreds) *httptest.ResponseRecorder {
+	t.Helper()
+	return e.request(t, e.h, c, "sts", cbForm("GetCallerIdentity"))
+}
+
+// cbIAMActions are the IAM actions these tests send; every other action is STS.
+var cbIAMActions = map[string]bool{"ListUsers": true, "CreateUser": true, "GetUser": true}
+
+func cbForm(action string, kv ...string) url.Values {
+	v := url.Values{"Action": {action}, "Version": {"2011-06-15"}}
+	if cbIAMActions[action] {
+		v.Set("Version", "2010-05-08")
+	}
+	for i := 0; i+1 < len(kv); i += 2 {
+		v.Set(kv[i], kv[i+1])
+	}
+	return v
+}
+
+func cbXML(body, tag string) string {
+	m := regexp.MustCompile(`<` + tag + `>([^<]*)</` + tag + `>`).FindStringSubmatch(body)
+	if m == nil {
+		return ""
+	}
+	return m[1]
+}
+
+// cbOutcome is the wire verdict a client sees: status, AWS error code, message.
+type cbOutcome struct {
+	status  int
+	code    string
+	message string
+}
+
+func cbOutcomeOf(rec *httptest.ResponseRecorder) cbOutcome {
+	body := rec.Body.String()
+	return cbOutcome{status: rec.Code, code: cbXML(body, "Code"), message: cbXML(body, "Message")}
+}
+
+func cbRegistry(code string) cbOutcome {
+	m := awserrors.ErrorLookup[code]
+	return cbOutcome{status: m.HTTPCode, code: code, message: m.Message}
+}
+
+func cbSessionFrom(t *testing.T, rec *httptest.ResponseRecorder) cbCreds {
+	t.Helper()
+	require.Equalf(t, http.StatusOK, rec.Code, "mint failed: %s", rec.Body.String())
+	body := rec.Body.String()
+	c := cbCreds{akid: cbXML(body, "AccessKeyId"), secret: cbXML(body, "SecretAccessKey"), token: cbXML(body, "SessionToken")}
+	require.True(t, strings.HasPrefix(c.akid, "ASIA"), "session AKID %q", c.akid)
+	require.NotEmpty(t, c.token)
+	return c
+}
+
+// createAssumableRole creates role name trusting alice, and grants alice
+// sts:AssumeRole on it.
+func (e *cbEnv) createAssumableRole(t *testing.T, name string) (roleARN, roleID string) {
+	t.Helper()
+	trust := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::` +
+		e.accountID + `:user/alice"},"Action":"sts:AssumeRole"}]}`
+	out, err := e.iam.CreateRole(e.accountID, &iam.CreateRoleInput{
+		RoleName: aws.String(name), AssumeRolePolicyDocument: aws.String(trust),
+	})
+	require.NoError(t, err)
+	roleARN = aws.StringValue(out.Role.Arn)
+	e.putUserPolicy(t, "alice", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"*"}]}`)
+	return roleARN, aws.StringValue(out.Role.RoleId)
+}
+
+func (e *cbEnv) assumeRole(t *testing.T, roleARN, session string) cbCreds {
+	t.Helper()
+	return cbSessionFrom(t, e.request(t, e.h, e.alice, "sts",
+		cbForm("AssumeRole", "RoleArn", roleARN, "RoleSessionName", session)))
+}
+
+func (e *cbEnv) getSessionToken(t *testing.T, c cbCreds) cbCreds {
+	t.Helper()
+	return cbSessionFrom(t, e.request(t, e.h, c, "sts", cbForm("GetSessionToken")))
+}
+
+// expireSession rewrites a stored session record's expiry into the past.
+func (e *cbEnv) expireSession(t *testing.T, akid string) {
+	t.Helper()
+	kv, err := e.js.KeyValue(t.Context(), handlers_sts.KVBucketSessionCredentials)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), akid)
+	require.NoError(t, err)
+	var cred handlers_sts.SessionCredential
+	require.NoError(t, json.Unmarshal(entry.Value(), &cred))
+	cred.ExpiresAt = time.Now().UTC().Add(-time.Minute)
+	data, err := json.Marshal(cred)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), akid, data)
+	require.NoError(t, err)
+}
+
+func (e *cbEnv) sessionStored(t *testing.T, akid string) bool {
+	t.Helper()
+	cred, err := e.sts.LookupSessionCredential(akid)
+	require.NoError(t, err)
+	return cred != nil
+}
+
+func tamper(s string) string {
+	b := []byte(s)
+	if b[0] == 'A' {
+		b[0] = 'B'
+	} else {
+		b[0] = 'A'
+	}
+	return string(b)
+}
+
+// Pins CURRENT behaviour: how a valid long-lived key resolves to a principal and account.
+func TestCurrentBehaviour_Auth_LongLivedKeyResolvesPrincipal(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+
+	t.Run("iam_user", func(t *testing.T) {
+		rec := e.callerIdentity(t, e.alice)
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		body := rec.Body.String()
+		assert.Equal(t, "arn:aws:iam::"+e.accountID+":user/alice", cbXML(body, "Arn"))
+		assert.Equal(t, e.aliceID, cbXML(body, "UserId"))
+		assert.Equal(t, e.accountID, cbXML(body, "Account"))
+	})
+
+	t.Run("global_root", func(t *testing.T) {
+		rec := e.callerIdentity(t, cbCreds{akid: cbRootAKID, secret: cbRootSecret})
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		body := rec.Body.String()
+		assert.Equal(t, "arn:aws:iam::"+awsidentifiers.GlobalAccountID+":root", cbXML(body, "Arn"))
+		assert.Equal(t, awsidentifiers.GlobalAccountID, cbXML(body, "UserId"))
+		assert.Equal(t, awsidentifiers.GlobalAccountID, cbXML(body, "Account"))
+	})
+
+	// CURRENT: a tenant IAM user literally named "root" is resolved with the
+	// account ID as its UserId (aws:userid), the value AWS reserves for the
+	// account root, rather than the AIDA ID stored on its own user record.
+	t.Run("tenant_user_named_root", func(t *testing.T) {
+		creds, storedID := e.newUser(t, "root")
+		require.True(t, strings.HasPrefix(storedID, "AIDA"))
+		rec := e.callerIdentity(t, creds)
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		body := rec.Body.String()
+		assert.Equal(t, "arn:aws:iam::"+e.accountID+":user/root", cbXML(body, "Arn"))
+		assert.Equal(t, e.accountID, cbXML(body, "UserId"))
+		assert.NotEqual(t, storedID, cbXML(body, "UserId"))
+	})
+}
+
+// Pins CURRENT behaviour: wire outcomes for invalid long-lived credentials, and
+// that key status changes apply to the very next request (no credential cache).
+func TestCurrentBehaviour_Auth_LongLivedRejections(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	gci := cbForm("GetCallerIdentity")
+
+	t.Run("unknown_key", func(t *testing.T) {
+		got := cbOutcomeOf(e.request(t, e.h, cbCreds{akid: cbUnknownKey, secret: cbBadSecret}, "sts", gci))
+		assert.Equal(t, cbRegistry(awserrors.ErrorInvalidClientTokenId), got)
+	})
+
+	t.Run("wrong_secret", func(t *testing.T) {
+		got := cbOutcomeOf(e.request(t, e.h, cbCreds{akid: e.alice.akid, secret: cbBadSecret}, "sts", gci))
+		assert.Equal(t, cbRegistry(awserrors.ErrorSignatureDoesNotMatch), got)
+	})
+
+	t.Run("malformed_authorization_header", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(gci.Encode()))
+		req.RemoteAddr = "10.200.0.1:40000"
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=garbage")
+		rec := httptest.NewRecorder()
+		e.h.ServeHTTP(rec, req)
+		assert.Equal(t, cbRegistry(awserrors.ErrorIncompleteSignature), cbOutcomeOf(rec))
+	})
+
+	t.Run("key_made_inactive_after_use", func(t *testing.T) {
+		creds := e.newKey(t, "alice")
+		require.Equal(t, http.StatusOK, e.callerIdentity(t, creds).Code)
+		_, err := e.iam.UpdateAccessKey(e.accountID, &iam.UpdateAccessKeyInput{
+			UserName: aws.String("alice"), AccessKeyId: aws.String(creds.akid), Status: aws.String(handlers_iam.AccessKeyStatusInactive),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, cbRegistry(awserrors.ErrorInvalidClientTokenId), cbOutcomeOf(e.callerIdentity(t, creds)))
+	})
+
+	t.Run("key_deleted_after_use", func(t *testing.T) {
+		u, _ := e.newUser(t, "dave")
+		require.Equal(t, http.StatusOK, e.callerIdentity(t, u).Code)
+		_, err := e.iam.DeleteAccessKey(e.accountID, &iam.DeleteAccessKeyInput{
+			UserName: aws.String("dave"), AccessKeyId: aws.String(u.akid),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, cbRegistry(awserrors.ErrorInvalidClientTokenId), cbOutcomeOf(e.callerIdentity(t, u)))
+	})
+}
+
+// Pins CURRENT behaviour: AssumeRole session credentials at the SigV4 door.
+func TestCurrentBehaviour_Session_AssumeRoleCredentials(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	roleARN, roleID := e.createAssumableRole(t, "app")
+	s := e.assumeRole(t, roleARN, "s1")
+	other := e.assumeRole(t, roleARN, "s2")
+
+	t.Run("valid_session", func(t *testing.T) {
+		rec := e.callerIdentity(t, s)
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		body := rec.Body.String()
+		assert.Equal(t, "arn:aws:sts::"+e.accountID+":assumed-role/app/s1", cbXML(body, "Arn"))
+		assert.Equal(t, roleID+":s1", cbXML(body, "UserId"))
+		assert.Equal(t, e.accountID, cbXML(body, "Account"))
+	})
+
+	invalid := cbRegistry(awserrors.ErrorInvalidClientTokenId)
+	for name, c := range map[string]cbCreds{
+		"missing_token":        {akid: s.akid, secret: s.secret},
+		"other_sessions_token": {akid: s.akid, secret: s.secret, token: other.token},
+		"tampered_token":       {akid: s.akid, secret: s.secret, token: tamper(s.token)},
+		"unknown_session_akid": {akid: cbUnknownSes, secret: s.secret, token: s.token},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, c)))
+		})
+	}
+
+	// CURRENT: an AKIA key ignores X-Amz-Security-Token entirely; the header is
+	// only signed, never checked, so a long-lived key plus any token succeeds.
+	t.Run("long_lived_with_session_token", func(t *testing.T) {
+		rec := e.callerIdentity(t, cbCreds{akid: e.alice.akid, secret: e.alice.secret, token: s.token})
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		assert.Equal(t, "arn:aws:iam::"+e.accountID+":user/alice", cbXML(rec.Body.String(), "Arn"))
+	})
+
+	t.Run("valid_token_wrong_secret", func(t *testing.T) {
+		got := cbOutcomeOf(e.callerIdentity(t, cbCreds{akid: s.akid, secret: cbBadSecret, token: s.token}))
+		assert.Equal(t, cbRegistry(awserrors.ErrorSignatureDoesNotMatch), got)
+	})
+}
+
+// Pins CURRENT behaviour: an expired session answers ExpiredToken (HTTP 400)
+// before its session token is examined.
+func TestCurrentBehaviour_Session_Expired(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	roleARN, _ := e.createAssumableRole(t, "app")
+	s := e.assumeRole(t, roleARN, "s1")
+	require.Equal(t, http.StatusOK, e.callerIdentity(t, s).Code)
+	e.expireSession(t, s.akid)
+
+	// CURRENT: ExpiredToken is 400 here; AWS STS/IAM answer it as 403 (unverified).
+	expired := cbRegistry(awserrors.ErrorExpiredToken)
+	require.Equal(t, http.StatusBadRequest, expired.status)
+	for name, c := range map[string]cbCreds{
+		"valid_token": s,
+		// CURRENT: expiry is checked ahead of the token, so a caller holding only
+		// the cleartext ASIA key ID learns the session has expired.
+		"no_token":    {akid: s.akid, secret: s.secret},
+		"wrong_token": {akid: s.akid, secret: cbBadSecret, token: tamper(s.token)},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, expired, cbOutcomeOf(e.callerIdentity(t, c)))
+		})
+	}
+}
+
+// Pins CURRENT behaviour: a GetSessionToken session resolves to the IAM user and
+// outlives the long-lived key that minted it.
+func TestCurrentBehaviour_Session_GetSessionTokenOutlivesMintingKey(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	mintKey := e.newKey(t, "alice")
+	s := e.getSessionToken(t, mintKey)
+
+	rec := e.callerIdentity(t, s)
+	require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+	assert.Equal(t, "arn:aws:iam::"+e.accountID+":user/alice", cbXML(rec.Body.String(), "Arn"))
+	assert.Equal(t, e.aliceID, cbXML(rec.Body.String(), "UserId"))
+
+	_, err := e.iam.UpdateAccessKey(e.accountID, &iam.UpdateAccessKeyInput{
+		UserName: aws.String("alice"), AccessKeyId: aws.String(mintKey.akid), Status: aws.String(handlers_iam.AccessKeyStatusInactive),
+	})
+	require.NoError(t, err)
+	_, err = e.iam.DeleteAccessKey(e.accountID, &iam.DeleteAccessKeyInput{
+		UserName: aws.String("alice"), AccessKeyId: aws.String(mintKey.akid),
+	})
+	require.NoError(t, err)
+
+	// CURRENT: the session record carries no reference to the minting key, so
+	// deactivating and deleting that key leaves the session authenticating.
+	assert.Equal(t, http.StatusOK, e.callerIdentity(t, s).Code)
+
+	// A session cannot mint a further session.
+	assert.Equal(t, cbRegistry(awserrors.ErrorAccessDenied).code,
+		cbOutcomeOf(e.request(t, e.h, s, "sts", cbForm("GetSessionToken"))).code)
+}
+
+// Pins CURRENT behaviour: an AssumeRoleWithWebIdentity session authenticates at
+// the SigV4 door exactly as an AssumeRole session does.
+func TestCurrentBehaviour_Session_WebIdentityCredentials(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	issuer := testDiscIssuer
+
+	iamKV, err := kvutil.GetOrCreateBucket(t.Context(), e.js,
+		handlers_iam.IAMAccountBucketName(e.accountID), handlers_iam.KVBucketIAMAccountVersion)
+	require.NoError(t, err)
+	_, err = iamKV.Put(t.Context(), handlers_iam.OIDCProviderKey(issuer), []byte(`{"registered":true}`))
+	require.NoError(t, err)
+
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	point, err := priv.PublicKey.Bytes()
+	require.NoError(t, err)
+	jwks, err := json.Marshal(handlers_sts.JWKS{Keys: []handlers_sts.JWK{{
+		Kty: "EC", Kid: "k1", Alg: "ES256", Use: "sig", Crv: "P-256",
+		X: base64.RawURLEncoding.EncodeToString(point[1:33]),
+		Y: base64.RawURLEncoding.EncodeToString(point[33:65]),
+	}}})
+	require.NoError(t, err)
+	eksKV, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), e.js, e.accountID)
+	require.NoError(t, err)
+	_, err = eksKV.Put(t.Context(), handlers_eks.OIDCJWKSKey(testDiscCluster), jwks)
+	require.NoError(t, err)
+
+	federated := handlers_iam.OIDCProviderARN(e.accountID, strings.TrimPrefix(issuer, "https://"))
+	role, err := e.iam.CreateRole(e.accountID, &iam.CreateRoleInput{
+		RoleName: aws.String("irsa"),
+		AssumeRolePolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"` +
+			federated + `"},"Action":"sts:AssumeRoleWithWebIdentity"}]}`),
+	})
+	require.NoError(t, err)
+
+	now := time.Now().UTC()
+	tok := jwt.NewWithClaims(jwt.SigningMethodES256, jwt.MapClaims{
+		"iss": issuer, "sub": "system:serviceaccount:default:sa", "aud": []string{"sts.amazonaws.com"},
+		"exp": now.Add(10 * time.Minute).Unix(), "iat": now.Unix(),
+	})
+	tok.Header["kid"] = "k1"
+	signed, err := tok.SignedString(priv)
+	require.NoError(t, err)
+
+	// Unsigned: the anonymous STS interceptor serves it ahead of SigV4.
+	s := cbSessionFrom(t, e.request(t, e.h, cbCreds{}, "sts", cbForm("AssumeRoleWithWebIdentity",
+		"RoleArn", aws.StringValue(role.Role.Arn), "RoleSessionName", "pod", "WebIdentityToken", signed)))
+
+	rec := e.callerIdentity(t, s)
+	require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+	assert.Equal(t, "arn:aws:sts::"+e.accountID+":assumed-role/irsa/pod", cbXML(rec.Body.String(), "Arn"))
+	assert.Equal(t, cbRegistry(awserrors.ErrorInvalidClientTokenId),
+		cbOutcomeOf(e.callerIdentity(t, cbCreds{akid: s.akid, secret: s.secret, token: tamper(s.token)})))
+}
+
+// Pins CURRENT behaviour: deleting a user and recreating it under the same name
+// strands its old key and sessions, by revocation or by the continuity check.
+func TestCurrentBehaviour_Revocation_UserDeletedAndRecreated(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		noRevoker   bool
+		storedAfter bool
+	}{
+		{name: "revoker_installed", noRevoker: false, storedAfter: false},
+		{name: "revocation_missed", noRevoker: true, storedAfter: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			e := newCBEnv(t, cbEnvOpts{noRevoker: tc.noRevoker})
+			oldKey, oldID := e.newUser(t, "bob")
+			session := e.getSessionToken(t, oldKey)
+			require.Equal(t, http.StatusOK, e.callerIdentity(t, session).Code)
+
+			_, err := e.iam.DeleteAccessKey(e.accountID, &iam.DeleteAccessKeyInput{
+				UserName: aws.String("bob"), AccessKeyId: aws.String(oldKey.akid),
+			})
+			require.NoError(t, err)
+			_, err = e.iam.DeleteUser(e.accountID, &iam.DeleteUserInput{UserName: aws.String("bob")})
+			require.NoError(t, err)
+			assert.Equal(t, tc.storedAfter, e.sessionStored(t, session.akid))
+
+			invalid := cbRegistry(awserrors.ErrorInvalidClientTokenId)
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, session)), "session after delete")
+
+			newKey, newID := e.newUser(t, "bob")
+			require.NotEqual(t, oldID, newID)
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, oldKey)), "old key after recreate")
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, session)), "old session after recreate")
+
+			rec := e.callerIdentity(t, newKey)
+			require.Equal(t, http.StatusOK, rec.Code)
+			assert.Equal(t, newID, cbXML(rec.Body.String(), "UserId"))
+		})
+	}
+}
+
+// Pins CURRENT behaviour: deleting a role and recreating it under the same name
+// strands its old sessions, by revocation or by the continuity check.
+func TestCurrentBehaviour_Revocation_RoleDeletedAndRecreated(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		noRevoker   bool
+		storedAfter bool
+	}{
+		{name: "revoker_installed", noRevoker: false, storedAfter: false},
+		{name: "revocation_missed", noRevoker: true, storedAfter: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			e := newCBEnv(t, cbEnvOpts{noRevoker: tc.noRevoker})
+			roleARN, oldRoleID := e.createAssumableRole(t, "app")
+			s := e.assumeRole(t, roleARN, "s1")
+			require.Equal(t, http.StatusOK, e.callerIdentity(t, s).Code)
+
+			_, err := e.iam.DeleteRole(e.accountID, &iam.DeleteRoleInput{RoleName: aws.String("app")})
+			require.NoError(t, err)
+			assert.Equal(t, tc.storedAfter, e.sessionStored(t, s.akid))
+
+			invalid := cbRegistry(awserrors.ErrorInvalidClientTokenId)
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, s)), "session after delete")
+
+			newARN, newRoleID := e.createAssumableRole(t, "app")
+			require.Equal(t, roleARN, newARN)
+			require.NotEqual(t, oldRoleID, newRoleID)
+			assert.Equal(t, invalid, cbOutcomeOf(e.callerIdentity(t, s)), "old session after recreate")
+			assert.Equal(t, http.StatusOK, e.callerIdentity(t, e.assumeRole(t, newARN, "s2")).Code)
+		})
+	}
+}
+
+// Pins CURRENT behaviour: a long-lived key is bound to a user by name only, so
+// a key record orphaned from its user authenticates as a same-name replacement.
+func TestCurrentBehaviour_Revocation_OrphanedKeyAuthenticatesAsRecreatedUser(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	key, oldID := e.newUser(t, "erin")
+
+	e.orphanKeys(t, "erin")
+
+	// CURRENT: the orphaned key still authenticates, and nothing blocks DeleteUser.
+	require.Equal(t, http.StatusOK, e.callerIdentity(t, key).Code)
+	_, err := e.iam.DeleteUser(e.accountID, &iam.DeleteUserInput{UserName: aws.String("erin")})
+	require.NoError(t, err)
+
+	// CURRENT: with the user gone, the key authenticates and GetCallerIdentity
+	// fails on the missing record rather than at the door.
+	gone := cbOutcomeOf(e.callerIdentity(t, key))
+	assert.Equal(t, http.StatusNotFound, gone.status)
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, gone.code)
+
+	// CURRENT: a new user with the same name inherits the orphaned key; AWS binds
+	// keys to the user's unique ID, so a replacement would not (unverified).
+	_, err = e.iam.CreateUser(e.accountID, &iam.CreateUserInput{UserName: aws.String("erin")})
+	require.NoError(t, err)
+	rec := e.callerIdentity(t, key)
+	require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+	assert.NotEqual(t, oldID, cbXML(rec.Body.String(), "UserId"))
+	assert.Equal(t, "arn:aws:iam::"+e.accountID+":user/erin", cbXML(rec.Body.String(), "Arn"))
+}
+
+// orphanKeys rewrites a user record to list no access keys while leaving the
+// access-key records in place, the state an interrupted DeleteAccessKey leaves.
+func (e *cbEnv) orphanKeys(t *testing.T, user string) {
+	t.Helper()
+	usersKV, err := e.js.KeyValue(t.Context(), handlers_iam.KVBucketUsers)
+	require.NoError(t, err)
+	entry, err := usersKV.Get(t.Context(), e.accountID+"."+user)
+	require.NoError(t, err)
+	var u handlers_iam.User
+	require.NoError(t, json.Unmarshal(entry.Value(), &u))
+	u.AccessKeys = []string{}
+	data, err := json.Marshal(u)
+	require.NoError(t, err)
+	_, err = usersKV.Put(t.Context(), e.accountID+"."+user, data)
+	require.NoError(t, err)
+}
+
+// Pins CURRENT behaviour: whose identity policy the policy check applies to an
+// orphaned key once its user is deleted and recreated under the same name.
+func TestCurrentBehaviour_Revocation_OrphanedKeyPermissionsAfterRecreation(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	key, oldID := e.newUser(t, "frank")
+	listUsers := cbForm("ListUsers")
+	getUser := cbForm("GetUser", "UserName", "frank")
+
+	// Policy A: ListUsers only.
+	e.putUserPolicy(t, "frank", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:ListUsers","Resource":"*"}]}`)
+	require.Equal(t, http.StatusOK, e.request(t, e.h, key, "iam", listUsers).Code, "A allows ListUsers")
+	require.Equal(t, awserrors.ErrorAccessDenied, cbOutcomeOf(e.request(t, e.h, key, "iam", getUser)).code, "A denies GetUser")
+
+	e.orphanKeys(t, "frank")
+	_, err := e.iam.DeleteUserPolicy(e.accountID, &iam.DeleteUserPolicyInput{UserName: aws.String("frank"), PolicyName: aws.String("p")})
+	require.NoError(t, err)
+	_, err = e.iam.DeleteUser(e.accountID, &iam.DeleteUserInput{UserName: aws.String("frank")})
+	require.NoError(t, err)
+
+	// Policy B, on the replacement user: GetUser only.
+	u, err := e.iam.CreateUser(e.accountID, &iam.CreateUserInput{UserName: aws.String("frank")})
+	require.NoError(t, err)
+	require.NotEqual(t, oldID, aws.StringValue(u.User.UserId))
+	e.putUserPolicy(t, "frank", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:GetUser","Resource":"*"}]}`)
+
+	// CURRENT: the orphaned key is evaluated against the replacement user's
+	// policy B, so it gains GetUser and loses the ListUsers that A granted.
+	rec := e.request(t, e.h, key, "iam", getUser)
+	assert.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+	assert.Equal(t, awserrors.ErrorAccessDenied, cbOutcomeOf(e.request(t, e.h, key, "iam", listUsers)).code)
+}
+
+// Pins CURRENT behaviour: unsigned requests are refused with
+// MissingAuthenticationToken unless the action is anonymous STS.
+func TestCurrentBehaviour_Auth_MissingCredentials(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	missing := cbRegistry(awserrors.ErrorMissingAuthenticationToken)
+	for _, f := range []url.Values{
+		cbForm("ListUsers"),
+		cbForm("GetCallerIdentity"),
+		cbForm("GetSessionToken"),
+		cbForm("AssumeRole", "RoleArn", "arn:aws:iam::"+e.accountID+":role/x", "RoleSessionName", "s"),
+	} {
+		t.Run(f.Get("Action"), func(t *testing.T) {
+			assert.Equal(t, missing, cbOutcomeOf(e.request(t, e.h, cbCreds{}, "", f)))
+		})
+	}
+
+	// Session token alone, with no Authorization header.
+	t.Run("token_without_signature", func(t *testing.T) {
+		assert.Equal(t, missing, cbOutcomeOf(e.request(t, e.h, cbCreds{token: "anything"}, "", cbForm("GetCallerIdentity"))))
+	})
+
+	// The anonymous path reaches STS itself: its verdict, not the door's.
+	t.Run("AssumeRoleWithWebIdentity", func(t *testing.T) {
+		got := cbOutcomeOf(e.request(t, e.h, cbCreds{}, "", cbForm("AssumeRoleWithWebIdentity",
+			"RoleArn", "arn:aws:iam::"+e.accountID+":role/x", "RoleSessionName", "s1", "WebIdentityToken", "not-a-jwt")))
+		assert.Equal(t, http.StatusBadRequest, got.status)
+		assert.Equal(t, awserrors.ErrorInvalidIdentityToken, got.code)
+	})
+}
+
+// Pins CURRENT behaviour: credential lookup outcomes when IAM or STS backing
+// state cannot be read.
+func TestCurrentBehaviour_Auth_BackingStateUnavailable(t *testing.T) {
+	gci := cbForm("GetCallerIdentity")
+	internal := cbRegistry(awserrors.ErrorInternalError)
+
+	t.Run("gateway_connection_closed", func(t *testing.T) {
+		e := newCBEnv(t, cbEnvOpts{})
+		roleARN, _ := e.createAssumableRole(t, "app")
+		s := e.assumeRole(t, roleARN, "s1")
+		e.nc.Close()
+		want := cbOutcome{status: http.StatusServiceUnavailable, code: awserrors.ErrorServiceUnavailable, message: clusterUnavailableMsg}
+		for name, c := range map[string]cbCreds{"long_lived": e.alice, "session": s, "unknown": {akid: cbUnknownKey, secret: cbBadSecret}} {
+			assert.Equal(t, want, cbOutcomeOf(e.request(t, e.h, c, "sts", gci)), name)
+		}
+	})
+
+	t.Run("iam_connection_closed", func(t *testing.T) {
+		e := newCBEnv(t, cbEnvOpts{separateIAMConn: true})
+		roleARN, _ := e.createAssumableRole(t, "app")
+		s := e.assumeRole(t, roleARN, "s1")
+		e.iamNC.Close()
+		for name, c := range map[string]cbCreds{"long_lived": e.alice, "session": s, "unknown": {akid: cbUnknownKey, secret: cbBadSecret}} {
+			assert.Equal(t, internal, cbOutcomeOf(e.request(t, e.h, c, "sts", gci)), name)
+		}
+	})
+
+	t.Run("access_key_bucket_deleted", func(t *testing.T) {
+		e := newCBEnv(t, cbEnvOpts{})
+		require.NoError(t, e.js.DeleteKeyValue(t.Context(), handlers_iam.KVBucketAccessKeys))
+		for name, c := range map[string]cbCreds{"known": e.alice, "unknown": {akid: cbUnknownKey, secret: cbBadSecret}} {
+			assert.Equal(t, internal, cbOutcomeOf(e.request(t, e.h, c, "sts", gci)), name)
+		}
+	})
+
+	t.Run("session_bucket_deleted", func(t *testing.T) {
+		e := newCBEnv(t, cbEnvOpts{})
+		roleARN, _ := e.createAssumableRole(t, "app")
+		s := e.assumeRole(t, roleARN, "s1")
+		require.NoError(t, e.js.DeleteKeyValue(t.Context(), handlers_sts.KVBucketSessionCredentials))
+		assert.Equal(t, internal, cbOutcomeOf(e.callerIdentity(t, s)))
+	})
+
+	t.Run("accounts_bucket_deleted", func(t *testing.T) {
+		e := newCBEnv(t, cbEnvOpts{})
+		require.NoError(t, e.js.DeleteKeyValue(t.Context(), handlers_iam.KVBucketAccounts))
+		// Fails closed only once the signature holds; a bad one still reads as such.
+		assert.Equal(t, internal, cbOutcomeOf(e.callerIdentity(t, e.alice)))
+		assert.Equal(t, cbRegistry(awserrors.ErrorSignatureDoesNotMatch),
+			cbOutcomeOf(e.callerIdentity(t, cbCreds{akid: e.alice.akid, secret: cbBadSecret})))
+	})
+}
+
+// Pins CURRENT behaviour: identity-policy allow, implicit deny and explicit deny
+// on iam actions, evaluated after authentication and before the handler runs.
+func TestCurrentBehaviour_Policy_IdentityPolicyOutcomes(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	listUsers := cbForm("ListUsers")
+	denied := cbOutcome{status: http.StatusForbidden, code: awserrors.ErrorAccessDenied,
+		message: awserrors.LookupErrorMessage("iam", awserrors.ErrorAccessDenied).Message}
+
+	t.Run("implicit_deny", func(t *testing.T) {
+		assert.Equal(t, denied, cbOutcomeOf(e.request(t, e.h, e.alice, "iam", listUsers)))
+	})
+
+	t.Run("allow", func(t *testing.T) {
+		e.putUserPolicy(t, "alice", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"iam:ListUsers","Resource":"*"}]}`)
+		rec := e.request(t, e.h, e.alice, "iam", listUsers)
+		require.Equalf(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
+		assert.Contains(t, rec.Body.String(), "<UserName>alice</UserName>")
+	})
+
+	t.Run("explicit_deny_blocks_before_dispatch", func(t *testing.T) {
+		e.putUserPolicy(t, "alice", `{"Version":"2012-10-17","Statement":[`+
+			`{"Effect":"Allow","Action":"iam:*","Resource":"*"},`+
+			`{"Effect":"Deny","Action":"iam:CreateUser","Resource":"*"}]}`)
+		assert.Equal(t, denied, cbOutcomeOf(e.request(t, e.h, e.alice, "iam", cbForm("CreateUser", "UserName", "mallory"))))
+		_, err := e.iam.GetUser(e.accountID, &iam.GetUserInput{UserName: aws.String("mallory")})
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity, "denied CreateUser must not have run")
+		assert.Equal(t, http.StatusOK, e.request(t, e.h, e.alice, "iam", listUsers).Code)
+	})
+
+	// CURRENT: the root-name policy bypass is limited to the global account, so a
+	// tenant user named "root" with no policy is implicitly denied.
+	t.Run("tenant_user_named_root_implicit_deny", func(t *testing.T) {
+		creds, _ := e.newUser(t, "root")
+		assert.Equal(t, denied, cbOutcomeOf(e.request(t, e.h, creds, "iam", listUsers)))
+	})
+}
+
+// Pins CURRENT behaviour: which credential failures a caller can tell apart on
+// the wire, and whether the door decrypted a secret before answering.
+func TestCurrentBehaviour_AntiEnumeration_RejectionShapes(t *testing.T) {
+	e := newCBEnv(t, cbEnvOpts{})
+	bobKey, _ := e.newUser(t, "bob")
+	_, err := e.iam.UpdateAccessKey(e.accountID, &iam.UpdateAccessKeyInput{
+		UserName: aws.String("bob"), AccessKeyId: aws.String(bobKey.akid), Status: aws.String(handlers_iam.AccessKeyStatusInactive),
+	})
+	require.NoError(t, err)
+	carolKey, _ := e.newUser(t, "carol")
+	_, err = e.iam.DeleteAccessKey(e.accountID, &iam.DeleteAccessKeyInput{UserName: aws.String("carol"), AccessKeyId: aws.String(carolKey.akid)})
+	require.NoError(t, err)
+	live := e.getSessionToken(t, e.alice)
+	expired := e.getSessionToken(t, e.alice)
+	e.expireSession(t, expired.akid)
+
+	invalid := cbRegistry(awserrors.ErrorInvalidClientTokenId)
+	sigMismatch := cbRegistry(awserrors.ErrorSignatureDoesNotMatch)
+	for _, tc := range []struct {
+		name     string
+		creds    cbCreds
+		want     cbOutcome
+		decrypts int32
+	}{
+		{"unknown_key", cbCreds{akid: cbUnknownKey, secret: cbBadSecret}, invalid, 0},
+		{"unknown_prefix", cbCreds{akid: "AKIBCBUNKNOWNPREFIXA", secret: cbBadSecret}, invalid, 0},
+		{"deleted_key_right_secret", carolKey, invalid, 0},
+		{"inactive_key_right_secret", bobKey, invalid, 0},
+		// CURRENT: an inactive key answers InvalidClientTokenId without a
+		// signature check, while an active key with a bad signature answers
+		// SignatureDoesNotMatch, so a caller with no secret can tell the two apart.
+		{"inactive_key_bad_signature", cbCreds{akid: bobKey.akid, secret: cbBadSecret}, invalid, 0},
+		{"active_key_bad_signature", cbCreds{akid: e.alice.akid, secret: cbBadSecret}, sigMismatch, 1},
+		{"unknown_session", cbCreds{akid: cbUnknownSes, secret: cbBadSecret, token: live.token}, invalid, 0},
+		{"session_no_token", cbCreds{akid: live.akid, secret: cbBadSecret}, invalid, 0},
+		{"session_wrong_token", cbCreds{akid: live.akid, secret: cbBadSecret, token: tamper(live.token)}, invalid, 0},
+		{"session_expired_no_token", cbCreds{akid: expired.akid, secret: cbBadSecret}, cbRegistry(awserrors.ErrorExpiredToken), 0},
+		{"session_right_token_bad_signature", cbCreds{akid: live.akid, secret: cbBadSecret, token: live.token}, sigMismatch, 1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			e.counter.decrypts.Store(0)
+			assert.Equal(t, tc.want, cbOutcomeOf(e.callerIdentity(t, tc.creds)))
+			assert.Equal(t, tc.decrypts, e.counter.decrypts.Load(), "secret decrypts before the verdict")
+		})
+	}
+}
+
+// Pins CURRENT behaviour: the door caches only an account's ACTIVE status, per
+// gateway, for accountActiveTTL; keys and sessions are re-read on every request.
+func TestCurrentBehaviour_Cache_AccountStatusIsTheOnlyAuthCache(t *testing.T) {
+	require.Equal(t, 5*time.Second, accountActiveTTL)
+
+	for _, status := range []string{handlers_iam.AccountStatusSuspended, handlers_iam.AccountStatusTerminating} {
+		t.Run(status, func(t *testing.T) {
+			e := newCBEnv(t, cbEnvOpts{})
+			require.Equal(t, http.StatusOK, e.callerIdentity(t, e.alice).Code)
+			start := time.Now()
+			_, err := e.iam.SetAccountStatus(e.accountID, status)
+			require.NoError(t, err)
+
+			// CURRENT: an out-of-band status write is not seen by a gateway
+			// holding a warm ACTIVE entry until that entry expires.
+			warm := e.callerIdentity(t, e.alice)
+			if time.Since(start) >= accountActiveTTL {
+				t.Skip("cache window elapsed before the warm request; outcome not attributable")
+			}
+			assert.Equal(t, http.StatusOK, warm.Code)
+
+			denied := cbOutcome{status: http.StatusForbidden, code: awserrors.ErrorAccessDenied,
+				message: awserrors.ErrorLookup[awserrors.ErrorAccessDenied].Message}
+			assert.Equal(t, denied, cbOutcomeOf(e.request(t, e.gateway(), e.alice, "sts", cbForm("GetCallerIdentity"))))
+		})
+	}
+}

spinifex/gateway/ecr_principal_test.go

@@ -285,6 +285,18 @@ func TestResolveECRPrincipal_LongLivedUser_Rejections(t *testing.T) {
 		assert.False(t, isECRDependencyFailure(err))
 	})
 
+	// CURRENT: a registry token is rehydrated against the account record, so a
+	// terminating account's already-issued token is rejected as invalid.
+	t.Run("terminating account", func(t *testing.T) {
+		iamSvc := newECRMockIAMService()
+		seedECRTestUser(iamSvc, ecrPrincipalTestAccount, "dev", ecrPrincipalTestAKID)
+		iamSvc.accounts[ecrPrincipalTestAccount].Status = handlers_iam.AccountStatusTerminating
+		gw := &GatewayConfig{IAMService: iamSvc}
+		_, err := gw.resolveECRPrincipal(validClaims())
+		require.Error(t, err)
+		assert.False(t, isECRDependencyFailure(err))
+	})
+
 	t.Run("principalType claim mismatch", func(t *testing.T) {
 		iamSvc := newECRMockIAMService()
 		seedECRTestUser(iamSvc, ecrPrincipalTestAccount, "dev", ecrPrincipalTestAKID)

spinifex/handlers/iam/account_status_reader.go

@@ -0,0 +1,56 @@
+package handlers_iam
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// AccountStatusReader reads account status from spinifex-accounts and never
+// writes. It is the read-only capability EC2's settings seal uses to check
+// its TERMINATING precondition.
+type AccountStatusReader struct {
+	accounts *kvstore.Bucket // AttachOnly: never creates the bucket
+}
+
+// NewAccountStatusReader attaches to the existing spinifex-accounts bucket.
+// It needs no master key and runs no migration: a caller that only reads
+// account status must not bootstrap IAM's bucket out from under it.
+func NewAccountStatusReader(js jetstream.JetStream) *AccountStatusReader {
+	return &AccountStatusReader{
+		accounts: kvstore.NewBucket(js, kvstore.Config{
+			Name:       KVBucketAccounts,
+			Missing:    "account status reader: nil jetstream client",
+			AttachOnly: true,
+		}),
+	}
+}
+
+// AccountTerminating is a direct read, which suffices because TERMINATING is
+// one-way until the record is deleted. A missing record is (false, nil); only
+// a failed read or decode is an error.
+func (r *AccountStatusReader) AccountTerminating(ctx context.Context, accountID string) (bool, error) {
+	kv, err := r.accounts.KV(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	entry, err := kv.Get(ctx, accountID)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	var record struct {
+		Status string `json:"status"`
+	}
+	if err := json.Unmarshal(entry.Value(), &record); err != nil {
+		return false, err
+	}
+	return record.Status == AccountStatusTerminating, nil
+}

spinifex/handlers/iam/account_status_reader_test.go

@@ -0,0 +1,102 @@
+package handlers_iam_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newStatusReaderFixture starts a real IAM service (which creates and
+// migrates spinifex-accounts) and a reader attached to the same JetStream.
+func newStatusReaderFixture(t *testing.T) (handlers_iam.IAMService, *handlers_iam.AccountStatusReader, jetstream.JetStream) {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+
+	return svc, handlers_iam.NewAccountStatusReader(js), js
+}
+
+func TestAccountStatusReader_TerminatingIsTrue(t *testing.T) {
+	t.Parallel()
+	svc, reader, _ := newStatusReaderFixture(t)
+	account, err := svc.CreateAccount("tenant@example.com")
+	require.NoError(t, err)
+	_, err = svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusTerminating)
+	require.NoError(t, err)
+
+	terminating, err := reader.AccountTerminating(t.Context(), account.AccountID)
+	require.NoError(t, err)
+	assert.True(t, terminating)
+}
+
+func TestAccountStatusReader_ActiveAndSuspendedAreFalse(t *testing.T) {
+	t.Parallel()
+	svc, reader, _ := newStatusReaderFixture(t)
+	active, err := svc.CreateAccount("active@example.com")
+	require.NoError(t, err)
+	terminating, err := reader.AccountTerminating(t.Context(), active.AccountID)
+	require.NoError(t, err)
+	assert.False(t, terminating)
+
+	suspended, err := svc.CreateAccount("suspended@example.com")
+	require.NoError(t, err)
+	_, err = svc.SetAccountStatus(suspended.AccountID, handlers_iam.AccountStatusSuspended)
+	require.NoError(t, err)
+	terminating, err = reader.AccountTerminating(t.Context(), suspended.AccountID)
+	require.NoError(t, err)
+	assert.False(t, terminating)
+}
+
+func TestAccountStatusReader_MissingRecordIsFalseNotError(t *testing.T) {
+	t.Parallel()
+	_, reader, _ := newStatusReaderFixture(t)
+	terminating, err := reader.AccountTerminating(t.Context(), "999999999999")
+	require.NoError(t, err)
+	assert.False(t, terminating)
+}
+
+// A missing bucket is an error, and the reader never creates one: a caller
+// that only reads account status must not bootstrap IAM's bucket.
+func TestAccountStatusReader_MissingBucketIsErrorAndNeverCreated(t *testing.T) {
+	t.Parallel()
+	_, nc, js := testutil.StartTestJetStream(t)
+	reader := handlers_iam.NewAccountStatusReader(js)
+
+	_, err := reader.AccountTerminating(t.Context(), "111111111111")
+	require.Error(t, err)
+
+	_, bucketErr := js.KeyValue(t.Context(), handlers_iam.KVBucketAccounts)
+	assert.ErrorIs(t, bucketErr, jetstream.ErrBucketNotFound)
+	_ = nc
+}
+
+func TestAccountStatusReader_UndecodableValueIsError(t *testing.T) {
+	t.Parallel()
+	_, nc, js := testutil.StartTestJetStream(t)
+	_, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, mustMasterKey(t))
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), handlers_iam.KVBucketAccounts)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), "222222222222", []byte(`{"status":`))
+	require.NoError(t, err)
+
+	reader := handlers_iam.NewAccountStatusReader(js)
+	_, err = reader.AccountTerminating(t.Context(), "222222222222")
+	require.Error(t, err)
+}
+
+func mustMasterKey(t *testing.T) []byte {
+	t.Helper()
+	key, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	return key
+}

spinifex/operator/cli/admin_account_delete.go

@@ -11,6 +11,7 @@ import (
 	"slices"
 	"strings"
 	"time"
+	"uuid"
 
 	"github.com/mulgadc/spinifex/spinifex/accountteardown"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -101,13 +102,7 @@ func runAccountDelete(cmd *cobra.Command, args []string) error {
 	}
 
 	if dryRun {
-		result, err := engine.Inventory(ctx, accountID)
-		if err != nil {
-			return err
-		}
-		printTeardownPlan(result)
-		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount())
-		return nil
+		return runAccountDryRun(ctx, engine, accountID)
 	}
 
 	// Always show the inventory before destroying it. An operator who has not
@@ -149,6 +144,25 @@ func runAccountDelete(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+// accountDryRunner is the engine's explicit dry run.
+type accountDryRunner interface {
+	DryRun(ctx context.Context, accountID, correlationRef string) (*accountteardown.Result, error)
+}
+
+var _ accountDryRunner = (*accountteardown.Engine)(nil)
+
+// runAccountDryRun prints what a teardown would remove. Each run gets a fresh
+// correlation reference, which only ties it to its log records.
+func runAccountDryRun(ctx context.Context, engine accountDryRunner, accountID string) error {
+	result, err := engine.DryRun(ctx, accountID, uuid.NewV4().String())
+	if err != nil {
+		return err
+	}
+	printTeardownPlan(result)
+	fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount())
+	return nil
+}
+
 // promptAccountName asks for the confirmation name on a terminal.
 func promptAccountName(accountID string) (string, error) {
 	fmt.Printf("\nThis permanently deletes account %s and everything above.\n", accountID)

spinifex/operator/cli/admin_account_delete_output_test.go

@@ -2,9 +2,11 @@ package cli_test
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"os"
 	"testing"
+	"uuid"
 
 	"github.com/mulgadc/spinifex/spinifex/accountteardown"
 	"github.com/mulgadc/spinifex/spinifex/operator/cli"
@@ -160,3 +162,47 @@ func TestPromptAccountNameTreatsAnEmptyReplyAsCancelled(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "cancelled")
 }
+
+// dryRunEngine stands in for the teardown engine's explicit dry run and keeps
+// the correlation reference each run was given.
+type dryRunEngine struct {
+	result *accountteardown.Result
+	err    error
+	refs   []string
+}
+
+func (e *dryRunEngine) DryRun(_ context.Context, _, correlationRef string) (*accountteardown.Result, error) {
+	e.refs = append(e.refs, correlationRef)
+	return e.result, e.err
+}
+
+// The dry run prints what it always printed, and each run hands the engine a
+// fresh UUID to log its shadow comparison under.
+func TestRunAccountDryRunPrintsThePlanUnderAFreshCorrelationRef(t *testing.T) {
+	engine := &dryRunEngine{result: teardownPlan()}
+
+	var err error
+	output := captureOutput(t, func() { err = cli.RunAccountDryRun(t.Context(), engine, "000000000042") })
+	require.NoError(t, err)
+	assert.Equal(t, "Account 000000000042 (tenant@example.com)\n\n"+
+		"  compute\n    instance i-1 (running)\n"+
+		"  storage\n    volume vol-1\n"+
+		"\nDry run: nothing was deleted. 2 resources would be removed.\n", output)
+
+	captureOutput(t, func() { err = cli.RunAccountDryRun(t.Context(), engine, "000000000042") })
+	require.NoError(t, err)
+	require.Len(t, engine.refs, 2)
+	for _, ref := range engine.refs {
+		_, parseErr := uuid.Parse(ref)
+		require.NoError(t, parseErr, ref)
+	}
+	assert.NotEqual(t, engine.refs[0], engine.refs[1])
+}
+
+func TestRunAccountDryRunReturnsTheEngineError(t *testing.T) {
+	engine := &dryRunEngine{err: accountteardown.ErrProtectedAccount}
+	var err error
+	output := captureOutput(t, func() { err = cli.RunAccountDryRun(t.Context(), engine, "000000000001") })
+	require.ErrorIs(t, err, accountteardown.ErrProtectedAccount)
+	assert.Empty(t, output)
+}

spinifex/operator/cli/admin_kv_compare.go

@@ -32,6 +32,12 @@ A replica that is merely behind is not divergent, so only these fail:
   - replicas disagree on the replica count;
   - no sequence range is common to every replica, so nothing can be checked.
 
+A stream with a max_age whose replicas, copied at different times, share no
+sequence range is reported as skipped (volatile) rather than diverged: a short
+lived bucket such as a leader lease turns over between two copies. Sequences
+two of its replicas do share must still hold the same content, and each
+message a replica lacks must be superseded, expired, or written after its copy.
+
 Sequences past the slowest replica's last one are not compared. Digests are
 taken from live stores a few seconds apart, so re-run before trusting a
 failure that could be a write landing between two copies.
@@ -110,6 +116,8 @@ type streamVerdict struct {
 	Holders  []string
 	Problems []string
 	Notes    []string
+	// Volatile says why a stream that turned over between copies was not compared.
+	Volatile string
 }
 
 type kvCompareResult struct {
@@ -228,6 +236,16 @@ func compareStream(name string, hosts []string, views []*replicaView) streamVerd
 		}
 	}
 	if lo > hi {
+		maxAge, spread := views[0].d.MaxAge, copySpread(views)
+		if maxAge > 0 && spread > 0 {
+			mismatches, unexplained := turnoverFindings(views)
+			v.Problems = append(v.Problems, mismatches...)
+			if len(unexplained) == 0 {
+				v.Volatile = fmt.Sprintf("volatile: max_age %s and copies %s apart share no sequence range, so it turned over between them; not compared", maxAge, spread)
+				return v
+			}
+			v.Problems = append(v.Problems, unexplained...)
+		}
 		v.Problems = append(v.Problems, "no sequence range is common to every replica, so nothing could be compared")
 		return v
 	}
@@ -253,18 +271,8 @@ func compareStream(name string, hosts []string, views []*replicaView) streamVerd
 		if len(present) == 0 {
 			continue
 		}
-		byHash := map[string][]string{}
-		for _, r := range present {
-			h := r.bySeq[seq].Hash
-			byHash[h] = append(byHash[h], r.host)
-		}
-		if len(byHash) > 1 {
-			var parts []string
-			for h, hs := range byHash {
-				parts = append(parts, strings.Join(hs, ",")+"="+h)
-			}
-			sort.Strings(parts)
-			problems = append(problems, fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " ")))
+		if p := contentMismatch(seq, present); p != "" {
+			problems = append(problems, p)
 			continue
 		}
 		msg := present[0].bySeq[seq]
@@ -277,10 +285,7 @@ func compareStream(name string, hosts []string, views []*replicaView) streamVerd
 				seq, msg.Subject, hostsOf(present), r.host))
 		}
 	}
-	if len(problems) > maxFindingsPerStream {
-		problems = append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
-	}
-	v.Problems = append(v.Problems, problems...)
+	v.Problems = append(v.Problems, capFindings(problems)...)
 
 	note := fmt.Sprintf("seqs %d-%d compared", lo, hi)
 	if explained > 0 {
@@ -299,6 +304,84 @@ func compareStream(name string, hosts []string, views []*replicaView) streamVerd
 	return v
 }
 
+// contentMismatch describes seq holding different content on the replicas
+// present at it, or returns "" when they agree.
+func contentMismatch(seq uint64, present []*replicaView) string {
+	byHash := map[string][]string{}
+	for _, r := range present {
+		h := r.bySeq[seq].Hash
+		byHash[h] = append(byHash[h], r.host)
+	}
+	if len(byHash) < 2 {
+		return ""
+	}
+	var parts []string
+	for h, hs := range byHash {
+		parts = append(parts, strings.Join(hs, ",")+"="+h)
+	}
+	sort.Strings(parts)
+	return fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " "))
+}
+
+// turnoverFindings compares every sequence held by two or more replicas, since
+// no timing explains two values at one sequence, and lists each absence that
+// neither absenceExplained nor a write after that replica's copy accounts for.
+func turnoverFindings(views []*replicaView) (mismatches, unexplained []string) {
+	seqSet := map[uint64]struct{}{}
+	for _, r := range views {
+		for s := range r.bySeq {
+			seqSet[s] = struct{}{}
+		}
+	}
+	for _, seq := range slices.Sorted(maps.Keys(seqSet)) {
+		var present, absent []*replicaView
+		for _, r := range views {
+			if e, ok := r.bySeq[seq]; ok && e.Hash != deletedHash {
+				present = append(present, r)
+			} else {
+				absent = append(absent, r)
+			}
+		}
+		if len(present) == 0 {
+			continue
+		}
+		if p := contentMismatch(seq, present); p != "" {
+			mismatches = append(mismatches, p)
+			continue
+		}
+		msg := present[0].bySeq[seq]
+		for _, r := range absent {
+			if absenceExplained(r, msg) || (seq > r.d.LastSeq && msg.Time.After(r.copiedAt)) {
+				continue
+			}
+			unexplained = append(unexplained, fmt.Sprintf("seq %d %s is on %s but missing from %s, with no later write to that subject, expiry, limit or copy order to explain it",
+				seq, msg.Subject, hostsOf(present), r.host))
+		}
+	}
+	return capFindings(mismatches), capFindings(unexplained)
+}
+
+func capFindings(problems []string) []string {
+	if len(problems) > maxFindingsPerStream {
+		return append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
+	}
+	return problems
+}
+
+// copySpread is how far apart the earliest and latest replica copies were taken.
+func copySpread(views []*replicaView) time.Duration {
+	first, last := views[0].copiedAt, views[0].copiedAt
+	for _, r := range views[1:] {
+		if r.copiedAt.Before(first) {
+			first = r.copiedAt
+		}
+		if r.copiedAt.After(last) {
+			last = r.copiedAt
+		}
+	}
+	return last.Sub(first)
+}
+
 // absenceExplained reports whether a replica lacking msg removed it legitimately:
 // the subject was written again later, the message aged out before that replica
 // was copied, or a stream limit dropped it from a head the replica has moved past.
@@ -338,12 +421,16 @@ func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
 		fmt.Fprintf(w, " %s@%s", n.Host, n.CopiedAt.Format(time.TimeOnly))
 	}
 	fmt.Fprintln(w)
-	bad := 0
+	bad, volatile := 0, 0
 	for _, v := range res.Streams {
 		status := "ok"
-		if len(v.Problems) > 0 {
+		switch {
+		case len(v.Problems) > 0:
 			status = "DIVERGED"
 			bad++
+		case v.Volatile != "":
+			status = "skipped"
+			volatile++
 		}
 		fmt.Fprintf(w, "%-9s %s  [%s]\n", status, v.Name, strings.Join(v.Holders, ","))
 		for _, p := range v.Problems {
@@ -352,6 +439,13 @@ func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
 		for _, n := range v.Notes {
 			fmt.Fprintf(w, "      %s\n", n)
 		}
+		if v.Volatile != "" {
+			fmt.Fprintf(w, "      %s\n", v.Volatile)
+		}
 	}
-	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged\n", len(res.Streams), len(res.Streams)-bad, bad)
+	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged", len(res.Streams), len(res.Streams)-bad-volatile, bad)
+	if volatile > 0 {
+		fmt.Fprintf(w, ", %d skipped as volatile", volatile)
+	}
+	fmt.Fprintln(w)
 }

spinifex/operator/cli/admin_kv_compare_test.go

@@ -160,6 +160,83 @@ func TestKVCompareNoCommonRange(t *testing.T) {
 	assert.Contains(t, report, "no sequence range is common")
 }
 
+// A 60s lease bucket renewed every few seconds turns over between digests
+// taken a couple of seconds apart, so its replicas share no window.
+func TestKVCompareVolatileStream(t *testing.T) {
+	t1, t2, t3 := t0, t0.Add(2*time.Second), t0.Add(4*time.Second)
+	lease := func(maxAge time.Duration, msgs ...msg) stream {
+		return stream{name: "KV_bedrock-leader", replicas: 3, maxAge: maxAge, msgs: msgs}
+	}
+	early := kvs(msg{3984, "$KV.bedrock-leader.leader", "h3984", t1.Add(-time.Second)})
+	late := kvs(msg{3985, "$KV.bedrock-leader.leader", "h3985", t2.Add(-time.Second)}, msg{3986, "$KV.bedrock-leader.leader", "h3986", t2})
+
+	t.Run("adjacent windows of a max_age stream are skipped, not diverged", func(t *testing.T) {
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(time.Minute, early...)),
+			doc(t, "node2", t2, lease(time.Minute, late...)),
+			doc(t, "node3", t3, lease(time.Minute, late...)))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+		assert.Contains(t, report, "skipped   KV_bedrock-leader")
+		assert.Contains(t, report, "volatile: max_age 1m0s")
+		assert.Contains(t, report, "1 streams: 0 consistent, 0 diverged, 1 skipped as volatile")
+	})
+
+	t.Run("a sequence two replicas share must still match", func(t *testing.T) {
+		mid := kvs(msg{3984, "$KV.bedrock-leader.leader", "other", t1.Add(-time.Second)}, late[0])
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(time.Minute, early...)),
+			doc(t, "node2", t2, lease(time.Minute, mid...)),
+			doc(t, "node3", t3, lease(time.Minute, late[1])))
+		require.NoError(t, err)
+		assert.True(t, divergent, report)
+		assert.Contains(t, report, "seq 3984 $KV.bedrock-leader.leader holds different content: node1=h3984 node2=other")
+	})
+
+	t.Run("a long-lived bucket with no common window is still diverged", func(t *testing.T) {
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(0, early...)),
+			doc(t, "node2", t2, lease(0, late...)),
+			doc(t, "node3", t3, lease(0, late...)))
+		require.NoError(t, err)
+		assert.True(t, divergent, report)
+		assert.Contains(t, report, "no sequence range is common")
+	})
+
+	t.Run("a long max_age stream whose missing messages nothing explains is still diverged", func(t *testing.T) {
+		old := kvs(msg{1, "a", "h1", t1.Add(-time.Minute)}, msg{2, "b", "h2", t1.Add(-time.Minute)})
+		newer := kvs(msg{10, "c", "h10", t1.Add(-time.Minute)}, msg{11, "d", "h11", t1.Add(-time.Minute)})
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(30*24*time.Hour, old...)),
+			doc(t, "node2", t2, lease(30*24*time.Hour, newer...)),
+			doc(t, "node3", t3, lease(30*24*time.Hour, newer...)))
+		require.NoError(t, err)
+		assert.True(t, divergent, report)
+		assert.Contains(t, report, "no sequence range is common")
+		assert.NotContains(t, report, "volatile")
+	})
+
+	t.Run("a replica copied last but holding the older window is still diverged", func(t *testing.T) {
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(time.Minute, late...)),
+			doc(t, "node2", t2, lease(time.Minute, late...)),
+			doc(t, "node3", t3, lease(time.Minute, early...)))
+		require.NoError(t, err)
+		assert.True(t, divergent, report)
+		assert.Contains(t, report, "seq 3985 $KV.bedrock-leader.leader is on node1,node2 but missing from node3")
+	})
+
+	t.Run("digests copied at the same instant are not excused", func(t *testing.T) {
+		divergent, report, err := cli.CompareKVDigests(
+			doc(t, "node1", t1, lease(time.Minute, early...)),
+			doc(t, "node2", t1, lease(time.Minute, late...)),
+			doc(t, "node3", t1, lease(time.Minute, late...)))
+		require.NoError(t, err)
+		assert.True(t, divergent, report)
+		assert.Contains(t, report, "no sequence range is common")
+	})
+}
+
 func TestKVCompareReplicaPlacement(t *testing.T) {
 	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0})}
 	other := stream{name: "KV_b", replicas: 1, msgs: kvs(msg{1, "k", "h1", t0})}

spinifex/operator/cli/cluster.go

@@ -600,6 +600,8 @@ func drainDHCPLeases(nc *nats.Conn, timeout time.Duration) (released, responders
 // collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
 // When nodeFilter is non-empty, only ACKs from that node count toward nodeCount
 // and all others are ignored (used by the single-node local-drain path).
+// Only the first ACK from each node counts; repeats are dropped so a duplicate
+// cannot stand in for a node that never answered.
 func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, nodeFilter string, timeout time.Duration) ([]daemon.ShutdownACK, error) {
 	inbox := nats.NewInbox()
 	sub, err := nc.SubscribeSync(inbox)
@@ -614,6 +616,7 @@ func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount
 	nc.Flush()
 
 	var acks []daemon.ShutdownACK
+	seen := make(map[string]bool, nodeCount)
 	deadline := time.Now().Add(timeout)
 	for len(acks) < nodeCount {
 		remaining := time.Until(deadline)
@@ -631,6 +634,10 @@ func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount
 		if nodeFilter != "" && ack.Node != nodeFilter {
 			continue
 		}
+		if seen[ack.Node] {
+			continue
+		}
+		seen[ack.Node] = true
 		acks = append(acks, ack)
 	}
 	return acks, nil

spinifex/operator/cli/cluster_test.go

@@ -541,3 +541,39 @@ func TestReportGuestsLeftRunningSevereLogsErrorAndNamesGuests(t *testing.T) {
 	assert.Contains(t, out, "i-abandoned")
 	assert.NotContains(t, out, "i-stopped")
 }
+
+// shutdownACKResponder answers each request on topic with one ACK per entry
+// in nodes, so a repeated name simulates one node replying twice.
+func shutdownACKResponder(t *testing.T, nc *nats.Conn, topic string, nodes ...string) {
+	t.Helper()
+	sub, err := nc.Subscribe(topic, func(msg *nats.Msg) {
+		for _, node := range nodes {
+			body, _ := json.Marshal(daemon.ShutdownACK{Node: node, Phase: "gate"})
+			_ = msg.Respond(body)
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func TestCollectShutdownACKsCountsADuplicateNodeOnce(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	shutdownACKResponder(t, nc, "test.shutdown.dup", "node1", "node1")
+
+	acks, err := collectShutdownACKs(nc, "test.shutdown.dup", []byte(`{}`), 2, "", 300*time.Millisecond)
+	require.NoError(t, err)
+	require.Len(t, acks, 1, "a second ACK from node1 must not stand in for the silent node2")
+	assert.Equal(t, "node1", acks[0].Node)
+}
+
+func TestCollectShutdownACKsNodeFilterIgnoresOtherNodes(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	shutdownACKResponder(t, nc, "test.shutdown.filter", "node2", "node1", "node1")
+
+	acks, err := collectShutdownACKs(nc, "test.shutdown.filter", []byte(`{}`), 1, "node1", 300*time.Millisecond)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "node1", acks[0].Node)
+}

spinifex/operator/cli/export_test.go

@@ -100,6 +100,9 @@ func SetAccountDeletePollInterval(d time.Duration) func() {
 // PrintTeardownPlan exposes printTeardownPlan for testing.
 var PrintTeardownPlan = printTeardownPlan
 
+// RunAccountDryRun exposes runAccountDryRun for testing.
+var RunAccountDryRun = runAccountDryRun
+
 // PrintTeardownResult exposes printTeardownResult for testing.
 var PrintTeardownResult = printTeardownResult
 

spinifex/providers/objectstore/objectstore.go

@@ -189,8 +189,9 @@ func (s *S3ObjectStore) EnsureBucket(ctx context.Context, bucket string) error {
 
 // MemoryObjectStore implements ObjectStore using in-memory storage for testing.
 type MemoryObjectStore struct {
-	objects map[string][]byte // key: bucket/key -> value: object data
-	mu      sync.RWMutex
+	objects  map[string][]byte    // key: bucket/key -> value: object data
+	modified map[string]time.Time // key: bucket/key -> last write time
+	mu       sync.RWMutex
 }
 
 var _ ObjectStore = (*MemoryObjectStore)(nil)
@@ -198,7 +199,8 @@ var _ ObjectStore = (*MemoryObjectStore)(nil)
 // NewMemoryObjectStore creates an in-memory ObjectStore for testing.
 func NewMemoryObjectStore() *MemoryObjectStore {
 	return &MemoryObjectStore{
-		objects: make(map[string][]byte),
+		objects:  make(map[string][]byte),
+		modified: make(map[string]time.Time),
 	}
 }
 
@@ -259,6 +261,7 @@ func (m *MemoryObjectStore) PutObject(_ context.Context, input *s3.PutObjectInpu
 	}
 
 	m.objects[storageKey] = data
+	m.modified[storageKey] = time.Now().UTC()
 
 	return &s3.PutObjectOutput{}, nil
 }
@@ -272,6 +275,7 @@ func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObje
 
 	storageKey := makeKey(*input.Bucket, *input.Key)
 	delete(m.objects, storageKey)
+	delete(m.modified, storageKey)
 
 	return &s3.DeleteObjectOutput{}, nil
 }
@@ -311,8 +315,9 @@ func (m *MemoryObjectStore) ListObjectsV2(_ context.Context, input *s3.ListObjec
 		}
 
 		contents = append(contents, &s3.Object{
-			Key:  aws.String(objectKey),
-			Size: aws.Int64(int64(len(data))),
+			Key:          aws.String(objectKey),
+			Size:         aws.Int64(int64(len(data))),
+			LastModified: aws.Time(m.modified[key]),
 		})
 	}
 
@@ -340,6 +345,7 @@ func (m *MemoryObjectStore) Clear() {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.objects = make(map[string][]byte)
+	m.modified = make(map[string]time.Time)
 }
 
 // Count returns the number of objects in the memory store.

spinifex/runtime/roles/spinifexui/frontend/dist/assets/DirectionContext-C7j-PYlj.js

@@ -0,0 +1 @@
+import{H as e,z as t}from"./httpRequest-DQZd8wR4.js";var n=e(t(),1),r=n.createContext(void 0);function i(){return n.useContext(r)?.direction??`ltr`}export{i as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/DirectionContext-C9rdm8xn.js

@@ -1 +0,0 @@
-import{o as e,r as t}from"./compiler-runtime-D_vZfn79.js";var n=e(t(),1),r=n.createContext(void 0);function i(){return n.useContext(r)?.direction??`ltr`}export{i as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/Match-D6VSR81X.js

@@ -1 +0,0 @@
-import{n as e,o as t,r as n}from"./compiler-runtime-D_vZfn79.js";import{n as r}from"./matchContext-DjfNTEDZ.js";import{i,n as a,t as o}from"./useSelector-C6_B_5v6.js";function s(e={}){if(e.isNotFound=!0,e.throw)throw e;return e}function c(e){return e?.isNotFound===!0}var l=t(n(),1),u=e(),d=class extends l.Component{constructor(...e){super(...e),this.state={error:0},this.reset=()=>{this.setState({error:0})}}static getDerivedStateFromProps(e,t){let n=e.getResetKey();return t.error&&t.resetKey!==n?{resetKey:n,error:0}:{resetKey:n}}static getDerivedStateFromError(e){return{error:[e]}}componentDidCatch(e,t){this.props.onCatch?.(e,t)}render(){let e=this.state.error;return e?l.createElement(this.props.errorComponent??f,{error:e[0],reset:this.reset}):this.props.children}};function f({error:e}){let[t,n]=l.useState(!1);return(0,u.jsxs)(`div`,{style:{padding:`.5rem`,maxWidth:`100%`},children:[(0,u.jsxs)(`div`,{style:{display:`flex`,alignItems:`center`,gap:`.5rem`},children:[(0,u.jsx)(`strong`,{style:{fontSize:`1rem`},children:`Something went wrong!`}),(0,u.jsx)(`button`,{style:{appearance:`none`,fontSize:`.6em`,border:`1px solid currentColor`,padding:`.1rem .2rem`,fontWeight:`bold`,borderRadius:`.25rem`},onClick:()=>n(e=>!e),children:t?`Hide Error`:`Show Error`})]}),(0,u.jsx)(`div`,{style:{height:`.25rem`}}),t?(0,u.jsx)(`div`,{children:(0,u.jsx)(`pre`,{style:{fontSize:`.7em`,border:`1px solid red`,borderRadius:`.25rem`,padding:`.3rem`,color:`red`,overflow:`auto`},children:e?.message?(0,u.jsx)(`code`,{children:e.message}):null})}):null]})}function p(e){let t=a(),n=`not-found-${o(t.stores.location,e=>e.pathname)}-${o(t.stores.status)}`;return(0,u.jsx)(d,{getResetKey:()=>n,onCatch:(t,n)=>{if(c(t))e.onCatch?.(t,n);else throw t},errorComponent:({error:t})=>{if(c(t))return e.fallback?.(t);throw t},children:e.children})}function m(){return(0,u.jsx)(`p`,{children:`Not Found`})}function h(e,t,n){return t.options.notFoundComponent?(0,u.jsx)(t.options.notFoundComponent,{...n}):e.options.defaultNotFoundComponent?(0,u.jsx)(e.options.defaultNotFoundComponent,{...n}):(0,u.jsx)(m,{})}function g(e,t){let n=t?.options.pendingComponent??e.options.defaultPendingComponent;return n?(0,u.jsx)(n,{}):null}var _=(e,t)=>e[0]===t[0]&&e[1]===t[1],v=(e,t,n)=>!t.isRoot||t.options.shellComponent||t.options.wrapInSuspense||n===!1||n===`data-only`||!e.ssr,y=l.memo(function({routeId:e}){let t=a();return(0,u.jsx)(b,{router:t,match:o(t.stores.getMatchStore(e))})});function b({router:e,match:t}){let n=e.routesById[t.routeId],a=g(e,n),o=n.options.errorComponent??e.options.defaultErrorComponent,s=n.options.onCatch??e.options.defaultOnCatch,f=n.isRoot?n.options.notFoundComponent??e.options.notFoundRoute?.options.component:n.options.notFoundComponent,m=t.ssr===!1||t.ssr===`data-only`,h=v(e,n,t.ssr)&&(n.options.wrapInSuspense??a??(n.options.errorComponent?.preload||m)),_=(0,u.jsx)(x,{match:t});m&&(_=(0,u.jsx)(i,{fallback:a,children:_})),f&&(_=(0,u.jsx)(p,{fallback:e=>{if(e.routeId??=t.routeId,e.routeId!==t.routeId)throw e;return l.createElement(f,e)},children:_})),o&&(_=(0,u.jsx)(d,{getResetKey:()=>t,errorComponent:o,onCatch:(e,n)=>{if(c(e))throw e.routeId??=t.routeId,e;s?.(e,n)},children:_})),h&&(_=(0,u.jsx)(l.Suspense,{fallback:a,children:_}));let y=n.isRoot?n.options.shellComponent:void 0;return(0,u.jsx)(r.Provider,{value:t.routeId,children:y?(0,u.jsxs)(y,{children:[_,null]}):(0,u.jsxs)(u.Fragment,{children:[_,null]})})}var x=l.memo(function({match:e}){let t=a(),n=e.routeId,r=t.routesById[n],i=l.useMemo(()=>{let i=(r.options.remountDeps??t.options.defaultRemountDeps)?.({routeId:n,loaderDeps:e.loaderDeps,params:e._strictParams,search:e._strictSearch});return i?JSON.stringify(i):void 0},[n,e.loaderDeps,e._strictParams,e._strictSearch,r.options.remountDeps,t.options.defaultRemountDeps]),o=l.useMemo(()=>{let e=r.options.component??t.options.defaultComponent;return e?(0,u.jsx)(e,{},i):(0,u.jsx)(S,{})},[i,r.options.component,t.options.defaultComponent]);if(e.status===`pending`){if(t.ssr&&!v(t,r,e.ssr))return o;if(t._tx)throw t._tx[5];return g(t,r)}if(e.status===`notFound`)return h(t,r,e.error);if(e.status===`error`)throw e.error;return o}),S=l.memo(function(){let e=a(),t=l.useContext(r),n,i,s;{let r=e.stores.getMatchStore(t);[n,i]=o(r,e=>[!!e._notFound,e.error],{compare:_}),s=o(e.stores.ids,e=>e[e.indexOf(t)+1])}if(n)return h(e,e.routesById[t],i);if(!s)return null;let c=(0,u.jsx)(y,{routeId:s});return t===`__root__`?(0,u.jsx)(l.Suspense,{fallback:g(e),children:c}):c});export{c as a,d as i,S as n,s as o,g as r,y as t};

spinifex/runtime/roles/spinifexui/frontend/dist/assets/_-B0RDrcyg.js

@@ -1,2 +0,0 @@
-const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_-c3lwS0JB.js","assets/compiler-runtime-D_vZfn79.js","assets/useSuspenseQuery-CcAupCOZ.js","assets/useBaseQuery-DIEQ3N00.js","assets/removable-Dwr9otKB.js","assets/link-BokOReHc.js","assets/useSelector-C6_B_5v6.js","assets/with-selector-DrM041uv.js","assets/httpRequest-C2Lm0gsH.js","assets/hex-encoding-Dp0dmlFs.js","assets/back-link-wsXUoJ12.js","assets/createLucideIcon-BQgzNCBR.js","assets/upload-button-XFhUazJJ.js","assets/download-B-6MA128.js","assets/trash-DVab4pEz.js","assets/button-CK8q3iGP.js","assets/dist-ezrYKpGB.js","assets/awsClient-Cs97pDBK.js","assets/index.browser-BsX6pxfE.js","assets/s3-DtqEKnCO.js","assets/useMutation-6rdxVQpi.js","assets/mutation-BziiWz5u.js","assets/error-banner-DeOeVBKS.js","assets/delete-confirmation-dialog-C8hMA1LN.js","assets/alert-dialog-BAbNi2ni.js","assets/DialogTitle-D5W33H9I.js","assets/useValueAsRef-DUYSYu2j.js","assets/createBaseUIEventDetails-DLoDVuTV.js","assets/useScrollLock-CSV5CwQU.js","assets/s3-CShBhbpV.js","assets/queryOptions-Dfvzj6n2.js"])))=>i.map(i=>d[i]);
-import{n as e,t}from"./lazyRouteComponent-BowAthaT.js";import{E as n,O as r}from"./httpRequest-C2Lm0gsH.js";import{go as i}from"./awsClient-Cs97pDBK.js";import{t as a}from"./s3-CShBhbpV.js";var o=e(`/_auth/s3/ls/$bucket/$`)({loader:async({context:e,params:t})=>{let n=r(t._splat??``);await e.queryClient.query({...a(t.bucket,n),staleTime:`static`})},head:({params:e})=>({meta:[{title:`${n(e._splat??``,`${e.bucket}/`)} | S3 | Mulga`}]}),component:t(()=>i(()=>import(`./_-c3lwS0JB.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30])),`component`)});export{o as t};