feat(ebs): record an unpublish obligation on force detach and terminate (#1504)

* feat(ebs): record an unpublish obligation on force detach and terminate

A force detach, or a terminate whose unmount did not complete, clears a
volume's attachment while the node that served it may still export it.
The request now records an UnpublishObligation naming that publish in the
new spinifex-ebs-unpublish-obligations bucket before the clear, and
returns without waiting on the node.

The node-local unpublishObligationReaper settles its own node's
obligations under the mounter's per-volume lock, through the existing
UnpublishVolume client: it seals a surviving volume, drops a deleting or
absent one unsealed, and closes without a provider call when a newer
attachment or a newer publish on the node owns the export. A cluster-wide
pass closes the obligations of nodes removed from the cluster, and
spx admin volumes obligations lists the open ones.

The force-complete terminate and a stopped-instance terminate after a
failed seal record the same obligation; a terminate whose unmount
completed records none. ForceDetachVolume now answers with the attachment
in state detaching, as AWS answers a detach.

* fix(ebs): refuse a publish while the obligation reaper seals the volume

A publish waited on the volume lock while the unpublish obligation reaper
held it through a seal, so AttachVolume waited on work that scales with
the volume. A publish now refuses at once with ErrVolumeSettling, which
AttachVolume answers as IncorrectState, and the reaper takes the lock only
when nobody holds or awaits it.

* fix(ebs): refuse a publish behind any seal of the volume, a stop's included

A stop's or detach's sealing unpublish now marks the volume lock sealing,
so an attach after a force detach answers IncorrectState instead of
waiting out the seal. A waiting seal lets waiting publishes go first.

* fix(ebs): close a superseded unpublish obligation past a newer publish here

After a force detach and an attach again on the same node, the reaper
returned on the newer publish before it read the volume, so the old
obligation stayed open until the new attachment ended.

* fix(vm): recover a wedged terminate without waiting on its stalled unmount

The unmount records every volume as owed before it starts, so recovery
keeps that record rather than wait for the request lock the stalled
seal holds.

.github/CODEOWNERS

@@ -1,15 +0,0 @@
-# CODEOWNERS — GitHub PR review enforcement
-#
-# Format: <path pattern>  <owner> [<owner> …]
-# Order matters — last matching pattern wins.
-#
-# Set the matching branch protection rule on `dev` and `main` to
-# "Require review from Code Owners" or this file is advisory only.
-
-# Network layer is governed by its invariant tests.
-# Changes here must respect those clauses
-# see spinifex/domains/network/invariants/
-# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
-/spinifex/domains/network/          @joshsiv-mulga
-/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
-/spinifex/domains/network/invariants/ @joshsiv-mulga

.github/pull_request_template.md

@@ -6,20 +6,6 @@
 
 <!-- The notable changes, as bullets. -->
 
-## Account-scoped lifecycle impact
-
-<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
-
-- Lifecycle owner and resource family:
-- Owner-local ordering, recovery, retained-state, or force impact:
-- Cross-domain dependency or ownership impact:
-- Evidence added or rechecked:
-- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
-
-## Mulga design
-
-<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
-
 ## Community Contribution License
 
 - [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/scripts/hypervisor-disk-preflight_test.sh

@@ -1,87 +0,0 @@
-#!/bin/sh
-# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
-# a stub ssh that answers for each hypervisor from a canned file, so every
-# verdict is checked without a real host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
-# when that host has no file, the way ssh does for a host it cannot reach.
-mkdir -p "$TMP/bin" "$TMP/hosts"
-cat > "$TMP/bin/ssh" <<'STUB'
-#!/bin/sh
-for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
-cat >/dev/null
-if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
-echo "ssh: Could not resolve hostname $host" >&2
-exit 255
-STUB
-chmod +x "$TMP/bin/ssh"
-PATH="$TMP/bin:$PATH"
-FAKE="$TMP/hosts"
-export PATH FAKE
-
-cat > "$TMP/env.tfvars" <<'TFVARS'
-nodes = [
-  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
-  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
-]
-TFVARS
-
-GiB=1048576
-plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
-
-# run <label> <want-rc> <want-text>
-run() {
-    rc=0
-    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
-    if [ "$rc" -ne "$2" ]; then
-        fail "$1: exit $rc, want $2"
-        echo "$out" | sed 's/^/        /'
-        return
-    fi
-    case $out in
-        *"$3"*) pass "$1" ;;
-        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
-    esac
-}
-
-echo "-- hypervisor-disk-preflight.sh"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$plenty" > "$FAKE/banksia"
-run "every host has room" 0 "banksia: root fs 51200 MiB free"
-
-echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
-run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
-run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
-
-echo "$plenty" > "$FAKE/banksia"
-rm -f "$FAKE/wattle"
-run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
-
-echo "$plenty" > "$FAKE/wattle"
-echo "$((50 * GiB)) - -" > "$FAKE/banksia"
-run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
-
-echo "0 0 /x" > "$FAKE/retired"
-echo "$plenty" > "$FAKE/banksia"
-run "a commented-out node is never checked" 0 "wattle: root fs"
-
-if [ "$fails" -ne 0 ]; then
-    echo "$fails failure(s)"
-    exit 1
-fi
-echo "  hypervisor-disk-preflight ok"

.github/scripts/mulga-design-check.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi

.github/scripts/mulga_design_check.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
-
-import os
-import re
-
-COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
-SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
-LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
-PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
-
-
-def filled(body):
-    inside = False
-    for line in COMMENT_RE.sub("", body or "").splitlines():
-        if SECTION_RE.match(line):
-            inside = True
-        elif inside and line.lstrip().startswith("#"):
-            return False
-        elif inside:
-            m = LINE_RE.match(line)
-            if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
-                return True
-    return False
-
-
-def main():
-    if filled(os.environ.get("PR_BODY", "")):
-        print("The Mulga design line is filled in.")
-    else:
-        print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.github/scripts/test_mulga_design_check.py

@@ -1,23 +0,0 @@
-import unittest
-
-from mulga_design_check import filled
-
-
-class FilledTest(unittest.TestCase):
-    def test_accepts_either_form(self):
-        self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
-        self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
-
-    def test_rejects_unanswered(self):
-        self.assertFalse(filled(""))
-        self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
-        self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
-        self.assertFalse(filled("## Mulga design\nUpdated:"))
-
-    def test_scoped_to_the_section(self):
-        self.assertFalse(filled("Updated: mulga#1"))
-        self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
-
-
-if __name__ == "__main__":
-    unittest.main()

.github/scripts/wait-for-memory_test.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-# wait-for-memory_test.sh — drives wait-for-memory.sh against a fake meminfo
-# that a background writer changes mid-wait, so each verdict is checked without
-# a loaded host.
-set -eu
-
-SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-WAIT="$SCRIPT_DIR/wait-for-memory.sh"
-TMP=$(mktemp -d)
-trap 'rm -rf "$TMP"' EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-
-meminfo() { printf 'MemTotal:       65536000 kB\nMemAvailable:   %s kB\n' "$(( $1 * 1024 ))" > "$TMP/meminfo"; }
-MEMINFO="$TMP/meminfo"
-MEM_WAIT_POLL_S=1
-export MEMINFO MEM_WAIT_POLL_S
-
-meminfo 30000
-if MEM_WAIT_TIMEOUT_S=5 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  pass "enough memory returns at once"
-else
-  fail "enough memory returns at once: $(cat "$TMP/out")"
-fi
-
-meminfo 21577
-( sleep 2; meminfo 30000 ) &
-if MEM_WAIT_TIMEOUT_S=20 bash "$WAIT" 26624 > "$TMP/out" 2>&1 && grep -q waiting "$TMP/out"; then
-  pass "memory freed mid-wait is admitted"
-else
-  fail "memory freed mid-wait is admitted: $(cat "$TMP/out")"
-fi
-wait
-
-meminfo 21577
-if MEM_WAIT_TIMEOUT_S=2 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
-  fail "memory that never frees fails: exited 0"
-elif grep -q '::error::need 26624 MiB available, have 21577 MiB' "$TMP/out" && grep -q 'RSS' "$TMP/out"; then
-  pass "memory that never frees fails and names the biggest processes"
-else
-  fail "memory that never frees fails and names the biggest processes: $(cat "$TMP/out")"
-fi
-
-[ "$fails" -eq 0 ] || { echo "wait-for-memory: $fails failed"; exit 1; }
-echo "wait-for-memory: all passed"

.github/workflows/e2e-nightly.yml

@@ -1017,7 +1017,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1275,7 +1275,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
       - uses: actions/checkout@v7.0.1
@@ -1423,7 +1423,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore
@@ -1547,7 +1547,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Predastore

.github/workflows/mulga-design.yml

@@ -2,7 +2,8 @@ name: Mulga design
 
 # Reminds the author to say whether the Mulga design records need to change.
 # It reads only the pull request body, uses no secrets, and warns without ever
-# failing: the check is a prompt, not a gate.
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.
 
 on:
   pull_request:
@@ -16,7 +17,7 @@ permissions:
 jobs:
   mulga_design:
     name: Mulga design line
-    if: ${{ !endsWith(github.actor, '[bot]') }}
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: ubuntu-26.04
     steps:
       - name: Checkout Code
@@ -28,6 +29,4 @@ jobs:
         env:
           PR_BODY: ${{ github.event.pull_request.body }}
         working-directory: .github/scripts
-        run: |
-          python3 -m unittest -q test_mulga_design_check
-          python3 mulga_design_check.py
+        run: bash mulga-design-check.sh

.github/workflows/perf-3node.yml

@@ -113,7 +113,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ env.GIT_BRANCH }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - uses: actions/checkout@v7.0.1

.github/workflows/publish-system-ami.yml

@@ -84,7 +84,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Viperblock
         uses: actions/checkout@v7.0.1

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -85,7 +85,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1
@@ -335,7 +335,7 @@ jobs:
 
       - name: Resolve sibling release tags from go.mod
         id: siblings
-        run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
+        run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
 
       - name: Checkout Predastore
         uses: actions/checkout@v7.0.1

.github/workflows/segscan-oracle.yml

@@ -34,7 +34,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

.github/workflows/spinifex.yml

@@ -36,7 +36,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -166,7 +166,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock
@@ -268,7 +268,7 @@ jobs:
         id: siblings
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
+        run: bash .github/scripts/resolve-sibling-refs.sh
         working-directory: spinifex
 
       - name: Checkout Viperblock

Makefile

@@ -42,11 +42,15 @@ ifdef QUIET
   _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
   _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
   _SECQ  = >
+  _HDR   = :
+  _RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
 else
   _Q     =
   _COVQ  =
   _RACEQ =
   _SECQ  = 2>&1 | tee
+  _HDR   = echo -e
+  _RUNSH = echo "-- $$t"; sh "$$t" || exit 1
 endif
 
 build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -240,27 +244,25 @@ preflight:
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
 
 test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running .github/scripts/*_test.sh...."
-	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
-		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
-	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
+	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
+		.github/scripts/wait-for-memory.sh \
+		.github/scripts/mulga-design-check.sh
+	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
+	@$(_HDR) "  test-build-scripts ok"
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no

README.md

@@ -70,8 +70,8 @@ The platform serves **405 AWS API operations** across the services below. See ex
 | **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
 | **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
 | **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
-| **Lambda** | Serverless functions | | Q1 2027 |
-| **SQS** | Message queues | | Q1 2027 |
+| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
+| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |
 
 Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

docs/testing/e2e-coverage.md

@@ -110,7 +110,7 @@ Three entry points, in the order they run:
 
 ### `iam` — everything IAM and STS
 
-Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
+Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
 
 ### `cert` — the cluster CA and TLS identity
 

internal/ecsgw/client.go

@@ -7,9 +7,10 @@ package ecsgw
 
 import (
 	"bytes"
-	"context"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,10 +18,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
-
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )
 
 // targetPrefix is the X-Amz-Target service prefix the gateway's ECS dispatch
@@ -28,31 +27,26 @@ import (
 // suffix); this one mirrors the AWS JSON 1.1 convention.
 const targetPrefix = "AmazonEC2ContainerServiceV20141113."
 
-// CredentialsFunc yields the current SigV4 credentials. It is called once per
-// request so a provider backed by IMDS can rotate instance-role creds (and their
-// session token) transparently. Must be safe for concurrent use.
-type CredentialsFunc func(ctx context.Context) (accessKey, secretKey, sessionToken string, err error)
-
 // Client posts SigV4-signed AWS JSON 1.1 requests to the gateway for service
 // "ecs". One client is reused for register/heartbeat/state/poll.
 type Client struct {
 	baseURL    string
-	creds      CredentialsFunc
+	signer     *gwsign.Signer
 	region     string
 	httpClient *http.Client
 }
 
 // New builds a client. caPath optionally pins the gateway TLS CA; empty relies on
-// the system trust store. creds supplies the per-call instance-role credentials.
+// the system trust store. signer supplies the per-call instance-role credentials.
 // region defaults to us-east-1 when empty (SigV4 requires a non-empty region).
 // timeout bounds a single call — callers needing a long-poll pass a larger value
 // than the default register/heartbeat timeout.
-func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout time.Duration) (*Client, error) {
+func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout time.Duration) (*Client, error) {
 	if baseURL == "" {
 		return nil, fmt.Errorf("ecsgw: baseURL is required")
 	}
-	if creds == nil {
-		return nil, fmt.Errorf("ecsgw: credentials func is required")
+	if signer == nil {
+		return nil, fmt.Errorf("ecsgw: signer is required")
 	}
 	if region == "" {
 		region = "us-east-1"
@@ -79,7 +73,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t
 
 	return &Client{
 		baseURL: strings.TrimRight(baseURL, "/"),
-		creds:   creds,
+		signer:  signer,
 		region:  region,
 		httpClient: &http.Client{
 			Timeout:   timeout,
@@ -90,7 +84,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t
 
 // Call SigV4-signs and POSTs body under X-Amz-Target "<prefix><action>" to the
 // gateway root, returning the 2xx response body. Credentials are fetched per call
-// and the v4 signer emits X-Amz-Security-Token for instance-role (temporary)
+// and the signer emits X-Amz-Security-Token for instance-role (temporary)
 // creds, so the gateway's ASIA path verifies them. No retry; callers wrap.
 func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req, err := http.NewRequest(http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
@@ -100,12 +94,8 @@ func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
 	req.Header.Set("X-Amz-Target", targetPrefix+action)
 
-	akid, secret, token, err := c.creds(req.Context())
-	if err != nil {
-		return nil, fmt.Errorf("retrieve credentials: %w", err)
-	}
-	signer := v4.NewSigner(credentials.NewStaticCredentials(akid, secret, token))
-	if _, err := signer.Sign(req, bytes.NewReader(body), "ecs", c.region, time.Now()); err != nil {
+	sum := sha256.Sum256(body)
+	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), "ecs", c.region); err != nil {
 		return nil, fmt.Errorf("sign request: %w", err)
 	}
 

internal/ecsgw/client_test.go

@@ -1,24 +1,29 @@
 package ecsgw
 
 import (
-	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )
 
-// testCreds returns a fixed temporary credential set (with a session token, as
-// IMDS instance-role creds carry).
-func testCreds(context.Context) (string, string, string, error) {
-	return "ASIATEST", "secret", "session-token-xyz", nil
+const testSecret = "secret"
+
+// testSigner signs with a fixed temporary credential set (with a session token,
+// as IMDS instance-role creds carry).
+func testSigner() *gwsign.Signer {
+	return gwsign.New(credentials.NewStaticCredentialsProvider("ASIATEST", testSecret, "session-token-xyz"))
 }
 
 func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 	t.Helper()
-	c, err := New(srv.URL, "", testCreds, "ap-southeast-2", time.Second)
+	c, err := New(srv.URL, "", testSigner(), "ap-southeast-2", time.Second)
 	if err != nil {
 		t.Fatalf("New: %v", err)
 	}
@@ -30,7 +35,14 @@ func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 // 2xx body.
 func TestClient_CallSignsAndTargets(t *testing.T) {
 	var gotTarget, gotAuth, gotBody, gotToken string
+	var verifyErr error
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// sigv4.Parse reads and hashes the body itself, then restores it.
+		sr, err := sigv4.Parse(r)
+		if err == nil {
+			_, err = sr.Verify(testSecret, "ap-southeast-2", "ecs")
+		}
+		verifyErr = err
 		gotTarget = r.Header.Get("X-Amz-Target")
 		gotAuth = r.Header.Get("Authorization")
 		gotToken = r.Header.Get("X-Amz-Security-Token")
@@ -45,6 +57,9 @@ func TestClient_CallSignsAndTargets(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Call: %v", err)
 	}
+	if verifyErr != nil {
+		t.Fatalf("gateway-side SigV4 verify failed: %v", verifyErr)
+	}
 	if string(out) != `{"ok":true}` {
 		t.Errorf("body = %q", out)
 	}
@@ -78,10 +93,10 @@ func TestClient_CallErrorStatus(t *testing.T) {
 }
 
 func TestNew_Validation(t *testing.T) {
-	if _, err := New("", "", testCreds, "r", 0); err == nil {
+	if _, err := New("", "", testSigner(), "r", 0); err == nil {
 		t.Error("want error for empty baseURL")
 	}
 	if _, err := New("https://gw:9999", "", nil, "r", 0); err == nil {
-		t.Error("want error for nil credentials func")
+		t.Error("want error for nil signer")
 	}
 }

internal/gwsign/signer.go

@@ -23,6 +23,12 @@ type Signer struct {
 	provider aws.CredentialsProvider
 }
 
+// New returns a Signer over provider. Wrap a provider that fetches remotely in
+// aws.NewCredentialsCache, since Sign retrieves on every call.
+func New(provider aws.CredentialsProvider) *Signer {
+	return &Signer{provider: provider}
+}
+
 // NewStatic returns a Signer using fixed long-lived keys (no session token).
 func NewStatic(accessKey, secretKey string) *Signer {
 	return &Signer{provider: credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")}

internal/imdscreds/imdscreds.go

@@ -21,31 +21,18 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )
 
-// Credentials is a resolved instance-role credential set with parsed expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // Fetch resolves the instance role then its credentials via the SDK's IMDSv2
 // client and the ec2rolecreds provider. base is the IMDS root, e.g.
 // http://169.254.169.254/latest.
-func Fetch(client *http.Client, base string) (Credentials, error) {
+func Fetch(client *http.Client, base string) (aws.Credentials, error) {
 	creds, err := NewProvider(client, base).Retrieve(context.Background())
 	if err != nil {
-		return Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
 	}
 	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
-		return Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
+		return aws.Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
 	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
+	return creds, nil
 }
 
 // NewProvider builds an ec2rolecreds provider against base, an IMDS root URL

internal/imdscreds/imdscreds_test.go

@@ -1,9 +1,11 @@
 package imdscreds
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -60,8 +62,8 @@ func TestFetch_OK(t *testing.T) {
 	if got.AccessKeyID != "AKIATEST" || got.SecretAccessKey != "secrettest" || got.SessionToken != "sessiontoken" {
 		t.Errorf("creds mismatch: %+v", got)
 	}
-	if !got.Expiration.Equal(exp) {
-		t.Errorf("Expiration = %v, want %v", got.Expiration, exp)
+	if !got.Expires.Equal(exp) {
+		t.Errorf("Expires = %v, want %v", got.Expires, exp)
 	}
 }
 
@@ -132,3 +134,46 @@ func TestNewProvider_WithholdsExtendOnFailure(t *testing.T) {
 		t.Error("provider dropped AdjustExpiresBy, which a caller using an ExpiryWindow relies on")
 	}
 }
+
+// Behind aws.CredentialsCache, as the ecs-agent uses it, a refresh that fails
+// against an expired credential must surface as an error rather than hand back
+// the dead key with a pushed-out expiry.
+func TestNewProvider_CachedFailedRefreshErrors(t *testing.T) {
+	var hits atomic.Int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits.Add(1) > 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"AccessKeyId":     "AKIA",
+			"SecretAccessKey": "secret",
+			"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+		})
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+
+	cache := aws.NewCredentialsCache(NewProvider(srv.Client(), srv.URL+"/latest"))
+	first, err := cache.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("first Retrieve: %v", err)
+	}
+	got, err := cache.Retrieve(context.Background())
+	if err == nil {
+		t.Fatalf("refresh failure was reported as success: %+v", got)
+	}
+	if got.Expires.After(first.Expires) {
+		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expires, got.Expires)
+	}
+}

internal/stsauth/stsauth.go

@@ -8,8 +8,8 @@ package stsauth
 import (
 	"fmt"
 	"net/http"
-	"time"
 
+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -20,24 +20,15 @@ import (
 // scope and the gateway does not enforce a specific value.
 const defaultRegion = "us-east-1"
 
-// Credentials is the decoded result of AssumeRole: a SigV4 credential set bound
-// to the assumed role, with its expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // AssumeRole builds an aws-sdk STS client pointed at the gateway and calls
 // AssumeRole with the supplied static (instance-role) credentials, returning the
 // assumed-role credential set. httpClient must trust the gateway CA.
-func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (Credentials, error) {
+func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (awsv2.Credentials, error) {
 	if region == "" {
 		region = defaultRegion
 	}
 	if roleARN == "" || sessionName == "" {
-		return Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
+		return awsv2.Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
 	}
 	sess, err := session.NewSession(&aws.Config{
 		Region:      aws.String(region),
@@ -47,7 +38,7 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		DisableSSL:  aws.Bool(false),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("new session: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("new session: %w", err)
 	}
 
 	out, err := sts.New(sess).AssumeRole(&sts.AssumeRoleInput{
@@ -55,16 +46,17 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		RoleSessionName: aws.String(sessionName),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("AssumeRole: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole: %w", err)
 	}
 	if out.Credentials == nil {
-		return Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
 	}
 	c := out.Credentials
-	return Credentials{
+	return awsv2.Credentials{
 		AccessKeyID:     aws.StringValue(c.AccessKeyId),
 		SecretAccessKey: aws.StringValue(c.SecretAccessKey),
 		SessionToken:    aws.StringValue(c.SessionToken),
-		Expiration:      aws.TimeValue(c.Expiration),
+		CanExpire:       c.Expiration != nil,
+		Expires:         aws.TimeValue(c.Expiration),
 	}, nil
 }

internal/stsauth/stsauth_test.go

@@ -41,8 +41,8 @@ func TestAssumeRole_DecodesCredentials(t *testing.T) {
 		t.Errorf("bad creds: %+v", creds)
 	}
 	want, _ := time.Parse(time.RFC3339, "2031-01-02T03:04:05Z")
-	if !creds.Expiration.Equal(want) {
-		t.Errorf("Expiration = %v, want %v", creds.Expiration, want)
+	if !creds.Expires.Equal(want) {
+		t.Errorf("Expires = %v, want %v", creds.Expires, want)
 	}
 	// The request must carry the AssumeRole action and the role ARN we asked for.
 	if !strings.Contains(gotBody, "Action=AssumeRole") {

internal/testkit/awserror.go

@@ -0,0 +1,29 @@
+package testutil
+
+import (
+	"testing"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// RequireAWSError asserts err carries the AWS code and exact client-facing
+// message; an empty message asserts the bare code with no message attached.
+func RequireAWSError(t *testing.T, err error, code, message string) {
+	t.Helper()
+	require.Error(t, err)
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+	assert.Equal(t, message, msg)
+}
+
+// RequireAWSCode asserts err carries the AWS code, whatever its message.
+func RequireAWSCode(t *testing.T, err error, code string) {
+	t.Helper()
+	require.Error(t, err)
+	got, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+}

internal/testkit/nats.go

@@ -167,11 +167,11 @@ func registerVpcdStub(t *testing.T, nc *nats.Conn, defaultPayload []byte) {
 	})
 }
 
-// SeedKV creates a KV bucket and populates it with the given entries.
+// SeedKV creates or updates a KV bucket and populates it with the given entries.
 // Returns the KV handle for further use.
 func SeedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue {
 	t.Helper()
-	kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
+	kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
 	require.NoError(t, err)
 	for k, v := range entries {
 		_, err := kv.Put(t.Context(), k, v)

internal/testkit/nats_test.go

@@ -0,0 +1,19 @@
+package testutil
+
+import (
+	"testing"
+
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSeedKV_ExistingBucket(t *testing.T) {
+	_, _, js := StartTestJetStream(t)
+	_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "seeded", History: 5})
+	require.NoError(t, err)
+
+	kv := SeedKV(t, js, "seeded", map[string][]byte{"k": []byte("v")})
+	entry, err := kv.Get(t.Context(), "k")
+	require.NoError(t, err)
+	require.Equal(t, []byte("v"), entry.Value())
+}

scripts/images/ecs-agent-gpu/ecs-agent.service

@@ -9,8 +9,7 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds,
-# containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
 # cloud-init at launch. Absent file is tolerated — the agent falls back to
 # its built-in defaults.
 EnvironmentFile=-/etc/spinifex-ecs/agent.env

scripts/images/ecs-agent/ecs-agent.initd

@@ -3,8 +3,7 @@
 # IMDSv2, registers + heartbeats + polls task assignments through the AWS gateway
 # over TLS+SigV4 (no NATS), and pulls task images through containerd.
 #
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-# socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.
 
 description="Spinifex ECS container-instance agent"
 command="/usr/local/bin/ecs-agent"
@@ -28,7 +27,7 @@ start_pre() {
         # shellcheck disable=SC1091
         . /etc/spinifex-ecs/agent.env
         export ECS_GATEWAY_URL ECS_GATEWAY_CA ECS_REGION ECS_CLUSTER \
-            ECS_ACCESS_KEY ECS_SECRET_KEY ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
+            ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
             ECS_HEARTBEAT_INTERVAL ECS_POLL_INTERVAL
     fi
 }

spinifex/agents/ecs/agent/agent.go

@@ -12,8 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 	ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )
 
@@ -102,7 +103,9 @@ func New(cfg Config) (*Agent, error) {
 		AgentVersion: cfg.AgentVersion,
 	}
 
-	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	// The cache refetches only once the real expiry passes, and a failed
+	// refresh is an error rather than an extended expiry.
+	creds := aws.NewCredentialsCache(imdscreds.NewProvider(imdsClient, cfg.IMDSBase))
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
 
 	// A dial that fails here is not fatal and not final: the node starts the

spinifex/agents/ecs/agent/config.go

@@ -21,17 +21,14 @@ const (
 
 // Config holds the static settings the agent reads at boot. Cluster identity
 // (account ID, instance ID) is discovered at runtime from IMDS, not configured;
-// only the cluster *name* the instance was launched into is static. AccessKey /
-// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
-// gateway calls with instance-role credentials from IMDS; the agent never holds
-// a NATS token.
+// only the cluster *name* the instance was launched into is static. The agent
+// SigV4-signs gateway calls with instance-role credentials from IMDS and never
+// holds a NATS token.
 type Config struct {
 	GatewayURL       string
 	GatewayCA        string
 	Region           string
 	ClusterName      string
-	AccessKey        string
-	SecretKey        string
 	IMDSBase         string
 	ContainerdSocket string
 	Heartbeat        time.Duration
@@ -53,8 +50,6 @@ func LoadConfig(envFile string) Config {
 		GatewayCA:        get("ECS_GATEWAY_CA"),
 		Region:           get("ECS_REGION"),
 		ClusterName:      get("ECS_CLUSTER"),
-		AccessKey:        get("ECS_ACCESS_KEY"),
-		SecretKey:        get("ECS_SECRET_KEY"),
 		IMDSBase:         get("ECS_IMDS_BASE"),
 		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
 	}

spinifex/agents/ecs/agent/config_test.go

@@ -10,7 +10,7 @@ import (
 func TestLoadConfig_Defaults(t *testing.T) {
 	// Ensure env vars don't leak in from the host.
 	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
-		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CLUSTER", "ECS_IMDS_BASE",
 		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
 		t.Setenv(k, "")
 	}
@@ -43,7 +43,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
-	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
 
 	cfg := LoadConfig(envFile)
 	if cfg.GatewayURL != "https://gw.env" {
@@ -55,9 +54,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 	if cfg.ClusterName != "prod" {
 		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
 	}
-	if cfg.AccessKey != "AKIAENV" {
-		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
-	}
 }
 
 func TestLoadConfig_HeartbeatOverride(t *testing.T) {

spinifex/agents/ecs/agent/controlplane.go

@@ -1,17 +1,17 @@
 package agent
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"time"
 
+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/ecsgw"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )
 
@@ -47,26 +47,14 @@ const pollTimeout = 30 * time.Second
 
 // newGatewayControlPlane builds the SigV4 client signing with the instance-role
 // credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
-func newGatewayControlPlane(cfg Config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
-	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+func newGatewayControlPlane(cfg Config, creds awsv2.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, gwsign.New(creds), cfg.Region, pollTimeout)
 	if err != nil {
 		return nil, err
 	}
 	return &gatewayControlPlane{client: client}, nil
 }
 
-// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
-// per-call credentials hook.
-func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
-	return func(ctx context.Context) (string, string, string, error) {
-		c, err := p.Retrieve(ctx)
-		if err != nil {
-			return "", "", "", err
-		}
-		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
-	}
-}
-
 func (g *gatewayControlPlane) Register(id identity) error {
 	in := &ecs.RegisterContainerInstanceInput{
 		Cluster:                  aws.String(id.ClusterName),

spinifex/agents/ecs/agent/controlplane_test.go

@@ -8,17 +8,17 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )
 
-// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// fixedCredsProvider is a static aws.CredentialsProvider for tests
 // that only need a valid SigV4 signature, not real IMDS rotation.
 type fixedCredsProvider struct{}
 
-func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
-	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+func (fixedCredsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
 }
 
 func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {

spinifex/agents/ecs/agent/credendpoint.go

@@ -13,9 +13,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
 	"github.com/mulgadc/spinifex/internal/stsauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 )
 
 const (
@@ -56,7 +56,7 @@ type credRole struct {
 
 type credEntry struct {
 	roleARN string
-	creds   stsauth.Credentials
+	creds   aws.Credentials
 }
 
 // credEndpoint is the host-netns HTTP server that hands a task its IAM role
@@ -64,7 +64,7 @@ type credEntry struct {
 // mints/caches credentials on demand by assuming the role over the gateway with
 // the agent's instance credentials.
 type credEndpoint struct {
-	provider   credentials.CredentialsProvider
+	provider   aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -75,7 +75,7 @@ type credEndpoint struct {
 
 	// assume mints fresh credentials for a role; defaults to AssumeRole over the
 	// gateway and is overridable in tests.
-	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+	assume func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error)
 
 	mu         sync.Mutex
 	roles      map[string]credRole  // credID -> task role
@@ -89,7 +89,7 @@ type credEndpoint struct {
 // newCredEndpoint builds the endpoint. An empty ip/port falls back to the
 // AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
 // adds the link-local address to the dummy interface (real or fake).
-func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
 	if ip == "" {
 		ip = defaultCredEndpointIP
 	}
@@ -165,7 +165,7 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
-		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+		Expiration:      creds.Expires.UTC().Format(time.RFC3339),
 	}); err != nil {
 		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
@@ -176,12 +176,12 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (aws.Credentials, credRole, error) {
 	c.mu.Lock()
 	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, credRole{}, errUnknownCredID
+		return aws.Credentials{}, credRole{}, errUnknownCredID
 	}
 	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
@@ -191,7 +191,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 
 	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, role, err
+		return aws.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
@@ -204,7 +204,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 // roleARN over the gateway — the execution-role path for ECR image pulls. It
 // caches the assumed set and re-assumes within the refresh margin. session scopes
 // the STS session name.
-func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+func (c *credEndpoint) AssumeProvider(roleARN, session string) aws.CredentialsProvider {
 	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
 }
 
@@ -216,39 +216,34 @@ type assumeProvider struct {
 	session string
 
 	mu     sync.Mutex
-	cached stsauth.Credentials
+	cached aws.Credentials
 }
 
-var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+var _ aws.CredentialsProvider = (*assumeProvider)(nil)
 
-func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+func (p *assumeProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if !credValid(p.cached, credRefreshMargin) {
 		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
 		if err != nil {
-			return credentials.Credentials{}, err
+			return aws.Credentials{}, err
 		}
 		p.cached = creds
 	}
-	return credentials.Credentials{
-		AccessKeyID:     p.cached.AccessKeyID,
-		SecretAccessKey: p.cached.SecretAccessKey,
-		SessionToken:    p.cached.SessionToken,
-		Expiration:      p.cached.Expiration,
-	}, nil
+	return p.cached, nil
 }
 
 // assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
 // against the gateway with the agent's instance credentials.
-func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 	instCreds, err := c.provider.Retrieve(ctx)
 	if err != nil {
-		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("instance credentials: %w", err)
 	}
 	client, err := c.client()
 	if err != nil {
-		return stsauth.Credentials{}, err
+		return aws.Credentials{}, err
 	}
 	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
 		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
@@ -387,14 +382,14 @@ func (c *credEndpoint) delEndpointAddr() {
 }
 
 // credValid reports whether creds are usable now with margin to spare.
-func credValid(c stsauth.Credentials, margin time.Duration) bool {
+func credValid(c aws.Credentials, margin time.Duration) bool {
 	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
 		return false
 	}
-	if c.Expiration.IsZero() {
+	if c.Expires.IsZero() {
 		return true
 	}
-	return time.Now().Add(margin).Before(c.Expiration)
+	return time.Now().Add(margin).Before(c.Expires)
 }
 
 // sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -9,15 +9,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )
 
 // newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
 // counting calls so cache behaviour is observable. run is nil (no iface plumbing).
-func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+func newTestCredEndpoint(creds aws.Credentials) (*credEndpoint, *int32) {
 	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 	var calls int32
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 		atomic.AddInt32(&calls, 1)
 		return creds, nil
 	}
@@ -26,8 +26,8 @@ func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
 
 func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expires: exp,
 	})
 	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
@@ -54,11 +54,11 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 // The STS session name lands in the assumed-role ARN and in STS logs, so it is
 // built from the public task ID and never from the secret credID.
 func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c, _ := newTestCredEndpoint(aws.Credentials{})
 	var gotSession string
-	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, session string) (aws.Credentials, error) {
 		gotSession = session
-		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+		return aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
 	}
 	c.Register("secret-cred", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -72,7 +72,7 @@ func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
 }
 
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, calls := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
 	if rr.Code != http.StatusNotFound {
@@ -84,7 +84,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 }
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
@@ -94,8 +94,8 @@ func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 }
 
 func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
@@ -112,8 +112,8 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 
 func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	// Expiry inside the refresh margin -> every hit re-assumes.
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Minute),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
@@ -126,7 +126,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 }
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
@@ -137,9 +137,9 @@ func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 }
 
 func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
-		return stsauth.Credentials{}, context.DeadlineExceeded
+	c, _ := newTestCredEndpoint(aws.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
+		return aws.Credentials{}, context.DeadlineExceeded
 	}
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -150,8 +150,8 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 }
 
 func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80

spinifex/agents/ecs/agent/credinject_test.go

@@ -5,7 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
@@ -30,7 +30,7 @@ func TestTaskCredID(t *testing.T) {
 // The task ID is public (it is the task ARN suffix), so naming another task's ID
 // at the credential endpoint must not yield that task's role.
 func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
 	a := Agent{cred: c}
 	get := func(id string) int {
 		rr := httptest.NewRecorder()

spinifex/agents/ecs/agent/doc.go

@@ -5,8 +5,7 @@
 // task assignments, runs them through containerd, and reports state back over
 // the gateway.
 //
-// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// Static config (gateway URL, CA, region, cluster, containerd socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
 // (KEY=value); real env vars override it. Host identity (account, instance, AZ)
 // comes from IMDS at boot.
 package agent

spinifex/agents/ecs/agent/ecrresolver.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	"github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )
 
@@ -22,7 +22,7 @@ import (
 // pulled, so the agent can still register + heartbeat on a host with absent or
 // wrong ECR config.
 type ecrResolver struct {
-	creds      credentials.CredentialsProvider
+	creds      aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -35,13 +35,13 @@ var _ runtime.Resolver = (*ecrResolver)(nil)
 
 // newECRResolver builds a resolver with an injected HTTP client (tests). The
 // client is used as-is and never rebuilt from caPath.
-func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+func newECRResolver(creds aws.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
 }
 
 // newLazyECRResolver builds a resolver that constructs its gateway HTTP client
 // from caPath on first use (production path).
-func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+func newLazyECRResolver(creds aws.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
 }
 

spinifex/agents/ecs/agent/ecrresolver_test.go

@@ -14,16 +14,16 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )
 
-// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+// stubCreds is a aws.CredentialsProvider returning fixed creds or error.
 type stubCreds struct {
-	c   credentials.Credentials
+	c   aws.Credentials
 	err error
 }
 
-func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+func (s stubCreds) Retrieve(context.Context) (aws.Credentials, error) {
 	return s.c, s.err
 }
 
@@ -58,9 +58,9 @@ func trusting(srv *httptest.Server) *http.Client {
 
 func TestECRResolver_Authorize(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 
@@ -85,7 +85,7 @@ func TestECRResolver_CredsError(t *testing.T) {
 
 func TestECRResolver_GatewayError(t *testing.T) {
 	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
 		t.Fatal("expected gateway error")
@@ -93,7 +93,7 @@ func TestECRResolver_GatewayError(t *testing.T) {
 }
 
 func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	// Construction with a bogus CA path must not fail — only Authorize does.
 	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
@@ -141,9 +141,9 @@ func TestIsECRRef(t *testing.T) {
 // registry, even though it lacks a ".dkr.ecr." name.
 func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 

spinifex/agents/ecs/credentials/imds.go

@@ -1,41 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/mulgadc/spinifex/internal/imdscreds"
-)
-
-// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
-// which refetches once the real expiry passes. A failed refresh is an error
-// here, not an extended expiry. Safe for concurrent use.
-type IMDSProvider struct {
-	cache *aws.CredentialsCache
-}
-
-var _ CredentialsProvider = (*IMDSProvider)(nil)
-
-// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
-// default HTTP client.
-func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	provider := imdscreds.NewProvider(client, base)
-	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
-}
-
-// Retrieve returns cached credentials when still valid, else fetches fresh ones.
-func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	creds, err := p.cache.Retrieve(ctx)
-	if err != nil {
-		return Credentials{}, err
-	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
-}

spinifex/agents/ecs/credentials/imds_test.go

@@ -1,164 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
-// real /latest prefix the SDK IMDS client always requests.
-func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
-	t.Helper()
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		// Real IMDS echoes the requested TTL back as a response header; the SDK
-		// client parses it from there, not the body.
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits != nil {
-			atomic.AddInt32(hits, 1)
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-func TestRetrieve_FetchesAndCaches(t *testing.T) {
-	var hits int32
-	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
-	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		"Expiration":      exp.Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	got, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("Retrieve: %v", err)
-	}
-	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
-		t.Errorf("creds mismatch: %+v", got)
-	}
-
-	// Second call well within validity must not re-hit IMDS.
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatalf("Retrieve 2: %v", err)
-	}
-	if hits != 1 {
-		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
-	}
-}
-
-func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
-	var hits int32
-	// Already expired: aws.CredentialsCache refetches once the stored
-	// credentials' real expiry has passed, regardless of ExpiryWindow.
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
-	}
-}
-
-// imdsStubFailAfterFirst serves creds on the first credential request and 500s
-// on every one after it, so a test can drive a refresh that fails against a
-// credential the cache has already stored.
-func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
-	t.Helper()
-	var hits atomic.Int32
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits.Add(1) > 1 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// A refresh that fails must surface as an error. Left to itself the SDK's
-// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
-// key with Expires pushed 5-15 minutes out, so the agent would sign requests
-// with a credential the control plane has already rejected — indefinitely,
-// since every subsequent refresh fails the same way.
-func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
-	srv := imdsStubFailAfterFirst(t, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		// Already expired, so the next Retrieve must go back to IMDS.
-		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	first, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("first Retrieve: %v", err)
-	}
-
-	got, err := p.Retrieve(context.Background())
-	if err == nil {
-		t.Fatalf("refresh failure was reported as success: %+v", got)
-	}
-	if got.Expiration.After(first.Expiration) {
-		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
-	}
-}
-
-func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	if _, err := p.Retrieve(ctx); err == nil {
-		t.Fatal("expected error from cancelled context")
-	}
-}
-
-func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err == nil {
-		t.Fatal("expected error when Code != Success")
-	}
-}

spinifex/agents/ecs/credentials/provider.go

@@ -1,24 +0,0 @@
-// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
-// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
-// with the host's instance-role credentials, fetched from IMDS and cached until
-// near expiry.
-package credentials
-
-import (
-	"context"
-	"time"
-)
-
-// Credentials is a SigV4 credential set with an expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
-// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
-// must be safe for concurrent use.
-type CredentialsProvider interface {
-	Retrieve(ctx context.Context) (Credentials, error)
-}

spinifex/agents/eks/konnectivitycert/cert.go

@@ -80,7 +80,17 @@ func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.I
 	if err != nil {
 		return "", "", err
 	}
-	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}, caCert, caKey)
 	if err != nil {
 		return "", "", err
 	}
@@ -181,8 +191,9 @@ func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
 	return ec, nil
 }
 
-// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
-func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+// Mint generates an ECDSA-P256 key, issues tmpl for it with a random serial and
+// signs it with parentKey, or self-signs when parent is nil. tmpl is not modified.
+func Mint(tmpl, parent *x509.Certificate, parentKey *ecdsa.PrivateKey) (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("generate key: %w", err)
@@ -191,19 +202,12 @@ func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns
 	if err != nil {
 		return nil, nil, fmt.Errorf("serial: %w", err)
 	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: cn},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		DNSNames:              dns,
-		IPAddresses:           ips,
+	t := *tmpl
+	t.SerialNumber = serial
+	if parent == nil {
+		parent, parentKey = &t, priv
 	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	der, err := x509.CreateCertificate(rand.Reader, &t, parent, &priv.PublicKey, parentKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("create cert: %w", err)
 	}

spinifex/agents/eks/podidentity/certs.go

@@ -1,15 +1,11 @@
 package podidentity
 
 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"path/filepath"
@@ -83,17 +79,8 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 	if leafCurrent(certPath, keyPath, ca, spec) {
 		return nil
 	}
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return fmt.Errorf("serial: %w", err)
-	}
 	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
 		Subject:               pkix.Name{CommonName: spec.cn},
 		NotBefore:             now.Add(-time.Hour),
 		NotAfter:              now.Add(certValidity),
@@ -101,20 +88,15 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 		ExtKeyUsage:           []x509.ExtKeyUsage{spec.usage},
 		BasicConstraintsValid: true,
 		IPAddresses:           spec.ips,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, ca, &priv.PublicKey, caKey)
+	}, ca, caKey)
 	if err != nil {
-		return fmt.Errorf("create certificate: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return fmt.Errorf("marshal key: %w", err)
-	}
-	// Key first: a cert without its key is never published.
-	if err := writeAtomic(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
 		return err
 	}
-	return writeAtomic(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
+	// Key first: a cert without its key is never published.
+	if err := writeAtomic(keyPath, keyPEM, 0o600); err != nil {
+		return err
+	}
+	return writeAtomic(certPath, certPEM, 0o644)
 }
 
 // leafCurrent reports whether the cached pair loads, matches, was signed by ca,

spinifex/agents/eks/tokenwebhook/cert.go

@@ -1,21 +1,17 @@
 package tokenwebhook
 
 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
-	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"time"
 
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.
@@ -41,7 +37,18 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 		}
 	}
 
-	certPEM, keyPEM, err := generateSelfSigned()
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
+		DNSNames:              []string{"localhost"},
+	}, nil, nil)
 	if err != nil {
 		return tls.Certificate{}, nil, err
 	}
@@ -58,41 +65,6 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }
 
-func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return nil, nil, fmt.Errorf("serial: %w", err)
-	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
-		DNSNames:              []string{"localhost"},
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create cert: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("marshal key: %w", err)
-	}
-	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	return certPEM, keyPEM, nil
-}
-
 // writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
 // --authentication-token-webhook-config-file. It points the apiserver at the
 // loopback webhook and trusts its self-signed serving cert as the CA bundle.

spinifex/agents/eks/webhookcert/cert.go

@@ -14,9 +14,6 @@ package webhookcert
 
 import (
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
@@ -24,11 +21,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math/big"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
 )
 
 // Config names the webhook certificate and where it is cached.
@@ -69,13 +67,25 @@ func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err
 
 	if c, cerr := os.ReadFile(certPath); cerr == nil {
 		if k, kerr := os.ReadFile(keyPath); kerr == nil {
-			if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
-				return c, k, nil
+			if blk, _ := pem.Decode(c); blk != nil && len(k) > 0 {
+				if _, perr := x509.ParseCertificate(blk.Bytes); perr == nil {
+					return c, k, nil
+				}
 			}
 		}
 	}
 
-	certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
+	now := time.Now().UTC()
+	certPEM, keyPEM, err = konnectivitycert.Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		DNSNames:              dnsNames,
+	}, nil, nil)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -91,51 +101,6 @@ func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err
 	return certPEM, keyPEM, nil
 }
 
-// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
-// a server cert, with the given common name and DNS SANs.
-func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return nil, nil, fmt.Errorf("serial: %w", err)
-	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: cn},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-		DNSNames:              dnsNames,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create cert: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("marshal key: %w", err)
-	}
-	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	return certPEM, keyPEM, nil
-}
-
-// decodeFirst returns the DER bytes of the first PEM block, or nil.
-func decodeFirst(pemBytes []byte) []byte {
-	blk, _ := pem.Decode(pemBytes)
-	if blk == nil {
-		return nil
-	}
-	return blk.Bytes
-}
-
 func splitCSV(s string) []string {
 	var out []string
 	for p := range strings.SplitSeq(s, ",") {

spinifex/agents/eks/webhookcert/cert_test.go

@@ -2,6 +2,7 @@ package webhookcert
 
 import (
 	"crypto/x509"
+	"encoding/pem"
 	"os"
 	"path/filepath"
 	"testing"
@@ -14,7 +15,11 @@ const (
 
 func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
 	t.Helper()
-	leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
+	blk, _ := pem.Decode(certPEM)
+	if blk == nil {
+		t.Fatal("no PEM block in cert")
+	}
+	leaf, err := x509.ParseCertificate(blk.Bytes)
 	if err != nil {
 		t.Fatalf("parse cert: %v", err)
 	}

spinifex/domains/admission/quota/override.go

@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case isRevisionConflict(err):
+		case kvutil.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue

spinifex/domains/admission/quota/seal.go

@@ -62,20 +62,6 @@ func (s *Service) checkTerminating(ctx context.Context, accountID string) error
 	return nil
 }
 
-// isRevisionConflict reports a lost CAS race to retry, including the bare
-// wrong-last-sequence APIError from Create's retry against a delete marker.
-// A private copy of domains/ec2/account's helper, which this package may not import.
-func isRevisionConflict(err error) bool {
-	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
-		return true
-	}
-	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
-		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
-			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
-	}
-	return false
-}
-
 // sealBucket returns this gateway's override bucket, or get-or-creates it
 // when quotas are disabled here. Called only after checkTerminating accepts,
 // so a refused seal never creates the bucket.
@@ -150,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case isRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)

spinifex/domains/admission/quota/seal_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"sync"
 	"testing"
 
@@ -433,16 +432,6 @@ func TestSealStoreErrorWritesNothing(t *testing.T) {
 	assert.Equal(t, before, lastOverrideSeq(t, js))
 }
 
-func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
-	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
-	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
-	assert.False(t, isRevisionConflict(nil))
-}
-
 // Clearing never writes when there is nothing active to clear: an absent key,
 // a tombstoned key, and a sealed key hidden behind a stale not-found Get all
 // leave the stream exactly where they found it.

spinifex/domains/ec2/account/seal.go

@@ -8,6 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -58,20 +59,6 @@ func (s *AccountSettingsServiceImpl) checkTerminating(ctx context.Context, accou
 	return nil
 }
 
-// isRevisionConflict reports a lost CAS race SealAccount should retry. Besides
-// the two sentinels, Create's retry against a delete marker returns a bare
-// *jetstream.APIError carrying a wrong-last-sequence code.
-func isRevisionConflict(err error) bool {
-	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
-		return true
-	}
-	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
-		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
-			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
-	}
-	return false
-}
-
 // SealAccount seals accountID's settings record so no later mutation can
 // recreate it. It never relies on a read for its claim: the only success is a
 // leader-accepted write, evidenced by the returned revision.
@@ -122,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case isRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/seal_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"sync"
 	"testing"
 
@@ -397,16 +396,6 @@ func TestSealStoreErrorWritesNothing(t *testing.T) {
 	assert.Equal(t, before, lastSettingsSeq(t, svc))
 }
 
-func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
-	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
-	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
-	assert.False(t, isRevisionConflict(nil))
-}
-
 // An account IAM does not hold as TERMINATING refuses the seal before any
 // settings-KV write, whether the key is present or absent.
 func TestSealRefusedForActiveAccount(t *testing.T) {

spinifex/domains/ec2/accountinventory/compute.go

@@ -0,0 +1,88 @@
+package accountinventory
+
+import (
+	"context"
+	"fmt"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// SpotRequestScanner reads one account's spot requests, failing on any record
+// it cannot read.
+type SpotRequestScanner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2spot.AccountScan, error)
+}
+
+// VolumeScanner reads one account's volume documents, failing on any document
+// it cannot read.
+type VolumeScanner interface {
+	ListVolumesStrict(ctx context.Context, accountID string) ([]ec2metadata.Volume, error)
+}
+
+var (
+	_ SpotRequestScanner = (*ec2spot.SpotInstanceServiceImpl)(nil)
+	_ VolumeScanner      = (*ec2metadata.Store)(nil)
+)
+
+// SpotRequestItem is one request the account holds.
+type SpotRequestItem struct {
+	RequestID string
+	State     string
+	Active    bool
+}
+
+// VolumeItem is one volume document the account holds.
+type VolumeItem struct {
+	VolumeID            string
+	State               string
+	AttachedInstance    string
+	DeleteOnTermination bool
+}
+
+// WithCompute adds the spot-request and volume owner stores.
+func (i *Inventory) WithCompute(spot SpotRequestScanner, volumes VolumeScanner) *Inventory {
+	i.spot = spot
+	i.volumes = volumes
+	return i
+}
+
+// SpotRequestInventory returns every spot request the account holds. A scan
+// that cannot be completed is an error, never a shorter list.
+func (i *Inventory) SpotRequestInventory(ctx context.Context, accountID string) ([]SpotRequestItem, error) {
+	if i.spot == nil {
+		return nil, fmt.Errorf("spot request inventory is not configured")
+	}
+	scan, err := i.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("spot request inventory incomplete: %w", err)
+	}
+	items := make([]SpotRequestItem, 0, len(scan.Active)+len(scan.Terminal))
+	for _, r := range scan.Active {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State, Active: true})
+	}
+	for _, r := range scan.Terminal {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State})
+	}
+	return items, nil
+}
+
+// VolumeInventory returns every volume document the account holds, including
+// those already marked deleting. An unreadable document is an error.
+func (i *Inventory) VolumeInventory(ctx context.Context, accountID string) ([]VolumeItem, error) {
+	if i.volumes == nil {
+		return nil, fmt.Errorf("volume inventory is not configured")
+	}
+	docs, err := i.volumes.ListVolumesStrict(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("volume inventory incomplete: %w", err)
+	}
+	items := make([]VolumeItem, 0, len(docs))
+	for _, v := range docs {
+		items = append(items, VolumeItem{
+			VolumeID: v.VolumeID, State: v.State, AttachedInstance: v.AttachedInstance,
+			DeleteOnTermination: v.DeleteOnTermination,
+		})
+	}
+	return items, nil
+}

spinifex/domains/ec2/accountinventory/compute_test.go

@@ -0,0 +1,82 @@
+package accountinventory
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+type fakeSpot struct {
+	scan *ec2spot.AccountScan
+	err  error
+}
+
+func (f fakeSpot) ScanAccount(context.Context, string) (*ec2spot.AccountScan, error) {
+	return f.scan, f.err
+}
+
+type fakeVolumes struct {
+	docs []ec2metadata.Volume
+	err  error
+}
+
+func (f fakeVolumes) ListVolumesStrict(context.Context, string) ([]ec2metadata.Volume, error) {
+	return f.docs, f.err
+}
+
+func TestSpotRequestInventoryMarksActiveAndTerminal(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{scan: &ec2spot.AccountScan{
+		Active:   []ec2spot.AccountSpotRequest{{RequestID: "sir-1", State: "active"}},
+		Terminal: []ec2spot.AccountSpotRequest{{RequestID: "sir-2", State: "cancelled"}},
+	}}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []SpotRequestItem{
+		{RequestID: "sir-1", State: "active", Active: true},
+		{RequestID: "sir-2", State: "cancelled"},
+	}, items)
+}
+
+func TestSpotRequestInventoryScanFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{err: errList}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestVolumeInventoryReportsAttachmentAndDeletingDocuments(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{docs: []ec2metadata.Volume{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []VolumeItem{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}, items)
+}
+
+func TestVolumeInventoryListFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{err: errList})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestComputeInventoriesRefuseWhenNotConfigured(t *testing.T) {
+	inv := New(nil, nil, nil, nil)
+	_, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.Error(t, err)
+	_, err = inv.VolumeInventory(t.Context(), testAccount)
+	require.Error(t, err)
+}

spinifex/domains/ec2/accountinventory/inventory.go

@@ -61,6 +61,8 @@ type Inventory struct {
 	groups    PlacementGroupScanner
 	templates LaunchTemplateScanner
 	tags      TagIndexScanner
+	spot      SpotRequestScanner
+	volumes   VolumeScanner
 	now       func() time.Time
 }
 

spinifex/domains/ec2/awsapi/account/DescribeAccountAttributes.go

@@ -4,6 +4,7 @@
 package account
 
 import (
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"log/slog"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -14,12 +15,7 @@ import (
 // but default-vpc is static; defaultVPCID looks that one up, is called only
 // when it is asked for, and returns "" for an account with no default VPC.
 func DescribeAccountAttributes(input *ec2.DescribeAccountAttributesInput, defaultVPCID func() (string, error)) (*ec2.DescribeAccountAttributesOutput, error) {
-	requestedAttrs := make(map[string]bool)
-	for _, name := range input.AttributeNames {
-		if name != nil {
-			requestedAttrs[*name] = true
-		}
-	}
+	requestedAttrs := identifiers.Set(input.AttributeNames)
 
 	returnAll := len(requestedAttrs) == 0
 

spinifex/domains/ec2/awsapi/actions.go

@@ -151,7 +151,7 @@ var ec2Actions = map[string]ec2Action{
 		launchQuotaCheck := func() error {
 			return c.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount)))
 		}
-		reservation, err := ec2instanceapi.RunInstances(ctx, input, c.NATS, c.IAM, accountID, passRoleCheck, launchQuotaCheck, c.ExpectedNodes)
+		reservation, err := ec2instanceapi.RunInstances(ctx, input, c.NATS, c.clientTokenBucket, c.IAM, accountID, passRoleCheck, launchQuotaCheck, c.ExpectedNodes)
 		if err != nil {
 			return nil, err
 		}
@@ -436,7 +436,7 @@ var ec2Actions = map[string]ec2Action{
 		passRoleCheck := func(roleARN string) error {
 			return c.passRole(roleARN)
 		}
-		return ec2spotinstanceapi.RequestSpotInstances(ctx, input, c.NATS, c.IAM, accountID, c.AZ, passRoleCheck, c.Quota, c.ExpectedNodes)
+		return ec2spotinstanceapi.RequestSpotInstances(ctx, input, c.NATS, c.clientTokenBucket, c.IAM, accountID, c.AZ, passRoleCheck, c.Quota, c.ExpectedNodes)
 	}),
 	"DescribeSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.DescribeSpotInstanceRequestsInput, c *call, accountID string) (any, error) {
 		return ec2spotinstanceapi.DescribeSpotInstanceRequests(ctx, input, c.NATS, accountID)

spinifex/domains/ec2/awsapi/idem/idem.go

@@ -4,10 +4,15 @@
 package idem
 
 import (
+	"context"
+	"errors"
+	"log/slog"
 	"reflect"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
+	"github.com/nats-io/nats.go/jetstream"
 )
 
 const (
@@ -54,3 +59,45 @@ func TokenAndParams(input any) (token, paramHash string, ok bool) {
 	clone.FieldByName(tokenField).Set(reflect.Zero(field.Type()))
 	return field.Elem().String(), idempotency.ParamHash(clone.Addr().Interface()), true
 }
+
+// Bucket returns the KV bucket that holds the token records. It is called only
+// when a request carries a token, so tokenless requests never bind it.
+type Bucket func(ctx context.Context) (jetstream.KeyValue, error)
+
+// Run runs work under the input's ClientToken, storing its result under
+// namespace. Requests without a token run unwrapped, which is what AWS does:
+// the token is how a client opts in.
+func Run[Out any](
+	ctx context.Context,
+	bucket Bucket,
+	namespace, accountID string,
+	input any,
+	work func() (Out, error),
+) (Out, error) {
+	var zero Out
+	// Hashed before work runs, so work that mutates its input cannot change what
+	// a retry of the same token hashes to.
+	token, paramHash, ok := TokenAndParams(input)
+	if !ok {
+		return work()
+	}
+
+	kv, kerr := bucket(ctx)
+	if kerr != nil {
+		// Running anyway would create the duplicate the token was sent to
+		// prevent, so this fails rather than degrading to no idempotency.
+		slog.ErrorContext(ctx, "client-token store unavailable", "namespace", namespace, "err", kerr)
+		return zero, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	store := idempotency.NewStore[Out](kv, namespace)
+	output, err := idempotency.Do(ctx, store, accountID, token, paramHash, work)
+	switch {
+	case errors.Is(err, idempotency.ErrParamMismatch):
+		return zero, errors.New(awserrors.ErrorIdempotentParameterMismatch)
+	case errors.Is(err, idempotency.ErrUnavailable):
+		slog.ErrorContext(ctx, "client-token replay unavailable", "namespace", namespace, "token", token, "err", err)
+		return zero, errors.New(awserrors.ErrorServerInternal)
+	}
+	return output, err
+}

spinifex/domains/ec2/awsapi/idem/idem_test.go

@@ -1,11 +1,15 @@
 package idem_test
 
 import (
+	"context"
+	"errors"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -79,3 +83,21 @@ func TestTokenAndParams_LeavesTheInputIntact(t *testing.T) {
 	require.NotNil(t, input.ClientToken)
 	assert.Equal(t, "tok", *input.ClientToken)
 }
+
+// A token the store cannot honour fails the request rather than running it
+// unprotected; a request without a token never touches the store.
+func TestRun_BucketFailure(t *testing.T) {
+	t.Parallel()
+	broken := func(context.Context) (jetstream.KeyValue, error) { return nil, errors.New("no jetstream") }
+	ran := false
+	work := func() (string, error) { ran = true; return "ok", nil }
+
+	_, err := ec2idemapi.Run(t.Context(), broken, "", "acct",
+		&ec2.RunInstancesInput{ClientToken: aws.String("tok")}, work)
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	assert.False(t, ran)
+
+	out, err := ec2idemapi.Run(t.Context(), broken, "", "acct", &ec2.RunInstancesInput{}, work)
+	require.NoError(t, err)
+	assert.Equal(t, "ok", out)
+}

spinifex/domains/ec2/awsapi/idempotency.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log/slog"
 
 	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -43,7 +42,9 @@ func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *
 			if !ok {
 				return nil, errors.New(awserrors.ErrorInternalError)
 			}
-			output, err := runEC2Idempotent(ctx, action, c, accountID, input, func() (Out, error) {
+			// Namespaced by action: one bucket holds every EC2 action's records, and
+			// without the prefix one action's payload could be decoded as another's Out.
+			output, err := ec2idemapi.Run(ctx, c.clientTokenBucket, action, accountID, input, func() (Out, error) {
 				return handler(ctx, input, c, accountID)
 			})
 			if err != nil {
@@ -53,43 +54,3 @@ func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *
 		},
 	}
 }
-
-// runEC2Idempotent runs work under the input's ClientToken. Requests without one
-// run unwrapped, which is what AWS does: the token is how a client opts in.
-func runEC2Idempotent[In, Out any](
-	ctx context.Context,
-	action string,
-	c *call,
-	accountID string,
-	input *In,
-	work func() (Out, error),
-) (Out, error) {
-	var zero Out
-	// Hashed before the handler runs, so a handler that mutates its input cannot
-	// change what a retry of the same token hashes to.
-	token, paramHash, ok := ec2idemapi.TokenAndParams(input)
-	if !ok {
-		return work()
-	}
-
-	kv, kerr := c.clientTokenBucket(ctx)
-	if kerr != nil {
-		// Running anyway would create the duplicate the token was sent to
-		// prevent, so this fails rather than degrading to no idempotency.
-		slog.ErrorContext(ctx, "client-token store unavailable", "action", action, "err", kerr)
-		return zero, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// Namespaced by action: one bucket holds every EC2 action's records, and
-	// without the prefix one action's payload could be decoded as another's Out.
-	store := idempotency.NewStore[Out](kv, action)
-	output, err := idempotency.Do(ctx, store, accountID, token, paramHash, work)
-	switch {
-	case errors.Is(err, idempotency.ErrParamMismatch):
-		return zero, errors.New(awserrors.ErrorIdempotentParameterMismatch)
-	case errors.Is(err, idempotency.ErrUnavailable):
-		slog.ErrorContext(ctx, "client-token replay unavailable", "action", action, "token", token, "err", err)
-		return zero, errors.New(awserrors.ErrorServerInternal)
-	}
-	return output, err
-}

spinifex/domains/ec2/awsapi/instance/DescribeAccountInstancesStrict.go

@@ -0,0 +1,151 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// accountInstanceRecordsSubject serves every non-terminated instance record an
+// account owns from the shared store, whichever node holds the instance.
+const accountInstanceRecordsSubject = "ec2.DescribeAccountInstanceRecords"
+
+// DescribeAccountInstancesStrict lists every non-terminated instance the
+// account owns, or returns a retryable ServiceUnavailable. Existence comes from
+// the shared record store, so an instance no answering node holds is still
+// listed with its recorded state; node answers are joined on for live state.
+// Anything that could make the list shorter than the truth is an error: a
+// configured node that did not answer, a frame from a node outside nodeIDs, a
+// record the owner could not project, or a store that did not answer.
+func DescribeAccountInstancesStrict(ctx context.Context, natsConn *nats.Conn, nodeIDs []string, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) {
+	if len(nodeIDs) == 0 {
+		slog.ErrorContext(ctx, "DescribeInstances: strict listing refused, configured node set unknown")
+		return nil, incompleteListingError()
+	}
+	cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout}
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
+	jsonData, err := json.Marshal(&ec2.DescribeInstancesInput{})
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "marshal describe input: %v", err)
+	}
+
+	records, ok := queryInstanceBucket(ctx, natsConn, accountInstanceRecordsSubject, jsonData, accountID)
+	if !ok {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, instance record store did not answer")
+		return nil, incompleteListingError()
+	}
+
+	// Collect to the deadline with no responder grace, so a reply from a node
+	// outside the set that lands after the configured nodes is still seen.
+	frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, natsmsg.GatherOpts{
+		Timeout:            cfg.fanoutTimeout,
+		AccountID:          accountID,
+		ExpectedResponders: len(nodeIDs),
+		Mode:               natsmsg.CollectUntilDeadline,
+	})
+	if err != nil {
+		return nil, err
+	}
+	live, complete := judgeIdentityCompleteness(ctx, frames, sum, nodeIDs)
+	if stray := unconfiguredNodes(frames, sum, nodeIDs); len(stray) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, frame from a node outside the configured set",
+			"unconfigured_nodes", stray)
+		return nil, incompleteListingError()
+	}
+	if !complete {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, fan-out incomplete", "configured_nodes", len(nodeIDs))
+		return nil, incompleteListingError()
+	}
+
+	joined, nodeOnly, malformed := joinLiveOntoRecords(records, live)
+	if malformed {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, a reservation or instance has no usable identity")
+		return nil, incompleteListingError()
+	}
+	if len(nodeOnly) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, node reports instances with no record",
+			"instances", nodeOnly)
+		return nil, incompleteListingError()
+	}
+	return &ec2.DescribeInstancesOutput{Reservations: joined}, nil
+}
+
+// unconfiguredNodes returns the node IDs that answered, with a success or an
+// error, but are not in nodeIDs.
+func unconfiguredNodes(frames []natsmsg.Frame, sum natsmsg.Summary, nodeIDs []string) []string {
+	var stray []string
+	note := func(id string) {
+		if id != "" && !slices.Contains(nodeIDs, id) && !slices.Contains(stray, id) {
+			stray = append(stray, id)
+		}
+	}
+	for _, frame := range frames {
+		note(frame.NodeID)
+	}
+	for id := range sum.Responders {
+		note(id)
+	}
+	slices.Sort(stray)
+	return stray
+}
+
+// joinLiveOntoRecords starts from the recorded instances and replaces each with
+// the answering node's view of it. Instances only a node reports are returned
+// by ID: a lost record write can cause one, and it must not be dropped. A nil
+// reservation, instance or instance ID is reported as malformed, never skipped.
+func joinLiveOntoRecords(records, live []*ec2.Reservation) ([]*ec2.Reservation, []string, bool) {
+	type entry struct {
+		reservation *ec2.Reservation
+		instance    *ec2.Instance
+	}
+	byID := map[string]*entry{}
+	var order []string
+	malformed := false
+	add := func(reservations []*ec2.Reservation) {
+		for _, res := range reservations {
+			if res == nil {
+				malformed = true
+				continue
+			}
+			for _, inst := range res.Instances {
+				if inst == nil || aws.StringValue(inst.InstanceId) == "" {
+					malformed = true
+					continue
+				}
+				id := aws.StringValue(inst.InstanceId)
+				if existing, ok := byID[id]; ok {
+					existing.instance = inst
+					continue
+				}
+				byID[id] = &entry{reservation: res, instance: inst}
+				order = append(order, id)
+			}
+		}
+	}
+	add(records)
+	recorded := len(order)
+	add(live)
+	nodeOnly := slices.Clone(order[recorded:])
+
+	out := make([]*ec2.Reservation, 0, len(order))
+	for _, id := range order {
+		e := byID[id]
+		out = append(out, &ec2.Reservation{
+			ReservationId: e.reservation.ReservationId,
+			OwnerId:       e.reservation.OwnerId,
+			Groups:        e.reservation.Groups,
+			Instances:     []*ec2.Instance{e.instance},
+		})
+	}
+	return mergeReservationsByID(out), nodeOnly, malformed
+}

spinifex/domains/ec2/awsapi/instance/DescribeAccountInstancesStrict_test.go

@@ -0,0 +1,229 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func strictInstance(id, state string) *ec2.Instance {
+	return &ec2.Instance{InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)}}
+}
+
+func strictReservations(instances ...*ec2.Instance) *ec2.DescribeInstancesOutput {
+	out := &ec2.DescribeInstancesOutput{}
+	for _, inst := range instances {
+		out.Reservations = append(out.Reservations, &ec2.Reservation{
+			ReservationId: aws.String("r-" + aws.StringValue(inst.InstanceId)),
+			OwnerId:       aws.String("123456789012"),
+			Instances:     []*ec2.Instance{inst},
+		})
+	}
+	return out
+}
+
+func subscribeAccountRecords(t *testing.T, nc *nats.Conn, data []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(accountInstanceRecordsSubject, func(msg *nats.Msg) { _ = msg.Respond(data) })
+	require.NoError(t, err)
+}
+
+const strictAccount = "123456789012"
+
+func strictList(nc *nats.Conn, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
+	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(2*time.Second))
+}
+
+// Existence comes from the record store, so an instance no node holds is still
+// listed, with the state the record carries.
+func TestDescribeAccountInstancesStrict_UnheldRecordIsReturnedWithItsRecordedState(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-unheld", "stopped"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	inst := out.Reservations[0].Instances[0]
+	assert.Equal(t, "i-unheld", aws.StringValue(inst.InstanceId))
+	assert.Equal(t, "stopped", aws.StringValue(inst.State.Name))
+}
+
+// A node's answer replaces the recorded view of the instance it holds.
+func TestDescribeAccountInstancesStrict_NodeAnswersAreJoinedOntoRecords(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "pending"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	states := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			states[aws.StringValue(inst.InstanceId)] = aws.StringValue(inst.State.Name)
+		}
+	}
+	assert.Equal(t, map[string]string{"i-held": "running"}, states)
+}
+
+// An instance a node reports with no record can come from a lost record write.
+// Returning it or dropping it would both answer for an unproven account.
+func TestDescribeAccountInstancesStrict_NodeOnlyInstanceFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "running"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"), strictInstance("i-node-only", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
+}
+
+// A frame from a node outside the configured set means the expected set is
+// wrong, and a listing built on it could omit that node's instances.
+func TestDescribeAccountInstancesStrict_FrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A stray node that answers with an error is still a node outside the set, and
+// its instances are unaccounted for.
+func TestDescribeAccountInstancesStrict_ErrorFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// The configured nodes answer at once, so a reply that lands later but inside
+// the fan-out deadline must still be seen.
+func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	late := mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running")))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(600*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = late
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-stray")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A record with no instance ID cannot be listed or counted, so it must fail
+// the read rather than vanish from it.
+func TestDescribeAccountInstancesStrict_RecordWithoutInstanceIDFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(&ec2.Instance{State: &ec2.InstanceState{Name: aws.String("stopped")}})))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_ConfiguredNodeSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1", "node-2")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreErrorFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnknownNodeSetFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc)
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnattributedFrameFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}

spinifex/domains/ec2/awsapi/instance/DescribeInstanceTypes.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
 	"sort"
@@ -74,12 +75,7 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
 		}
 	}
 
-	requestedTypes := make(map[string]bool)
-	for _, it := range input.InstanceTypes {
-		if it != nil {
-			requestedTypes[*it] = true
-		}
-	}
+	requestedTypes := identifiers.Set(input.InstanceTypes)
 
 	var finalInstanceTypes []*ec2.InstanceTypeInfo
 	if showCapacity {

spinifex/domains/ec2/awsapi/instance/DescribeInstances_current_behaviour_test.go

@@ -0,0 +1,63 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Pins today: the completeness judge merges the reservations of a frame from a
+// node outside the configured set and still reports the listing complete. The
+// stray node is never named, so a silent unconfigured node cannot be told from
+// one that does not exist.
+func TestCurrentBehaviour_JudgeIdentityCompleteness_FrameFromUnconfiguredNodeIsMerged(t *testing.T) {
+	t.Parallel()
+	stray := mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{
+		Reservations: []*ec2.Reservation{{ReservationId: aws.String("r-stray"), Instances: []*ec2.Instance{{InstanceId: aws.String("i-on-stray")}}}},
+	})
+	frames := []natsmsg.Frame{
+		{NodeID: "node-1", Data: []byte(`{}`)},
+		{NodeID: "node-stray", Data: stray},
+	}
+
+	reservations, complete := judgeIdentityCompleteness(context.Background(), frames, natsmsg.Summary{}, []string{"node-1"})
+
+	assert.True(t, complete, "CURRENT: a node outside nodeIDs does not make the listing incomplete")
+	require.Len(t, reservations, 1)
+	assert.Equal(t, "i-on-stray", *reservations[0].Instances[0].InstanceId,
+		"CURRENT: the unconfigured node's instances are merged into the result")
+}
+
+// Pins today: the fan-out counts every identified frame toward the expected
+// responders, so a stray frame can end collection before a configured node has
+// answered. The listing then refuses, which fails closed, but for a node that
+// was only slow.
+func TestCurrentBehaviour_DescribeAccountInstancesComplete_StrayFrameConsumesAResponderSlot(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeEmptyInstanceBuckets(t, nc)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", []byte(`{}`))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(500*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = []byte(`{}`)
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-2")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	output, err := DescribeAccountInstancesComplete(context.Background(), nc, []string{"node-1", "node-2"}, "123456789012",
+		WithFanoutTimeout(3*time.Second))
+
+	assertIncompleteListing(t, output, err)
+}

spinifex/domains/ec2/awsapi/instance/RunInstances.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"log/slog"
 	"math"
 	"slices"
 	"strings"
@@ -16,6 +15,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
 	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
@@ -126,12 +126,13 @@ func hasPinnedPrivateIP(input *ec2.RunInstancesInput) bool {
 
 // RunInstances validates input, resolves any IAM instance profile (normalising
 // to ARN for daemons), enforces iam:PassRole, then dispatches via NATS.
-// When ClientToken is set, wraps the launch in idempotency via the KV store.
+// When ClientToken is set, wraps the launch in idempotency via tokenBucket,
+// which may be nil only when no ClientToken is supplied.
 // iamSvc may be nil only when no IamInstanceProfile is supplied.
 // passRoleCheck may be nil to skip PassRole enforcement.
 // launchQuotaCheck may be nil to skip quota enforcement; when present it runs
 // after validation and PassRole authorization, but before any launch dispatch.
-func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
+func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, tokenBucket ec2idemapi.Bucket, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
 	// Expand a referenced launch template before validation so validation, quota,
 	// and downstream routing all see the effective (post-merge) parameters.
 	if err = ExpandLaunchTemplate(ctx, natsConn, input, accountID); err != nil {
@@ -142,21 +143,10 @@ func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *n
 		return reservation, err
 	}
 
-	token := aws.StringValue(input.ClientToken)
-	if token == "" {
-		return runInstancesInner(ctx, input, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
-	}
-
-	// ClientToken set: dedup concurrent/retried launches; store failure is fatal
-	// to avoid a double launch on retry.
-	store, serr := getClientTokenStore(ctx, natsConn)
-	if serr != nil {
-		slog.ErrorContext(ctx, "RunInstances: client-token store unavailable", "err", serr)
-		return reservation, errors.New(awserrors.ErrorServerInternal)
-	}
-	// Hash before any mutation so the same token+params always matches.
-	paramHash := clientTokenParamHash(input)
-	return runInstancesWithClientToken(ctx, store, accountID, token, paramHash, func() (ec2.Reservation, error) {
+	// Unnamespaced: every other EC2 action namespaces its keys by action, but
+	// RunInstances predates that, and re-keying live records would make a retry
+	// re-launch instead of replay. Hashed here, after template expansion.
+	return ec2idemapi.Run(ctx, tokenBucket, "", accountID, input, func() (ec2.Reservation, error) {
 		return runInstancesInner(ctx, input, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
 	})
 }

spinifex/domains/ec2/awsapi/instance/RunInstances_test.go

@@ -347,7 +347,7 @@ func TestParseRunInstances(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 			// For validation tests, we can pass nil conn since validation happens before NATS call
-			response, err := RunInstances(context.Background(), test.input, nil, nil, "123456789012", nil, nil, 1)
+			response, err := RunInstances(context.Background(), test.input, nil, nil, nil, "123456789012", nil, nil, 1)
 
 			// Use assert to check if the error is as expected
 			assert.Equal(t, test.want, err)

spinifex/domains/ec2/awsapi/instance/clienttoken.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// ClientTokenStore is the RunInstances idempotency store: the first caller owns
-// the launch, duplicates replay its reservation or poll for it.
-type ClientTokenStore = idempotency.Store[ec2.Reservation]
-
-var (
-	ctStore        *ClientTokenStore
-	ctOnce         sync.Once
-	errCTStoreInit error
-)
-
-// getClientTokenStore lazily initialises the process-wide client-token store via
-// sync.Once. Process-wide because RunInstances is a free function with no
-// instance to hang it on.
-func getClientTokenStore(ctx context.Context, nc *nats.Conn) (*ClientTokenStore, error) {
-	ctOnce.Do(func() {
-		js, err := jetstream.New(nc)
-		if err != nil {
-			errCTStoreInit = fmt.Errorf("clienttoken jetstream: %w", err)
-			return
-		}
-		// The bind happens once per process, so it must not inherit the first
-		// caller's cancellation: a client that disconnects mid-open would poison
-		// the store for every later launch. Deadline-free, so the open falls back
-		// to the JetStream API's own timeout.
-		ctStore, errCTStoreInit = newClientTokenStore(context.WithoutCancel(ctx), js)
-	})
-	return ctStore, errCTStoreInit
-}
-
-// newClientTokenStore takes the bucket unnamespaced. Every other EC2 action
-// namespaces its keys by action, but RunInstances predates that: re-keying live
-// records would make a retry re-launch instead of replay.
-func newClientTokenStore(ctx context.Context, js jetstream.JetStream) (*ClientTokenStore, error) {
-	return idempotency.OpenStore[ec2.Reservation](ctx, js, ec2idemapi.KVBucket, ec2idemapi.TTL)
-}
-
-// clientTokenParamHash hashes the request excluding ClientToken, so the same
-// params always produce the same hash. Must run before any input mutation.
-func clientTokenParamHash(input *ec2.RunInstancesInput) string {
-	clone := *input
-	clone.ClientToken = nil
-	return idempotency.ParamHash(&clone)
-}
-
-// runInstancesWithClientToken wraps a launch in ClientToken idempotency and maps
-// the token errors onto the AWS ones RunInstances returns. Extracted for
-// unit-testability.
-func runInstancesWithClientToken(
-	ctx context.Context,
-	store *ClientTokenStore,
-	accountID, token, paramHash string,
-	launch func() (ec2.Reservation, error),
-) (ec2.Reservation, error) {
-	res, err := idempotency.Do(ctx, store, accountID, token, paramHash, launch)
-	switch {
-	case errors.Is(err, idempotency.ErrParamMismatch):
-		return ec2.Reservation{}, errors.New(awserrors.ErrorIdempotentParameterMismatch)
-	case errors.Is(err, idempotency.ErrUnavailable):
-		slog.ErrorContext(ctx, "RunInstances: client-token claim failed", "token", token, "err", err)
-		return ec2.Reservation{}, errors.New(awserrors.ErrorServerInternal)
-	}
-	return res, err
-}

spinifex/domains/ec2/awsapi/instance/clienttoken_test.go

@@ -1,155 +1,55 @@
 package instance
 
 import (
-	"errors"
+	"context"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 const ctTestAccount = "111122223333"
 
-func newTestClientTokenStore(t *testing.T) *ClientTokenStore {
-	t.Helper()
+// RunInstances records predate per-action namespacing: they live at the bare
+// account/token key, hashed over the request minus its token. A record written
+// that way must still replay, or a retry across an upgrade would re-launch.
+func TestRunInstances_ClientTokenReplaysUnnamespacedRecord(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
-	store, err := newClientTokenStore(t.Context(), testutil.NewJetStream(t, nc))
+	kv, err := idempotency.OpenBucket(t.Context(), testutil.NewJetStream(t, nc), "ec2-tokens", time.Minute)
 	require.NoError(t, err)
-	return store
-}
+	bucket := func(context.Context) (jetstream.KeyValue, error) { return kv, nil }
 
-// The claim/replay/abort mechanics live in spinifex/foundation/lifecycle/idempotency and are tested
-// there. What is EC2's own is the param hash, the launch orchestration around
-// the store, and the AWS error a mismatch maps onto.
+	const tok = "rt-1"
+	input := defaults
+	input.ClientToken = aws.String(tok)
+	unTokened := input
+	unTokened.ClientToken = nil
+	hash := idempotency.ParamHash(&unTokened)
 
-// clientTokenParamHash ignores ClientToken (same params, different token →
-// same hash) but reflects a real parameter change.
-func TestClientTokenParamHash_IgnoresTokenReflectsParams(t *testing.T) {
-	t.Parallel()
-	base := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-123"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		ClientToken:  aws.String("tok-A"),
-	}
-	sameParamsDiffToken := *base
-	sameParamsDiffToken.ClientToken = aws.String("tok-B")
-	diffParams := *base
-	diffParams.InstanceType = aws.String("t3.large")
-
-	assert.Equal(t, clientTokenParamHash(base), clientTokenParamHash(&sameParamsDiffToken),
-		"token must not affect the param hash")
-	assert.NotEqual(t, clientTokenParamHash(base), clientTokenParamHash(&diffParams),
-		"a real param change must change the hash")
-}
-
-// The hash is taken from a copy, so hashing must not clear the caller's token.
-func TestClientTokenParamHash_LeavesTheInputAlone(t *testing.T) {
-	t.Parallel()
-	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-123"), ClientToken: aws.String("tok-A")}
-
-	clientTokenParamHash(input)
-
-	assert.Equal(t, "tok-A", aws.StringValue(input.ClientToken),
-		"hashing must not strip the token the launch still needs")
-}
-
-// A completed token replays its reservation and must NOT invoke the launcher.
-func TestRunInstancesWithClientToken_ReplaySkipsLaunch(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-1", "h"
+	store := idempotency.NewStore[ec2.Reservation](kv, "")
 	_, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash)
 	require.NoError(t, err)
 	require.True(t, owned)
 	require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, hash, ec2.Reservation{ReservationId: aws.String("r-x")}))
+	_, err = kv.Get(t.Context(), idempotency.Key(ctTestAccount, tok))
+	require.NoError(t, err, "record must sit at the unprefixed key")
 
-	launched := false
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		launched = true
-		return ec2.Reservation{}, nil
-	})
+	// A nil NATS connection: replay must not reach the launch path.
+	replay := input
+	res, err := RunInstances(t.Context(), &replay, nil, bucket, nil, ctTestAccount, nil, nil, 1)
 	require.NoError(t, err)
-	assert.False(t, launched, "replay must not launch")
 	assert.Equal(t, "r-x", aws.StringValue(res.ReservationId))
-}
 
-// The owner launches once and finalizes; a duplicate replays without launching.
-func TestRunInstancesWithClientToken_OwnerLaunchesOnceThenReplay(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-2", "h"
-	launches := 0
-	launch := func() (ec2.Reservation, error) {
-		launches++
-		return ec2.Reservation{ReservationId: aws.String("r-own")}, nil
-	}
-
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch)
-	require.NoError(t, err)
-	assert.Equal(t, "r-own", aws.StringValue(res.ReservationId))
-
-	res2, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch)
-	require.NoError(t, err)
-	assert.Equal(t, "r-own", aws.StringValue(res2.ReservationId))
-	assert.Equal(t, 1, launches, "duplicate must replay, not relaunch")
-}
-
-// A launch failure aborts the token so a retry re-launches.
-func TestRunInstancesWithClientToken_LaunchFailureAborts(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-3", "h"
-
-	_, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		return ec2.Reservation{}, errors.New("no capacity")
-	})
-	require.Error(t, err)
-
-	relaunched := false
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		relaunched = true
-		return ec2.Reservation{ReservationId: aws.String("r-retry")}, nil
-	})
-	require.NoError(t, err)
-	assert.True(t, relaunched, "after abort the retry must launch")
-	assert.Equal(t, "r-retry", aws.StringValue(res.ReservationId))
-}
-
-// Token reuse with different params maps to the AWS IdempotentParameterMismatch
-// error code and never launches.
-func TestRunInstancesWithClientToken_ParamMismatchMapsAWSError(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok = "rt-4"
-	_, owned, err := store.Claim(t.Context(), ctTestAccount, tok, "hA")
-	require.NoError(t, err)
-	require.True(t, owned)
-	require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, "hA", ec2.Reservation{ReservationId: aws.String("r")}))
-
-	launched := false
-	_, err = runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, "hB", func() (ec2.Reservation, error) {
-		launched = true
-		return ec2.Reservation{}, nil
-	})
+	changed := input
+	changed.InstanceType = aws.String("t3.large")
+	_, err = RunInstances(t.Context(), &changed, nil, bucket, nil, ctTestAccount, nil, nil, 1)
 	require.EqualError(t, err, awserrors.ErrorIdempotentParameterMismatch)
-	assert.False(t, launched)
-}
-
-// getClientTokenStore binds the process-wide store once and returns the same
-// instance on subsequent calls.
-func TestGetClientTokenStore_BindsOnce(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	s1, err := getClientTokenStore(t.Context(), nc)
-	require.NoError(t, err)
-	require.NotNil(t, s1)
-	s2, err := getClientTokenStore(t.Context(), nc)
-	require.NoError(t, err)
-	assert.Same(t, s1, s2, "store binds once")
 }

spinifex/domains/ec2/awsapi/instance/placement_test.go

@@ -830,7 +830,7 @@ func TestRunInstances_SingleInstanceDistributes(t *testing.T) {
 		MaxCount:     aws.Int64(1),
 	}
 
-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 1)
 	assert.Equal(t, "i-single", aws.StringValue(reservation.Instances[0].InstanceId))
@@ -1383,7 +1383,7 @@ func TestRunInstances_ClusterPlacementGroupRouting(t *testing.T) {
 		},
 	}
 
-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 2)
 }
@@ -1432,7 +1432,7 @@ func TestRunInstances_MultiInstanceUsesDistribution(t *testing.T) {
 		MaxCount:     aws.Int64(2),
 	}
 
-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 2)
 	assert.True(t, statusQueried, "multi-instance launch should query node status")

spinifex/domains/ec2/awsapi/spotinstance/RequestSpotInstances.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -62,7 +63,7 @@ func ValidateRequestSpotInstancesInput(input *ec2.RequestSpotInstancesInput) err
 // ClientToken passed through, same vCPU quota gate), then builds and persists one
 // active/fulfilled SpotInstanceRequest per launched instance. On a launch failure
 // (including InsufficientInstanceCapacity) it returns the error and persists nothing.
-func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, iamSvc ec2instanceapi.InstanceProfiles, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota LaunchQuota, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
+func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, tokenBucket ec2idemapi.Bucket, iamSvc ec2instanceapi.InstanceProfiles, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota LaunchQuota, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
 	var output ec2.RequestSpotInstancesOutput
 
 	if err := ValidateRequestSpotInstancesInput(input); err != nil {
@@ -80,7 +81,7 @@ func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesIn
 
 	// RunInstances normalises runInput in place (e.g. instance profile to ARN),
 	// so the launch spec echoed back is built from runInput afterwards.
-	reservation, err := ec2instanceapi.RunInstances(ctx, runInput, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
+	reservation, err := ec2instanceapi.RunInstances(ctx, runInput, natsConn, tokenBucket, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
 	if err != nil {
 		return output, err
 	}

spinifex/domains/ec2/awsapi/spotinstance/quota_gate_order_test.go

@@ -125,7 +125,7 @@ func TestRequestSpotInstances_QuotaGateRunsBeforeDispatchAndLaunch(t *testing.T)
 		InstanceCount: aws.Int64(1),
 	}
 
-	_, err = RequestSpotInstances(context.Background(), input, nc, nil, "test-account", "ap-southeast-2a", nil, quota, 1)
+	_, err = RequestSpotInstances(context.Background(), input, nc, nil, nil, "test-account", "ap-southeast-2a", nil, quota, 1)
 	require.NoError(t, err)
 
 	quotaIdx := order.indexOf("quota-check")

spinifex/domains/ec2/ebs/obligation/obligation.go

@@ -0,0 +1,185 @@
+// Package ebsobligation records the volume exports a detach left in place.
+//
+// A force detach, or a terminate whose unmount did not complete, clears a
+// volume's attachment while the node that served it may still export it. The
+// request cannot reach that node, so it records an UnpublishObligation here and
+// returns; the named node settles it once it answers.
+package ebsobligation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bucket holds one obligation per publish a cleared attachment left behind.
+// It has no TTL: an obligation ends only when it is settled or closed.
+const Bucket = "spinifex-ebs-unpublish-obligations"
+
+// BucketVersion is the bucket schema version RunKV stamps.
+const BucketVersion = 1
+
+// SchemaVersion is the version of the UnpublishObligation record.
+const SchemaVersion uint16 = 1
+
+// UnpublishObligation names a publish a cleared attachment left on a node: the
+// node, volume and handle, and the attachment it served. The attachment is what
+// tells it apart from a later publish of the same volume on the same node.
+type UnpublishObligation struct {
+	SchemaVersion uint16    `json:"schema_version"`
+	VolumeID      string    `json:"volume_id"`
+	AccountID     string    `json:"account_id"`
+	Handle        string    `json:"handle,omitempty"` // ProviderHandle when the attachment was cleared
+	NodeID        string    `json:"node_id"`          // the node holding the publish
+	InstanceID    string    `json:"instance_id"`      // the attachment that was cleared
+	AttachedAt    time.Time `json:"attached_at"`
+	RecordedAt    time.Time `json:"recorded_at"`
+	Attempts      int       `json:"attempts"`
+	Error         string    `json:"error,omitempty"`
+}
+
+// keyToken is one NATS subject token a key segment may be: the key is three
+// tokens, so a segment holding a dot would split it and escape the filters.
+var keyToken = regexp.MustCompile(`^[-_=a-zA-Z0-9]+$`)
+
+// Key is the obligation's key, <node>.<volume-id>.<attached-at>, with the
+// attachment time in Unix nanoseconds and an unknown time as 0.
+func Key(o UnpublishObligation) (string, error) {
+	if !keyToken.MatchString(o.NodeID) {
+		return "", fmt.Errorf("unpublish obligation: node %q is not a valid key token", o.NodeID)
+	}
+	if !keyToken.MatchString(o.VolumeID) {
+		return "", fmt.Errorf("unpublish obligation: volume %q is not a valid key token", o.VolumeID)
+	}
+	var nanos int64
+	if !o.AttachedAt.IsZero() {
+		nanos = o.AttachedAt.UnixNano()
+	}
+	return o.NodeID + "." + o.VolumeID + "." + strconv.FormatInt(nanos, 10), nil
+}
+
+// allKeys matches every obligation key and not the bucket's version stamp,
+// which is a single token.
+const allKeys = "*.*.*"
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        Bucket,
+		Description: "EBS exports a cleared attachment left on a node, until that node unpublishes them",
+		History:     1,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, Bucket, kv, BucketVersion)
+		},
+		Missing: "EBS unpublish obligation bucket not initialized",
+	}
+}
+
+// Store reads and writes the obligation bucket.
+type Store struct {
+	s *kvstore.Store[UnpublishObligation]
+}
+
+// NewStore returns a Store over js. The bucket is created on first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{s: kvstore.New[UnpublishObligation](js, bucketConfig())}
+}
+
+// Open creates or opens the bucket now, so a node that cannot reach it fails
+// at startup rather than on its first force detach.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.s.KV(ctx)
+	return err
+}
+
+// Record writes o if no obligation with its key exists. An existing key is the
+// same obligation written by an earlier attempt, so it counts as written.
+func (s *Store) Record(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	o.SchemaVersion = SchemaVersion
+	if _, err := s.s.Create(ctx, key, &o); err != nil && !errors.Is(err, kvstore.ErrExists) {
+		return fmt.Errorf("record unpublish obligation %s: %w", key, err)
+	}
+	return nil
+}
+
+// ListNode returns the open obligations naming node.
+func (s *Store) ListNode(ctx context.Context, node string) ([]UnpublishObligation, error) {
+	if !keyToken.MatchString(node) {
+		return nil, fmt.Errorf("unpublish obligation: node %q is not a valid key token", node)
+	}
+	return s.list(ctx, node+".*.*")
+}
+
+// List returns every open obligation.
+func (s *Store) List(ctx context.Context) ([]UnpublishObligation, error) {
+	return s.list(ctx, allKeys)
+}
+
+func (s *Store) list(ctx context.Context, filter string) ([]UnpublishObligation, error) {
+	items, _, err := s.s.Snapshot(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]UnpublishObligation, 0, len(items))
+	for _, item := range items {
+		out = append(out, item.Value)
+	}
+	return out, nil
+}
+
+// Close removes o. Removing one already gone is not an error.
+func (s *Store) Close(ctx context.Context, o UnpublishObligation) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	return s.s.Delete(ctx, key)
+}
+
+// RecordFailure counts a failed settlement on o and keeps its error, so an
+// operator listing the obligations sees why it is still open.
+func (s *Store) RecordFailure(ctx context.Context, o UnpublishObligation, cause error) error {
+	key, err := Key(o)
+	if err != nil {
+		return err
+	}
+	err = s.s.Mutate(ctx, key, func(cur *UnpublishObligation) (bool, error) {
+		cur.Attempts++
+		cur.Error = cause.Error()
+		return true, nil
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// ListOpen reads the open obligations for an operator tool without creating
+// the bucket: a cluster that has never force-detached a volume has none, and
+// that is not an error.
+func ListOpen(ctx context.Context, nc *nats.Conn) ([]UnpublishObligation, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("jetstream: %w", err)
+	}
+	if _, err := js.KeyValue(ctx, Bucket); errors.Is(err, jetstream.ErrBucketNotFound) {
+		return nil, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("unpublish obligation bucket: %w", err)
+	}
+	cfg := bucketConfig()
+	cfg.AttachOnly = true
+	cfg.OnOpen = nil
+	return (&Store{s: kvstore.New[UnpublishObligation](js, cfg)}).List(ctx)
+}

spinifex/domains/ec2/ebs/obligation/obligation_test.go

@@ -0,0 +1,141 @@
+package ebsobligation_test
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsobligation "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/obligation"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newStore(t *testing.T) *ebsobligation.Store {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	return store
+}
+
+func obligation(node, volumeID string, attachedAt time.Time) ebsobligation.UnpublishObligation {
+	return ebsobligation.UnpublishObligation{
+		VolumeID: volumeID, AccountID: "123456789012", NodeID: node,
+		InstanceID: "i-1", AttachedAt: attachedAt, RecordedAt: attachedAt.Add(time.Minute),
+	}
+}
+
+func TestKey_NamesNodeVolumeAndAttachment(t *testing.T) {
+	at := time.Unix(1_700_000_000, 5)
+	key, err := ebsobligation.Key(obligation("node-a", "vol-1", at))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.1700000000000000005", key)
+
+	key, err = ebsobligation.Key(obligation("node-a", "vol-1", time.Time{}))
+	require.NoError(t, err)
+	assert.Equal(t, "node-a.vol-1.0", key, "an unknown attachment time is 0, not a huge negative")
+}
+
+// A segment holding a subject separator or wildcard would split the key or
+// let one node's listing match another's.
+func TestKey_RejectsSegmentsThatAreNotOneToken(t *testing.T) {
+	for _, o := range []ebsobligation.UnpublishObligation{
+		obligation("node.a", "vol-1", time.Time{}),
+		obligation("node-a", "vol.1", time.Time{}),
+		obligation("*", "vol-1", time.Time{}),
+		obligation("node-a", ">", time.Time{}),
+		obligation("", "vol-1", time.Time{}),
+	} {
+		_, err := ebsobligation.Key(o)
+		assert.Error(t, err, "%s/%s", o.NodeID, o.VolumeID)
+	}
+}
+
+func TestStore_RecordIsIdempotentPerAttachment(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	first := obligation("node-a", "vol-1", at)
+	first.Handle = "h1"
+	require.NoError(t, store.Record(t.Context(), first))
+	retry := first
+	retry.RecordedAt = at.Add(time.Hour)
+	require.NoError(t, store.Record(t.Context(), retry), "a retry of the same obligation is not an error")
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at.Add(time.Second))),
+		"a different attachment of the same volume is a different obligation")
+
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 2)
+	for _, o := range open {
+		assert.Equal(t, ebsobligation.SchemaVersion, o.SchemaVersion)
+		if o.AttachedAt.Equal(at) {
+			assert.True(t, o.RecordedAt.Equal(at.Add(time.Minute)), "the first write is kept")
+		}
+	}
+}
+
+// The bucket's version stamp is a key in the same bucket; listing must not
+// return it as an obligation.
+func TestStore_ListSkipsTheVersionStampAndListNodeIsolatesNodes(t *testing.T) {
+	store := newStore(t)
+	at := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-2", at)))
+	require.NoError(t, store.Record(t.Context(), obligation("node-ab", "vol-3", at)))
+
+	all, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, all, 3)
+
+	mine, err := store.ListNode(t.Context(), "node-a")
+	require.NoError(t, err)
+	volumes := []string{}
+	for _, o := range mine {
+		volumes = append(volumes, o.VolumeID)
+	}
+	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, volumes, "a node whose name prefixes another's sees only its own")
+
+	_, err = store.ListNode(t.Context(), "*")
+	assert.Error(t, err)
+}
+
+func TestStore_CloseAndRecordFailure(t *testing.T) {
+	store := newStore(t)
+	o := obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())
+	require.NoError(t, store.Record(t.Context(), o))
+
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("first")))
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("second")))
+	open, err := store.List(t.Context())
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, 2, open[0].Attempts)
+	assert.Equal(t, "second", open[0].Error)
+
+	require.NoError(t, store.Close(t.Context(), o))
+	require.NoError(t, store.Close(t.Context(), o), "closing one already gone is not an error")
+	require.NoError(t, store.RecordFailure(t.Context(), o, errors.New("late")),
+		"a failure recorded after another sweep closed it does not resurrect it")
+	open, err = store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, open)
+}
+
+func TestListOpen_MissingBucketIsEmptyAndNotCreated(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	open, err := ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	assert.Empty(t, open)
+	_, err = js.KeyValue(t.Context(), ebsobligation.Bucket)
+	assert.Error(t, err, "an operator listing must not create the bucket")
+
+	store := ebsobligation.NewStore(js)
+	require.NoError(t, store.Open(t.Context()))
+	require.NoError(t, store.Record(t.Context(), obligation("node-a", "vol-1", time.Unix(1_700_000_000, 0).UTC())))
+	open, err = ebsobligation.ListOpen(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, open, 1)
+	assert.Equal(t, "vol-1", open[0].VolumeID)
+}

spinifex/domains/ec2/eigw/service_impl.go

@@ -158,12 +158,7 @@ var describeEIGWValidFilters = map[string]bool{
 func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx context.Context, input *ec2.DescribeEgressOnlyInternetGatewaysInput, accountID string) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
 	var egressOnlyIGWs []*ec2.EgressOnlyInternetGateway
 
-	eigwIDs := make(map[string]bool)
-	for _, id := range input.EgressOnlyInternetGatewayIds {
-		if id != nil {
-			eigwIDs[*id] = true
-		}
-	}
+	eigwIDs := awsidentifiers.Set(input.EgressOnlyInternetGatewayIds)
 
 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeEIGWValidFilters)
 	if err != nil {

spinifex/domains/ec2/eip/service_impl.go

@@ -572,19 +572,9 @@ var describeAddressesValidFilters = map[string]bool{
 
 // DescribeAddresses lists Elastic IPs with optional filtering by allocation ID.
 func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.DescribeAddressesInput, accountID string) (*ec2.DescribeAddressesOutput, error) {
-	allocIDs := make(map[string]bool)
-	for _, id := range input.AllocationIds {
-		if id != nil {
-			allocIDs[*id] = true
-		}
-	}
+	allocIDs := awsidentifiers.Set(input.AllocationIds)
 
-	publicIPs := make(map[string]bool)
-	for _, ip := range input.PublicIps {
-		if ip != nil {
-			publicIPs[*ip] = true
-		}
-	}
+	publicIPs := awsidentifiers.Set(input.PublicIps)
 
 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeAddressesValidFilters)
 	if err != nil {

spinifex/domains/ec2/guestmetadata/imdsv1.go

@@ -3,7 +3,6 @@ package guestmetadata
 import (
 	"context"
 	"log/slog"
-	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -14,50 +13,6 @@ import (
 // little and keeps the untokened path off the NATS fan-out.
 const v1AllowTTL = 30 * time.Second
 
-type v1AllowEntry struct {
-	allowed bool
-	expires time.Time
-}
-
-// v1AllowCache memoises whether an instance permits IMDSv1. It exists for
-// amplification control: the untokened path is reachable without credentials, so
-// resolving HttpTokens per request would let any guest drive unbounded
-// DescribeInstances fan-outs.
-type v1AllowCache struct {
-	mu      sync.Mutex
-	entries map[string]v1AllowEntry
-}
-
-func newV1AllowCache() *v1AllowCache {
-	return &v1AllowCache{entries: make(map[string]v1AllowEntry)}
-}
-
-func (c *v1AllowCache) get(instanceID string, now time.Time) (bool, bool) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	e, ok := c.entries[instanceID]
-	if !ok || now.After(e.expires) {
-		return false, false
-	}
-	return e.allowed, true
-}
-
-func (c *v1AllowCache) put(instanceID string, allowed bool, now time.Time) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.entries[instanceID] = v1AllowEntry{allowed: allowed, expires: now.Add(v1AllowTTL)}
-}
-
-func (c *v1AllowCache) sweep(now time.Time) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	for id, e := range c.entries {
-		if now.After(e.expires) {
-			delete(c.entries, id)
-		}
-	}
-}
-
 // imdsV1Allowed reports whether this ENI's instance was launched or modified
 // with HttpTokens=optional, which is the only case where an untokened GET is
 // served. Defaults closed: an unattached ENI, a lookup failure or a missing
@@ -66,7 +21,7 @@ func (s *IMDSServiceImpl) imdsV1Allowed(ctx context.Context, eni *eniFacts) bool
 	if eni.instanceID == "" {
 		return false
 	}
-	if allowed, ok := s.v1Allow.get(eni.instanceID, s.now()); ok {
+	if allowed, ok := s.v1Allow.get(eni.instanceID); ok {
 		return allowed
 	}
 
@@ -79,6 +34,6 @@ func (s *IMDSServiceImpl) imdsV1Allowed(ctx context.Context, eni *eniFacts) bool
 
 	// Cached even on failure: without that, a guest could replay a failing
 	// lookup to keep the fan-out running.
-	s.v1Allow.put(eni.instanceID, allowed, s.now())
+	s.v1Allow.put(eni.instanceID, allowed)
 	return allowed
 }

spinifex/domains/ec2/guestmetadata/imdsv1_test.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"net/http"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/stretchr/testify/assert"
@@ -107,36 +106,6 @@ func TestIMDSv1_LookupFailureDeniesAndCaches(t *testing.T) {
 	assert.Equal(t, 1, res.calls, "a failing lookup must not be retried per request")
 }
 
-// An expired entry is re-resolved, so flipping HttpTokens takes effect without a
-// vpcd restart.
-func TestIMDSv1_CacheExpires(t *testing.T) {
-	c := newV1AllowCache()
-	now := time.Unix(1_700_000_000, 0).UTC()
-
-	c.put("i-1", true, now)
-	allowed, ok := c.get("i-1", now.Add(v1AllowTTL-time.Second))
-	require.True(t, ok)
-	assert.True(t, allowed)
-
-	_, ok = c.get("i-1", now.Add(v1AllowTTL+time.Second))
-	assert.False(t, ok, "entry must expire after the TTL")
-}
-
-// sweep drops expired entries so the map cannot grow without bound across
-// instance churn.
-func TestIMDSv1_CacheSweep(t *testing.T) {
-	c := newV1AllowCache()
-	now := time.Unix(1_700_000_000, 0).UTC()
-
-	c.put("i-old", true, now)
-	c.put("i-new", true, now.Add(v1AllowTTL))
-	c.sweep(now.Add(v1AllowTTL + time.Second))
-
-	_, ok := c.get("i-old", now.Add(v1AllowTTL+time.Second))
-	assert.False(t, ok)
-	assert.Len(t, c.entries, 1, "only the live entry survives the sweep")
-}
-
 // An ENI with no attached instance denies without attempting a lookup.
 func TestIMDSv1_UnattachedENIDenies(t *testing.T) {
 	res := v1Resolver(ec2.HttpTokensStateOptional)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -150,10 +150,12 @@ func withTapConn(h http.Handler, eni *eniFacts, setTTL connTTLSetter) http.Handl
 // clock. The per-tap responder is left nil — the HTTP handler is exercised directly.
 func newTestService(res eniResolver, fIAM profileLookup, assumer stsAssumer) (*IMDSServiceImpl, time.Time) {
 	now := time.Unix(1_700_000_000, 0).UTC()
+	v1Allow := newTTLCache[bool](v1AllowTTL)
+	v1Allow.now = func() time.Time { return now }
 	return &IMDSServiceImpl{
 		resolver:       res,
 		tokens:         newTokenStore(),
-		v1Allow:        newV1AllowCache(),
+		v1Allow:        v1Allow,
 		creds:          newCredCache(assumer),
 		iam:            fIAM,
 		roleMiss:       newRoleMissLogger(func() time.Time { return now }),

spinifex/domains/ec2/guestmetadata/nats_clients.go

@@ -151,3 +151,14 @@ func (c *ttlCache[V]) put(key string, v V) {
 	defer c.mu.Unlock()
 	c.m[key] = ttlCacheEntry[V]{val: v, expiresAt: c.now().Add(c.ttl)}
 }
+
+// sweep evicts every entry that has expired as of now.
+func (c *ttlCache[V]) sweep(now time.Time) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for k, e := range c.m {
+		if now.After(e.expiresAt) {
+			delete(c.m, k)
+		}
+	}
+}

spinifex/domains/ec2/guestmetadata/nats_clients_test.go

@@ -197,3 +197,20 @@ func TestTTLCache_ExpiresEntries(t *testing.T) {
 	_, ok = c.get("k")
 	assert.False(t, ok)
 }
+
+// sweep drops expired entries so the map cannot grow without bound across
+// instance churn.
+func TestTTLCache_SweepDropsExpired(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	c := newTTLCache[bool](time.Minute)
+	c.now = func() time.Time { return now }
+
+	c.put("old", true)
+	now = now.Add(time.Minute)
+	c.put("new", true)
+	c.sweep(now.Add(time.Second))
+
+	assert.Len(t, c.m, 1, "only the live entry survives the sweep")
+	_, ok := c.m["new"]
+	assert.True(t, ok)
+}

spinifex/domains/ec2/guestmetadata/service.go

@@ -65,9 +65,11 @@ type eniResolver interface {
 // responder per local primary-ENI tap, each serving the shared mux with the
 // tap's ENI identity.
 type IMDSServiceImpl struct {
-	resolver       eniResolver
-	tokens         *tokenStore
-	v1Allow        *v1AllowCache
+	resolver eniResolver
+	tokens   *tokenStore
+	// v1Allow memoises HttpTokens per instance: the untokened path needs no
+	// credentials, so resolving it per request would allow unbounded fan-outs.
+	v1Allow        *ttlCache[bool]
 	creds          *credCache
 	iam            profileLookup
 	roleMiss       *roleMissLogger
@@ -169,7 +171,7 @@ func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer
 			lookup:   &localInstanceLookup{local: localState, records: records},
 		},
 		tokens:         newTokenStore(),
-		v1Allow:        newV1AllowCache(),
+		v1Allow:        newTTLCache[bool](v1AllowTTL),
 		creds:          newCredCache(sts),
 		iam:            iamSvc,
 		roleMiss:       newRoleMissLogger(time.Now),

spinifex/domains/ec2/igw/service_impl.go

@@ -219,12 +219,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
 		return nil, err
 	}
 
-	igwIDs := make(map[string]bool)
-	for _, id := range input.InternetGatewayIds {
-		if id != nil {
-			igwIDs[*id] = true
-		}
-	}
+	igwIDs := awsidentifiers.Set(input.InternetGatewayIds)
 
 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeIGWValidFilters)
 	if err != nil {

spinifex/domains/ec2/igw/service_impl_test.go

@@ -174,14 +174,14 @@ func TestDeleteInternetGateway_WhileAttached(t *testing.T) {
 	_, err = svc.DeleteInternetGateway(context.Background(), &ec2.DeleteInternetGatewayInput{
 		InternetGatewayId: aws.String(igwID),
 	}, testAccountID)
-	requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The internetGateway '"+igwID+"' has dependencies and cannot be deleted.")
+	testutil.RequireAWSError(t, err, awserrors.ErrorDependencyViolation, "The internetGateway '"+igwID+"' has dependencies and cannot be deleted.")
 
 	// Attaching again names the gateway and the network it is on
 	_, err = svc.AttachInternetGateway(context.Background(), &ec2.AttachInternetGatewayInput{
 		InternetGatewayId: aws.String(igwID),
 		VpcId:             aws.String("vpc-test123"),
 	}, testAccountID)
-	requireAWSError(t, err, awserrors.ErrorResourceAlreadyAssociated, "resource "+igwID+" is already attached to network vpc-test123")
+	testutil.RequireAWSError(t, err, awserrors.ErrorResourceAlreadyAssociated, "resource "+igwID+" is already attached to network vpc-test123")
 }
 
 func TestInternetGateway_NotFoundNamesTheID(t *testing.T) {
@@ -191,20 +191,12 @@ func TestInternetGateway_NotFoundNamesTheID(t *testing.T) {
 	_, err := svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
 		InternetGatewayIds: []*string{aws.String("igw-0000000000000dead")},
 	}, testAccountID)
-	requireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
+	testutil.RequireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
 
 	_, err = svc.DeleteInternetGateway(context.Background(), &ec2.DeleteInternetGatewayInput{
 		InternetGatewayId: aws.String("igw-0000000000000dead"),
 	}, testAccountID)
-	requireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
-}
-
-func requireAWSError(t *testing.T, err error, code, message string) {
-	t.Helper()
-	got, msg, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok, "error %v carries no AWS code", err)
-	assert.Equal(t, code, got)
-	assert.Equal(t, message, msg)
+	testutil.RequireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
 }
 
 func TestDescribeInternetGateways_All(t *testing.T) {

spinifex/domains/ec2/instance/describe_account_records_test.go

@@ -0,0 +1,89 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func recordsService(store *vmmock.StateStore) *InstanceServiceImpl {
+	return &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+}
+
+func recordIDs(out *ec2.DescribeInstancesOutput) map[string]string {
+	ids := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			ids[*inst.InstanceId] = *inst.State.Name
+		}
+	}
+	return ids
+}
+
+// The record read answers for instances the stopped listing leaves out: a
+// running or drain-stopped record that no node need be holding.
+func TestDescribeAccountInstanceRecords_ReturnsEveryNonTerminatedRecordOfTheAccount(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{
+		Stopped: map[string]*vm.VM{"i-stopped": nilRecordVM("i-stopped", owner, true, true, vm.StateStopped)},
+		Saved: map[string]map[string]*vm.VM{"node-gone": {
+			"i-running": nilRecordVM("i-running", owner, true, true, vm.StateRunning),
+			"i-other":   nilRecordVM("i-other", "999999999999", true, true, vm.StateRunning),
+		}},
+	}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"i-stopped": "stopped", "i-running": "running"}, recordIDs(out),
+		"the other account's record is not returned")
+}
+
+// A record that cannot be projected fails the read; it is never skipped, since
+// the caller takes existence from this answer.
+func TestDescribeAccountInstanceRecords_NilReservationOrInstanceFails(t *testing.T) {
+	owner := "111122223333"
+	for name, bad := range map[string]*vm.VM{
+		"nil reservation": nilRecordVM("i-bad", owner, false, true, vm.StateRunning),
+		"nil instance":    nilRecordVM("i-bad", owner, true, false, vm.StateRunning),
+	} {
+		t.Run(name, func(t *testing.T) {
+			store := &vmmock.StateStore{
+				Stopped: map[string]*vm.VM{"i-good": nilRecordVM("i-good", owner, true, true, vm.StateStopped)},
+				Saved:   map[string]map[string]*vm.VM{"node-1": {"i-bad": bad}},
+			}
+
+			out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+			require.Error(t, err)
+			assert.Nil(t, out, "no shorter list is returned")
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+		})
+	}
+}
+
+func TestDescribeAccountInstanceRecords_StoreErrorFails(t *testing.T) {
+	store := &vmmock.StateStore{ListLiveErr: errors.New("kv down")}
+
+	out, err := recordsService(store).DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}
+
+func TestDescribeAccountInstanceRecords_NoStoreFails(t *testing.T) {
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), config: &config.Config{}}
+
+	out, err := svc.DescribeAccountInstanceRecords(context.Background(), &ec2.DescribeInstancesInput{}, "111122223333")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+}

spinifex/domains/ec2/instance/describe_nil_records_current_behaviour_test.go

@@ -0,0 +1,64 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func nilRecordVM(id, owner string, withReservation, withInstance bool, status vm.InstanceState) *vm.VM {
+	v := &vm.VM{ID: id, InstanceType: "t3.micro", Status: status, AccountID: owner}
+	if withReservation {
+		v.Reservation = &ec2.Reservation{ReservationId: aws.String("r-" + id), OwnerId: aws.String(owner)}
+	}
+	if withInstance {
+		v.Instance = &ec2.Instance{InstanceId: aws.String(id), InstanceType: aws.String("t3.micro")}
+	}
+	return v
+}
+
+// Pins today: the node path drops an instance whose Reservation or Instance is
+// nil without a log line or an error, so it reads as absent from a listing the
+// caller may treat as complete.
+func TestCurrentBehaviour_DescribeInstances_NodePathSkipsNilReservationAndInstanceSilently(t *testing.T) {
+	owner := "111122223333"
+	vms := map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateRunning),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateRunning),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateRunning),
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(vms), config: &config.Config{}}
+
+	out, err := svc.DescribeInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}
+
+// Pins today: the KV path skips the same records, with a warning, and still
+// answers success.
+func TestCurrentBehaviour_DescribeStoppedInstances_KVPathSkipsNilReservationAndInstance(t *testing.T) {
+	owner := "111122223333"
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		"i-good":    nilRecordVM("i-good", owner, true, true, vm.StateStopped),
+		"i-no-res":  nilRecordVM("i-no-res", owner, false, true, vm.StateStopped),
+		"i-no-inst": nilRecordVM("i-no-inst", owner, true, false, vm.StateStopped),
+	}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store, config: &config.Config{}}
+
+	out, err := svc.DescribeStoppedInstances(context.Background(), &ec2.DescribeInstancesInput{}, owner)
+
+	require.NoError(t, err, "CURRENT: no error for the two skipped instances")
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-good", *out.Reservations[0].Instances[0].InstanceId, "CURRENT: only the complete record is returned")
+}

spinifex/domains/ec2/instance/instance_filter_test.go

@@ -164,6 +164,20 @@ func TestInstanceMatchesFilters_TagKey(t *testing.T) {
 	assert.False(t, instanceMatchesFilters(inst, ic, filters))
 }
 
+// A tag launched without a value carries an empty value in AWS, so tag-key
+// and an empty tag:Key filter still find it.
+func TestInstanceMatchesFilters_NilValueTag(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+		Tags:       []*ec2.Tag{{Key: aws.String("Name")}},
+	}
+	assert.True(t, instanceMatchesFilters(inst, ic, map[string][]string{"tag-key": {"Name"}}))
+	assert.True(t, instanceMatchesFilters(inst, ic, map[string][]string{"tag:Name": {""}}))
+	assert.False(t, instanceMatchesFilters(inst, ic, map[string][]string{"tag-key": {"Other"}}))
+}
+
 func TestInstanceMatchesFilters_TagValue(t *testing.T) {
 	inst := &vm.VM{ID: "i-abc"}
 	ic := &ec2.Instance{

spinifex/domains/ec2/instance/service.go

@@ -106,6 +106,8 @@ type GPUClaimer interface {
 type StoppedInstanceStore interface {
 	LoadStoppedInstance(instanceID string) (*vm.VM, error)
 	ListStoppedInstances() ([]*vm.VM, error)
+	ListNonTerminatedInstances() ([]*vm.VM, error)
+	ListTerminatedInPrimary() ([]*vm.VM, error)
 	ListTerminatedInstances() ([]*vm.VM, error)
 	WriteStoppedInstance(instanceID string, instance *vm.VM) error
 	DeleteStoppedInstance(instanceID string) error
@@ -139,6 +141,17 @@ type InstanceTagWriter interface {
 	DeleteAllTags(ctx context.Context, accountID, resourceID string) error
 }
 
+// OwedUnpublish names the publish a terminate leaves in place when its unmount
+// did not complete: the node still exporting the volume and the instance it
+// served. The zero value means the unmount completed and nothing is owed.
+type OwedUnpublish struct {
+	NodeID     string
+	InstanceID string
+}
+
+// Owed reports whether a publish is left in place.
+func (o OwedUnpublish) Owed() bool { return o.NodeID != "" }
+
 // VolumeDeleter deletes EBS volumes. Implemented by domains/ec2/volume's
 // VolumeServiceImpl (used by the daemon).
 type VolumeDeleter interface {
@@ -147,14 +160,16 @@ type VolumeDeleter interface {
 	// DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of
 	// an instance terminate: it clears any stale attachment (terminate
 	// implies detach) before deleting, so a still-attached boot volume left
-	// over from Stop is not rejected by DeleteVolume's in-use guard.
-	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// over from Stop is not rejected by DeleteVolume's in-use guard. When owed
+	// names a publish, an unpublish obligation is recorded before the clear.
+	DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
 
 	// DetachVolumeOnTerminate clears a still-attached volume's attachment on
 	// terminate without deleting it, matching AWS semantics for a
 	// DeleteOnTermination=false volume: terminate still implies detach, it
 	// just leaves the volume behind as available rather than deleting it.
-	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
+	// When owed names a publish, an unpublish obligation is recorded first.
+	DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string, owed OwedUnpublish) error
 }
 
 // VolumeCreator creates the volumes a launch names beyond the root device, and

spinifex/domains/ec2/instance/service_impl.go

@@ -1,6 +1,7 @@
 package instance
 
 import (
+	"cmp"
 	"context"
 	"encoding/base64"
 	"encoding/json"
@@ -1507,7 +1508,7 @@ func (s *InstanceServiceImpl) ReplaceIamProfileAssociation(ctx context.Context,
 // the caller. Supports "instance-id" and "state" filters; unknown filters return
 // InvalidParameterValue.
 func (s *InstanceServiceImpl) DescribeIamProfileAssociations(ctx context.Context, input *ec2.DescribeIamInstanceProfileAssociationsInput, accountID string) (*ec2.DescribeIamInstanceProfileAssociationsOutput, error) {
-	assocFilter := stringPtrSliceToSet(input.AssociationIds)
+	assocFilter := awsidentifiers.Set(input.AssociationIds)
 	instFilter := make(map[string]bool)
 	stateFilter := make(map[string]bool)
 	for _, f := range input.Filters {
@@ -1580,19 +1581,6 @@ func (s *InstanceServiceImpl) findInstanceByAssociationID(associationID, account
 	return owner, true
 }
 
-func stringPtrSliceToSet(in []*string) map[string]bool {
-	if len(in) == 0 {
-		return nil
-	}
-	out := make(map[string]bool, len(in))
-	for _, s := range in {
-		if s != nil && *s != "" {
-			out[*s] = true
-		}
-	}
-	return out
-}
-
 func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.RunInstancesInput, instance *vm.VM) ([]VolumeInfo, error) {
 	p := parseVolumeParams(input)
 	if input.ImageId != nil {
@@ -2035,8 +2023,7 @@ func IsInstanceVisibleToCaller(accountID string, v *vm.VM) bool {
 // instanceMatchesFilters checks whether a VM + its built ec2.Instance copy satisfy all parsed awsfilters.
 func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]string) bool {
 	for name, values := range filters {
-		if strings.HasPrefix(name, "tag:") {
-			// tag:Key filters are handled after all field awsfilters.
+		if awsfilters.IsTagFilter(name) {
 			continue
 		}
 
@@ -2075,16 +2062,6 @@ func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]
 				return false
 			}
 			continue
-		case "tag-key":
-			if !matchTagKey(ic.Tags, values) {
-				return false
-			}
-			continue
-		case "tag-value":
-			if !matchTagValue(ic.Tags, values) {
-				return false
-			}
-			continue
 		default:
 			// Filter name passed ParseFilters but has no case — treat as non-match
 			// to avoid silently ignoring it.
@@ -2096,7 +2073,6 @@ func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]
 		}
 	}
 
-	// Check tag:Key filters via the instance's Tag slice.
 	tags := awsfilters.EC2TagsToMap(ic.Tags)
 	return awsfilters.MatchesTags(filters, tags)
 }
@@ -2126,20 +2102,6 @@ func matchNICAnyPrivateIP(nics []*ec2.InstanceNetworkInterface, values []string)
 	})
 }
 
-// matchTagKey returns true if any tag key on the resource matches any of the filter values.
-func matchTagKey(tags []*ec2.Tag, values []string) bool {
-	return slices.ContainsFunc(tags, func(t *ec2.Tag) bool {
-		return t.Key != nil && awsfilters.MatchesAny(values, *t.Key)
-	})
-}
-
-// matchTagValue returns true if any tag value on the resource matches any of the filter values.
-func matchTagValue(tags []*ec2.Tag, values []string) bool {
-	return slices.ContainsFunc(tags, func(t *ec2.Tag) bool {
-		return t.Value != nil && awsfilters.MatchesAny(values, *t.Value)
-	})
-}
-
 // DescribeInstances returns instances on this node visible to the caller's account.
 func (s *InstanceServiceImpl) DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
 	slog.InfoContext(ctx, "Processing DescribeInstances request from this node", "accountID", accountID)
@@ -2273,7 +2235,18 @@ func (s *InstanceServiceImpl) DescribeStoppedInstances(ctx context.Context, inpu
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListStoppedInstances, 80, "stopped", "DescribeStoppedInstances", false)
+}
+
+// DescribeAccountInstanceRecords returns every non-terminated instance record
+// the account owns, whichever node holds it. It is the existence source for the
+// strict account inventory, so a record it cannot project fails the call rather
+// than being skipped.
+func (s *InstanceServiceImpl) DescribeAccountInstanceRecords(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListNonTerminatedInstances, 0, "pending", "DescribeAccountInstanceRecords", true)
 }
 
 // DescribeTerminatedInstances returns terminated instances from the terminated KV bucket.
@@ -2281,10 +2254,10 @@ func (s *InstanceServiceImpl) DescribeTerminatedInstances(ctx context.Context, i
 	if s.stoppedStore == nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances")
+	return s.describeInstancesFromKV(ctx, input, accountID, s.stoppedStore.ListTerminatedInstances, 48, "terminated", "DescribeTerminatedInstances", false)
 }
 
-func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string) (*ec2.DescribeInstancesOutput, error) {
+func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string, listFn func() ([]*vm.VM, error), fallbackCode int64, fallbackName, opName string, strict bool) (*ec2.DescribeInstancesOutput, error) {
 	instanceIDFilter, err := ParseInstanceIDFilter(input.InstanceIds)
 	if err != nil {
 		return nil, err
@@ -2312,6 +2285,11 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input
 			continue
 		}
 		if instance.Reservation == nil || instance.Instance == nil {
+			if strict {
+				slog.ErrorContext(ctx, opName+": record has nil Reservation/Instance, refusing a listing that would omit it",
+					"instanceId", instance.ID)
+				return nil, errors.New(awserrors.ErrorServerInternal)
+			}
 			slog.WarnContext(ctx, opName+": skipping instance with nil Reservation/Instance (data integrity issue)",
 				"instanceId", instance.ID)
 			continue
@@ -3070,6 +3048,15 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	// stopped-path delete failure.
 	var lastErr error
 
+	// A volume whose seal failed at stop is still exported on the node that
+	// stopped it, so clearing its attachment owes that node an unpublish.
+	owedFor := func(volumeID string) OwedUnpublish {
+		if instance.LastNode == "" || !slices.Contains(instance.UnpublishOwed, volumeID) {
+			return OwedUnpublish{}
+		}
+		return OwedUnpublish{NodeID: instance.LastNode, InstanceID: instance.ID}
+	}
+
 	for _, ebsRequest := range instance.EBSRequests.Requests {
 		// Internal volumes (EFI) are always cleaned up via ebs.delete.
 		if ebsRequest.EFI {
@@ -3100,7 +3087,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 				continue
 			}
 			// A volume already gone is the detach's own goal, not a failure.
-			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
+			if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil &&
 				!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
 				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
 				lastErr = err
@@ -3119,7 +3106,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 		// clears a Boot volume, runtime/daemon/vm_adapters.go) before deleting, so
 		// DeleteVolume's in-use guard does not reject an otherwise-legitimate
 		// delete. The resulting error is surfaced into lastErr, not swallowed.
-		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil {
+		if err := s.volumeDeleter.DeleteVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID, owedFor(ebsRequest.Name)); err != nil {
 			slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err)
 			lastErr = err
 		}
@@ -3928,3 +3915,57 @@ func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *
 		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
 	}, nil
 }
+
+// AccountTeardownOutstanding reports instances not terminated with every dependent done.
+// Terminated records expire after 1h, so it is never sole evidence an account is clear.
+// An unreadable record or one with no instance ID fails the read.
+func (s *InstanceServiceImpl) AccountTeardownOutstanding(accountID string) ([]OutstandingInstance, error) {
+	if s.stoppedStore == nil {
+		return nil, errors.New("instance record store not configured")
+	}
+	live, err := s.stoppedStore.ListNonTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list non-terminated instances: %w", err)
+	}
+	primaryTerminated, err := s.stoppedStore.ListTerminatedInPrimary()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances in primary store: %w", err)
+	}
+	terminated, err := s.stoppedStore.ListTerminatedInstances()
+	if err != nil {
+		return nil, fmt.Errorf("list terminated instances: %w", err)
+	}
+
+	// An instance can sit in both stores while a terminate is mid-flight. The
+	// non-terminated record is listed first and wins, so it is never reported
+	// as torn down on the strength of the terminated copy.
+	byID := map[string]OutstandingInstance{}
+	for _, store := range []struct {
+		name    string
+		records []*vm.VM
+	}{{"non-terminated", live}, {"primary terminated", primaryTerminated}, {"terminated", terminated}} {
+		for _, v := range store.records {
+			if v == nil || v.ID == "" {
+				return nil, fmt.Errorf("%s instance store holds a record with no instance ID", store.name)
+			}
+			if !IsInstanceVisible(accountID, v.AccountID) {
+				continue
+			}
+			if v.Status == vm.StateTerminated && v.TeardownComplete() {
+				continue
+			}
+			if _, seen := byID[v.ID]; seen {
+				continue
+			}
+			byID[v.ID] = OutstandingInstance{
+				InstanceID: v.ID,
+				State:      string(v.Status),
+				Teardown:   maps.Clone(v.Teardown),
+			}
+		}
+	}
+
+	return slices.SortedFunc(maps.Values(byID), func(a, b OutstandingInstance) int {
+		return cmp.Compare(a.InstanceID, b.InstanceID)
+	}), nil
+}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -1038,7 +1038,7 @@ func TestDescribeInstancesFromKV_FilterExcludesOnlyInstance_ReturnsZeroReservati
 	input := &ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{{Name: aws.String("instance-type"), Values: []*string{aws.String("t3.nano")}}},
 	}
-	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances")
+	out, err := svc.describeInstancesFromKV(context.Background(), input, awsidentifiers.GlobalAccountID, listFn, 80, "stopped", "DescribeStoppedInstances", false)
 	require.NoError(t, err)
 	assert.Empty(t, out.Reservations, "a reservation with every instance filtered out must not appear in the response")
 }
@@ -1734,6 +1734,7 @@ type fakeVolumeDeleter struct {
 	calls    []string
 	deleted  []string
 	detached []string
+	owed     []OwedUnpublish
 	err      error
 }
 
@@ -1749,7 +1750,8 @@ func (f *fakeVolumeDeleter) DeleteVolume(_ context.Context, input *ec2.DeleteVol
 
 // DeleteVolumeOnTerminate mirrors DeleteVolume's call/error bookkeeping; the
 // stopped-instance terminate path calls this instead of DeleteVolume.
-func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -1761,7 +1763,8 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
 // DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
 // bookkeeping but records into detached instead of deleted; the non-DoT
 // Boot-volume branch calls this instead of deleting.
-func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
+func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string, owed OwedUnpublish) error {
+	f.owed = append(f.owed, owed)
 	f.calls = append(f.calls, volumeID)
 	if f.err != nil {
 		return f.err
@@ -2009,6 +2012,45 @@ func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []string{"vol-user-001"}, vd.deleted, "only DeleteOnTermination=true volumes deleted")
 	assert.Equal(t, []string{"vol-keep-001"}, vd.detached, "terminate implies detach even for a volume that survives it")
+	assert.Equal(t, []OwedUnpublish{{}, {}}, vd.owed, "a stop whose unmount completed owes no unpublish")
+}
+
+// A stop whose seal failed left those volumes exported on the node that
+// stopped the instance, so terminate names that node for each of them alone.
+func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *testing.T) {
+	id := "i-seal-failed"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
+		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
+		{Name: "vol-keep-001", DeleteOnTermination: false},
+		{Name: "vol-sealed-001", DeleteOnTermination: false},
+	}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+
+	owed := OwedUnpublish{NodeID: "node-a", InstanceID: id}
+	assert.Equal(t, []string{"vol-root-001", "vol-keep-001", "vol-sealed-001"}, vd.calls)
+	assert.Equal(t, []OwedUnpublish{owed, owed, {}}, vd.owed)
+}
+
+// With no node recorded there is no node to name; the terminate owes nothing
+// rather than naming an empty node.
+func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
+	id := "i-seal-failed-nonode"
+	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+	vd := &fakeVolumeDeleter{}
+	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
+
+	_, err := svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+	require.NoError(t, err)
+	assert.Equal(t, []OwedUnpublish{{}}, vd.owed)
 }
 
 // TestTerminateStoppedInstance_StampsTeardownVolumesDone locks the

spinifex/domains/ec2/instance/teardown_outstanding.go

@@ -0,0 +1,12 @@
+package instance
+
+// OutstandingInstance is an instance of an account that is not yet fully torn
+// down, with the state its record carries.
+type OutstandingInstance struct {
+	InstanceID string
+	// State is the record's lifecycle state, such as running or terminated.
+	State string
+	// Teardown is the per-dependent progress (done, pending or failed) the
+	// record carries. It is empty when no dependent has been recorded.
+	Teardown map[string]string
+}

spinifex/domains/ec2/launchtemplate/service_impl.go

@@ -516,7 +516,7 @@ func (s *LaunchTemplateServiceImpl) commitHeader(ctx context.Context, op, accoun
 		if err == nil {
 			return nil
 		}
-		if !isRevisionConflict(err) {
+		if !kvutil.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, op+": header commit failed", "launchTemplateId", ltID, "err", err)
 			return err
 		}
@@ -541,11 +541,6 @@ func (s *LaunchTemplateServiceImpl) commitHeader(ctx context.Context, op, accoun
 	return errors.New("header commit retries exhausted")
 }
 
-// isRevisionConflict reports a CAS write that lost to a concurrent writer.
-func isRevisionConflict(err error) bool {
-	return errors.Is(err, jetstream.ErrKeyRevisionMismatch) || errors.Is(err, jetstream.ErrKeyExists)
-}
-
 // abandonVersion removes a body whose header commit failed. A failure is
 // logged and leaves a headerless body, which no ID, name or launch resolves.
 func (s *LaunchTemplateServiceImpl) abandonVersion(ctx context.Context, accountID, ltID string, n int64) {
@@ -616,7 +611,7 @@ func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accou
 		if err == nil {
 			return header, nil
 		}
-		if !isRevisionConflict(err) {
+		if !kvutil.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, "ModifyLaunchTemplate: header update failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -882,8 +877,8 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplates(ctx context.Context,
 		}
 	}
 
-	nameSet := stringSet(input.LaunchTemplateNames)
-	idSet := stringSet(input.LaunchTemplateIds)
+	nameSet := awsidentifiers.Set(input.LaunchTemplateNames)
+	idSet := awsidentifiers.Set(input.LaunchTemplateIds)
 
 	prefix := accountID + "."
 	keys, err := s.kv.Keys(ctx)
@@ -944,7 +939,7 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplates(ctx context.Context,
 
 func templateMatchesFilters(h *LaunchTemplateHeader, filters map[string][]string) bool {
 	for name, values := range filters {
-		if strings.HasPrefix(name, "tag:") {
+		if awsfilters.IsTagFilter(name) {
 			continue
 		}
 		switch name {
@@ -960,17 +955,6 @@ func templateMatchesFilters(h *LaunchTemplateHeader, filters map[string][]string
 			if !awsfilters.MatchesAny(values, h.CreateTime.Format(time.RFC3339)) {
 				return false
 			}
-		case "tag-key":
-			matched := false
-			for k := range h.Tags {
-				if awsfilters.MatchesAny(values, k) {
-					matched = true
-					break
-				}
-			}
-			if !matched {
-				return false
-			}
 		default:
 			return false
 		}
@@ -1177,19 +1161,6 @@ func isHeaderKey(k, prefix string) bool {
 	return strings.HasPrefix(rest, "lt-") && !strings.Contains(rest, ".")
 }
 
-func stringSet(in []*string) map[string]bool {
-	if len(in) == 0 {
-		return nil
-	}
-	set := make(map[string]bool, len(in))
-	for _, s := range in {
-		if s != nil {
-			set[*s] = true
-		}
-	}
-	return set
-}
-
 // validateTemplateID enforces the lt- id prefix so a syntactically invalid id
 // returns Malformed rather than NotFound, matching image/volume id handling.
 func validateTemplateID(id string) error {

spinifex/domains/ec2/launchtemplate/service_impl_test.go

@@ -178,6 +178,12 @@ func TestDescribeLaunchTemplates_TagFilter(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, out.LaunchTemplates)
+
+	out, err = svc.DescribeLaunchTemplates(context.Background(), &ec2.DescribeLaunchTemplatesInput{
+		Filters: []*ec2.Filter{{Name: aws.String("tag-key"), Values: []*string{aws.String("env")}}},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, out.LaunchTemplates, 1)
 }
 
 func TestDescribeLaunchTemplates_AccountIsolation(t *testing.T) {

spinifex/domains/ec2/natgw/lifecycle_invariants_test.go

@@ -3,6 +3,7 @@ package natgw
 import (
 	"encoding/json"
 	"fmt"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -40,7 +41,7 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) {
 		svc, js := setupTestServiceJS(t)
 		natgwID := createTestNatGateway(t, svc)
 
-		seedKV(t, js, kvBucketRouteTables, map[string][]byte{
+		testutil.SeedKV(t, js, kvBucketRouteTables, map[string][]byte{
 			kvutil.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
 				`{"route_table_id":"rtb-routed","vpc_id":"vpc-test1","routes":[{"destination_cidr_block":"0.0.0.0/0","nat_gateway_id":%q}]}`, natgwID),
 		})

spinifex/domains/ec2/natgw/service_impl.go

@@ -238,7 +238,7 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2
 
 // kvBucketRouteTables is the route-table KV bucket. Duplicated as a literal
 // (not imported from the routetable package) because routetable imports this
-// package for PublishAddEvent — importing it back would be a cycle.
+// package for KVBucketNatGateways — importing it back would be a cycle.
 const kvBucketRouteTables = "spinifex-vpc-route-tables"
 
 // publishDeleteEventsForNatGateway tears down the SNAT for every private subnet
@@ -368,12 +368,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 		return nil, err
 	}
 
-	natgwIDs := make(map[string]bool)
-	for _, id := range input.NatGatewayIds {
-		if id != nil {
-			natgwIDs[*id] = true
-		}
-	}
+	natgwIDs := awsidentifiers.Set(input.NatGatewayIds)
 
 	prefix := accountID + "."
 	keys, err := s.natgwKV.Keys(ctx)
@@ -470,17 +465,6 @@ func natgwMatchesFilters(record *NatGatewayRecord, filters map[string][]string)
 	return awsfilters.MatchesTags(filters, record.Tags)
 }
 
-// PublishAddEvent publishes a vpc.add-nat-gateway event for vpcd to create the SNAT rule.
-// Called by the route table service when CreateRoute targets a NAT GW.
-func (s *NatGatewayServiceImpl) PublishAddEvent(vpcId, natGatewayId, publicIp, subnetCidr string) {
-	projection.New(s.natsConn).AddNATGateway(networkv1.NATGatewayEvent{
-		VpcId:        vpcId,
-		NatGatewayId: natGatewayId,
-		PublicIp:     publicIp,
-		SubnetCidr:   subnetCidr,
-	})
-}
-
 // PublishDeleteEvent publishes a vpc.delete-nat-gateway event for vpcd to remove the SNAT rule.
 func (s *NatGatewayServiceImpl) PublishDeleteEvent(vpcId, natGatewayId, publicIp, subnetCidr string) {
 	projection.New(s.natsConn).RemoveNATGateway(networkv1.NATGatewayEvent{
@@ -491,23 +475,6 @@ func (s *NatGatewayServiceImpl) PublishDeleteEvent(vpcId, natGatewayId, publicIp
 	})
 }
 
-// GetNatGateway retrieves a NAT Gateway record by ID.
-func (s *NatGatewayServiceImpl) GetNatGateway(accountID, natgwID string) (*NatGatewayRecord, error) {
-	ctx := context.Background()
-	entry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
-	if err != nil {
-		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
-		}
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	var record NatGatewayRecord
-	if err := json.Unmarshal(entry.Value(), &record); err != nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	return &record, nil
-}
-
 func recordToEC2(record *NatGatewayRecord) *ec2.NatGateway {
 	ngw := &ec2.NatGateway{
 		NatGatewayId:     aws.String(record.NatGatewayId),

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -30,17 +30,17 @@ func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStre
 	js := testutil.NewJetStream(t, nc)
 
 	// Seed VPC KV
-	seedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
+	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
 		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed subnet KV (public subnet)
-	seedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
+	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
 		kvutil.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
 	})
 
 	// Seed EIP KV
-	seedKV(t, js, ec2eip.KVBucketEIPs, map[string][]byte{
+	testutil.SeedKV(t, js, ec2eip.KVBucketEIPs, map[string][]byte{
 		kvutil.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
 		kvutil.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
 	})
@@ -50,17 +50,6 @@ func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStre
 	return svc, js
 }
 
-func seedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue {
-	t.Helper()
-	kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
-	require.NoError(t, err)
-	for key, value := range entries {
-		_, err := kv.Put(t.Context(), key, value)
-		require.NoError(t, err)
-	}
-	return kv
-}
-
 func TestDeletedBucketTTL(t *testing.T) {
 	_, js := setupTestServiceJS(t)
 	kv, err := js.KeyValue(t.Context(), KVBucketDeletedNatGateways)
@@ -181,8 +170,8 @@ func TestDeleteNatGateway(t *testing.T) {
 	assert.Equal(t, natgwID, *deleteOut.NatGatewayId)
 
 	// Should be gone
-	_, err = svc.GetNatGateway(testAccountID, natgwID)
-	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
+	_, err = svc.natgwKV.Get(t.Context(), kvutil.AccountKey(testAccountID, natgwID))
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
 }
 
 func TestDescribeNatGateways(t *testing.T) {
@@ -245,28 +234,6 @@ func TestDescribeNatGateways_FilterByState(t *testing.T) {
 	assert.Empty(t, out.NatGateways)
 }
 
-func TestGetNatGateway(t *testing.T) {
-	svc := setupTestService(t)
-	createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{
-		SubnetId:     aws.String("subnet-pub1"),
-		AllocationId: aws.String("eipalloc-test1"),
-	}, testAccountID)
-	require.NoError(t, err)
-	natgwID := *createOut.NatGateway.NatGatewayId
-
-	record, err := svc.GetNatGateway(testAccountID, natgwID)
-	require.NoError(t, err)
-	assert.Equal(t, natgwID, record.NatGatewayId)
-	assert.Equal(t, "203.0.113.50", record.PublicIp)
-	assert.Equal(t, "eipalloc-test1", record.AllocationId)
-}
-
-func TestGetNatGateway_NotFound(t *testing.T) {
-	svc := setupTestService(t)
-	_, err := svc.GetNatGateway(testAccountID, "nat-nope")
-	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
-}
-
 func TestCreateNatGateway_MissingParams(t *testing.T) {
 	svc := setupTestService(t)
 

spinifex/domains/ec2/placementgroup/service_impl.go

@@ -227,18 +227,8 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
 	}
 
 	// Build filter maps for GroupNames/GroupIds parameters
-	nameSet := make(map[string]bool)
-	for _, name := range input.GroupNames {
-		if name != nil {
-			nameSet[*name] = true
-		}
-	}
-	idSet := make(map[string]bool)
-	for _, id := range input.GroupIds {
-		if id != nil {
-			idSet[*id] = true
-		}
-	}
+	nameSet := awsidentifiers.Set(input.GroupNames)
+	idSet := awsidentifiers.Set(input.GroupIds)
 
 	prefix := accountID + "."
 	keys, err := s.kv.Keys(ctx)

spinifex/domains/ec2/routetable/service_impl.go

@@ -615,12 +615,7 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
 		return nil, err
 	}
 
-	rtbIDs := make(map[string]bool)
-	for _, id := range input.RouteTableIds {
-		if id != nil {
-			rtbIDs[*id] = true
-		}
-	}
+	rtbIDs := awsidentifiers.Set(input.RouteTableIds)
 
 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeRouteTablesValidFilters)
 	if err != nil {