chore: suppress preflight build script output

test: stub the /dev/kvm probe so QEMU argv tests run on hosts without KVM
feat(ec2): complete-or-error account inventories for instances, spot requests and volumes (#1544 )
2026-11-04 02:03:39 +00:00 · 2026-11-04 11:41:09 +10:00 · 2026-11-04 11:37:01 +10:00 · 2026-11-04 11:19:28 +10:00 · 2026-11-04 11:54:35 +11:00 · 2026-11-04 11:50:49 +11:00
384 changed files with 7783 additions and 4156 deletions
@@ -42,11 +42,15 @@ ifdef QUIET
  _COVQ  = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
  _RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
  _SECQ  = >
+  _HDR   = :
+  _RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
 else
  _Q     =
  _COVQ  =
  _RACEQ =
  _SECQ  = 2>&1 | tee
+  _HDR   = echo -e
+  _RUNSH = echo "-- $$t"; sh "$$t" || exit 1
 endif

 build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -240,27 +244,24 @@ preflight:
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh

 test-build-scripts:
-	@echo -e "\n....Running build/scripts/**/*_test.sh...."
-	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
+		$(_RUNSH); \
 	done
-	@echo -e "\n....Running .github/scripts/*_test.sh...."
-	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		echo "-- $$t"; \
-		sh "$$t" || exit 1; \
+	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
+	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		$(_RUNSH); \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
+	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
 		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
-	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
-	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
-	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
-	@echo "  test-build-scripts ok"
+	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
+	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
+	@$(_HDR) "  test-build-scripts ok"

 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
@@ -37,6 +37,13 @@ contract or an approved compatibility plan; it is not a package refactor.
 Only the subject is owned here today.
 The request (`account_id`) and reply (`vpc_id`, `error`) bodies are still declared as anonymous structs on each side; moving them into this package is a contract change that needs its own characterization first.

+## Volume held elsewhere
+
+A start refused because another node holds one of the instance's volumes is an `IncorrectState` error payload on the `ec2.start`, `ec2.start.<node>` and `ec2.cmd.<instance-id>` replies, and the AWS body cannot say why.
+The owning node adds the `X-Volume-Held-Elsewhere: true` header to that reply, the `ec2.start` forwarder relays it, and `StartInstances` wraps `ErrVolumeHeldElsewhere` around the error it rebuilds, so a caller matches it with `errors.Is` rather than reading the message.
+The owning node sets the header only from the EBS provider's `volume_held_elsewhere` code, which every provider returns for this refusal; no reader matches the message text.
+`VolumeHeldElsewhereText` is the wording viperblockd puts in that refusal's message, not a signal.
+
 ## Platform inventory

 `PlatformInventorySubject` (`ec2.v1.PlatformInventory`) is a request/reply route on which a daemon in the `spinifex-workers` queue group reports one account's key pairs, placement groups, launch templates and their `ec2-tags` entries from the EC2 owner stores.
@@ -1,7 +1,13 @@
 package ec2v1

-// VolumeHeldElsewhereText is the text of a start refused because another node
-// holds one of the instance's volumes. It crosses NATS as message text, so
-// readers match on it; the provider that refuses and every reader share this
-// string so they cannot drift.
+import "errors"
+
+// VolumeHeldElsewhereText is the message viperblockd gives a lease refusal. It
+// is wording for the operator; callers classify the refusal by its code.
 const VolumeHeldElsewhereText = "volume is leased by another owner"
+
+// ErrVolumeHeldElsewhere marks a start refused because another node still
+// holds one of the instance's volumes. The owning node sets it from the EBS
+// provider's held-elsewhere code, and a start reply carries it across NATS as
+// the X-Volume-Held-Elsewhere header rather than as message text.
+var ErrVolumeHeldElsewhere = errors.New("a volume is leased by another node")
@@ -95,7 +95,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab; on a host whose IPsec stack is libreswan it defaults to false and `--ipsec=true` is refused), `--host` (leader to join; falls back to multicast discovery), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab; on a host whose IPsec stack is libreswan it defaults to false and `--ipsec=true` is refused), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--advertise` (defaults to `--bind`, or the auto-detected WAN IP when `--bind` is 0.0.0.0), `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services`, `--skip-host-dns`, `--email`, `--no-telemetry`, `--predastore-compaction-interval`, `--join-timeout` (how long to keep retrying while the formation server is unreachable, default 20m), `--force` (join even though this node is already initialized or holds JetStream streams, **discarding its own CA, master key and JetStream store**; the store is removed only after formation succeeds, and the join is refused while NATS is running even with `--force`) | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations → points the host resolver at this node's local northstar so the Spinifex zones resolve from the node itself (opt out with `--skip-host-dns`) |

 ### Version
@@ -135,7 +135,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
+| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
 | `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |

@@ -92,6 +92,7 @@ type Shape struct {
 	Min             *float64
 	Max             *float64
 	Pattern         string
+	Documentation   string
 	LocationName    string
 	TimestampFormat string
 	XMLNamespace    XMLNamespace
@@ -443,6 +443,9 @@ func (t *smithyTranslator) applyShapeTraits(shape *Shape, traits smithyTraits) e
 	if shape.Pattern, err = traits.string("smithy.api#pattern"); err != nil {
 		return err
 	}
+	if shape.Documentation, err = traits.string("smithy.api#documentation"); err != nil {
+		return err
+	}
 	if shape.LocationName, err = traits.string("smithy.api#xmlName"); err != nil {
 		return err
 	}
@@ -7,9 +7,10 @@ package ecsgw

 import (
 	"bytes"
-	"context"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,10 +18,8 @@ import (
 	"strings"
 	"time"

-	"github.com/aws/aws-sdk-go/aws/credentials"
-	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
-
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )

 // targetPrefix is the X-Amz-Target service prefix the gateway's ECS dispatch
@@ -28,31 +27,26 @@ import (
 // suffix); this one mirrors the AWS JSON 1.1 convention.
 const targetPrefix = "AmazonEC2ContainerServiceV20141113."

-// CredentialsFunc yields the current SigV4 credentials. It is called once per
-// request so a provider backed by IMDS can rotate instance-role creds (and their
-// session token) transparently. Must be safe for concurrent use.
-type CredentialsFunc func(ctx context.Context) (accessKey, secretKey, sessionToken string, err error)
-
 // Client posts SigV4-signed AWS JSON 1.1 requests to the gateway for service
 // "ecs". One client is reused for register/heartbeat/state/poll.
 type Client struct {
 	baseURL    string
-	creds      CredentialsFunc
+	signer     *gwsign.Signer
 	region     string
 	httpClient *http.Client
 }

 // New builds a client. caPath optionally pins the gateway TLS CA; empty relies on
-// the system trust store. creds supplies the per-call instance-role credentials.
+// the system trust store. signer supplies the per-call instance-role credentials.
 // region defaults to us-east-1 when empty (SigV4 requires a non-empty region).
 // timeout bounds a single call — callers needing a long-poll pass a larger value
 // than the default register/heartbeat timeout.
-func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout time.Duration) (*Client, error) {
+func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout time.Duration) (*Client, error) {
 	if baseURL == "" {
 		return nil, fmt.Errorf("ecsgw: baseURL is required")
 	}
-	if creds == nil {
-		return nil, fmt.Errorf("ecsgw: credentials func is required")
+	if signer == nil {
+		return nil, fmt.Errorf("ecsgw: signer is required")
 	}
 	if region == "" {
 		region = "us-east-1"
@@ -79,7 +73,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t

 	return &Client{
 		baseURL: strings.TrimRight(baseURL, "/"),
-		creds:   creds,
+		signer:  signer,
 		region:  region,
 		httpClient: &http.Client{
 			Timeout:   timeout,
@@ -90,7 +84,7 @@ func New(baseURL, caPath string, creds CredentialsFunc, region string, timeout t

 // Call SigV4-signs and POSTs body under X-Amz-Target "<prefix><action>" to the
 // gateway root, returning the 2xx response body. Credentials are fetched per call
-// and the v4 signer emits X-Amz-Security-Token for instance-role (temporary)
+// and the signer emits X-Amz-Security-Token for instance-role (temporary)
 // creds, so the gateway's ASIA path verifies them. No retry; callers wrap.
 func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req, err := http.NewRequest(http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
@@ -100,12 +94,8 @@ func (c *Client) Call(action string, body []byte) ([]byte, error) {
 	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
 	req.Header.Set("X-Amz-Target", targetPrefix+action)

-	akid, secret, token, err := c.creds(req.Context())
-	if err != nil {
-		return nil, fmt.Errorf("retrieve credentials: %w", err)
-	}
-	signer := v4.NewSigner(credentials.NewStaticCredentials(akid, secret, token))
-	if _, err := signer.Sign(req, bytes.NewReader(body), "ecs", c.region, time.Now()); err != nil {
+	sum := sha256.Sum256(body)
+	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), "ecs", c.region); err != nil {
 		return nil, fmt.Errorf("sign request: %w", err)
 	}

@@ -1,24 +1,29 @@
 package ecsgw

 import (
-	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 )

-// testCreds returns a fixed temporary credential set (with a session token, as
-// IMDS instance-role creds carry).
-func testCreds(context.Context) (string, string, string, error) {
-	return "ASIATEST", "secret", "session-token-xyz", nil
+const testSecret = "secret"
+
+// testSigner signs with a fixed temporary credential set (with a session token,
+// as IMDS instance-role creds carry).
+func testSigner() *gwsign.Signer {
+	return gwsign.New(credentials.NewStaticCredentialsProvider("ASIATEST", testSecret, "session-token-xyz"))
 }

 func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 	t.Helper()
-	c, err := New(srv.URL, "", testCreds, "ap-southeast-2", time.Second)
+	c, err := New(srv.URL, "", testSigner(), "ap-southeast-2", time.Second)
 	if err != nil {
 		t.Fatalf("New: %v", err)
 	}
@@ -30,7 +35,14 @@ func newTestClient(t *testing.T, srv *httptest.Server) *Client {
 // 2xx body.
 func TestClient_CallSignsAndTargets(t *testing.T) {
 	var gotTarget, gotAuth, gotBody, gotToken string
+	var verifyErr error
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// sigv4.Parse reads and hashes the body itself, then restores it.
+		sr, err := sigv4.Parse(r)
+		if err == nil {
+			_, err = sr.Verify(testSecret, "ap-southeast-2", "ecs")
+		}
+		verifyErr = err
 		gotTarget = r.Header.Get("X-Amz-Target")
 		gotAuth = r.Header.Get("Authorization")
 		gotToken = r.Header.Get("X-Amz-Security-Token")
@@ -45,6 +57,9 @@ func TestClient_CallSignsAndTargets(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Call: %v", err)
 	}
+	if verifyErr != nil {
+		t.Fatalf("gateway-side SigV4 verify failed: %v", verifyErr)
+	}
 	if string(out) != `{"ok":true}` {
 		t.Errorf("body = %q", out)
 	}
@@ -78,10 +93,10 @@ func TestClient_CallErrorStatus(t *testing.T) {
 }

 func TestNew_Validation(t *testing.T) {
-	if _, err := New("", "", testCreds, "r", 0); err == nil {
+	if _, err := New("", "", testSigner(), "r", 0); err == nil {
 		t.Error("want error for empty baseURL")
 	}
 	if _, err := New("https://gw:9999", "", nil, "r", 0); err == nil {
-		t.Error("want error for nil credentials func")
+		t.Error("want error for nil signer")
 	}
 }
@@ -15,8 +15,7 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 )

 // Signer holds a credential provider and signs requests in place.
@@ -24,6 +23,12 @@ type Signer struct {
 	provider aws.CredentialsProvider
 }

+// New returns a Signer over provider. Wrap a provider that fetches remotely in
+// aws.NewCredentialsCache, since Sign retrieves on every call.
+func New(provider aws.CredentialsProvider) *Signer {
+	return &Signer{provider: provider}
+}
+
 // NewStatic returns a Signer using fixed long-lived keys (no session token).
 func NewStatic(accessKey, secretKey string) *Signer {
 	return &Signer{provider: credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")}
@@ -46,12 +51,14 @@ func NewIMDS(ctx context.Context, region string) (*Signer, error) {

 // NewInstanceRole returns a Signer whose credentials come only from the
 // instance metadata service: no environment, profile or static-key fallback.
+// A failed refresh of expired credentials is an error, not the stale key.
 func NewInstanceRole() *Signer {
-	return newInstanceRole(imds.New(imds.Options{}))
+	return newInstanceRole("")
 }

-func newInstanceRole(client *imds.Client) *Signer {
-	return &Signer{provider: aws.NewCredentialsCache(ec2rolecreds.New(func(o *ec2rolecreds.Options) { o.Client = client }))}
+// newInstanceRole reads from the IMDS root base; "" means the default endpoint.
+func newInstanceRole(base string) *Signer {
+	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base))}
 }

 // EnsureCredentials retrieves once from the provider, surfacing the error so a
@@ -12,7 +12,6 @@ import (

 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
 )

@@ -145,7 +144,7 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}))
 	t.Cleanup(srv.Close)

-	s := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	s := newInstanceRole(srv.URL)
 	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
 	if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
 		t.Fatal(err)
@@ -155,10 +154,42 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}

 	srv.Close()
-	cold := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	cold := newInstanceRole(srv.URL)
 	if err := cold.EnsureCredentials(context.Background()); err == nil {
 		t.Fatal("environment keys were used when IMDS was unreachable")
 	}
 }

+// Once the cached instance-role key has expired, a refresh that cannot reach
+// IMDS must fail the sign rather than reuse the expired key with a new expiry.
+func TestNewInstanceRole_FailedRefreshErrorsInsteadOfStaleKey(t *testing.T) {
+	const staleKey = "ASIASTALEEXAMPLE0001"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + staleKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2000-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	s := newInstanceRole(srv.URL)
+	if err := s.EnsureCredentials(t.Context()); err != nil {
+		t.Fatalf("first retrieve: %v", err)
+	}
+
+	srv.Close()
+	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
+	err := s.Sign(req, emptySHA, testService, testRegion)
+	if err == nil {
+		t.Fatalf("Sign after failed refresh = nil, signed with %q", req.Header.Get("Authorization"))
+	}
+	if auth := req.Header.Get("Authorization"); auth != "" {
+		t.Fatalf("request carries Authorization %q after a failed refresh", auth)
+	}
+}
+
 const emptySHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
@@ -21,31 +21,18 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )

-// Credentials is a resolved instance-role credential set with parsed expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // Fetch resolves the instance role then its credentials via the SDK's IMDSv2
 // client and the ec2rolecreds provider. base is the IMDS root, e.g.
 // http://169.254.169.254/latest.
-func Fetch(client *http.Client, base string) (Credentials, error) {
+func Fetch(client *http.Client, base string) (aws.Credentials, error) {
 	creds, err := NewProvider(client, base).Retrieve(context.Background())
 	if err != nil {
-		return Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("fetch IMDS credentials: %w", err)
 	}
 	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
-		return Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
+		return aws.Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
 	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
+	return creds, nil
 }

 // NewProvider builds an ec2rolecreds provider against base, an IMDS root URL
@@ -1,9 +1,11 @@
 package imdscreds

 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 	"time"

@@ -60,8 +62,8 @@ func TestFetch_OK(t *testing.T) {
 	if got.AccessKeyID != "AKIATEST" || got.SecretAccessKey != "secrettest" || got.SessionToken != "sessiontoken" {
 		t.Errorf("creds mismatch: %+v", got)
 	}
-	if !got.Expiration.Equal(exp) {
-		t.Errorf("Expiration = %v, want %v", got.Expiration, exp)
+	if !got.Expires.Equal(exp) {
+		t.Errorf("Expires = %v, want %v", got.Expires, exp)
 	}
 }

@@ -132,3 +134,46 @@ func TestNewProvider_WithholdsExtendOnFailure(t *testing.T) {
 		t.Error("provider dropped AdjustExpiresBy, which a caller using an ExpiryWindow relies on")
 	}
 }
+
+// Behind aws.CredentialsCache, as the ecs-agent uses it, a refresh that fails
+// against an expired credential must surface as an error rather than hand back
+// the dead key with a pushed-out expiry.
+func TestNewProvider_CachedFailedRefreshErrors(t *testing.T) {
+	var hits atomic.Int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits.Add(1) > 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"AccessKeyId":     "AKIA",
+			"SecretAccessKey": "secret",
+			"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+		})
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+
+	cache := aws.NewCredentialsCache(NewProvider(srv.Client(), srv.URL+"/latest"))
+	first, err := cache.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("first Retrieve: %v", err)
+	}
+	got, err := cache.Retrieve(context.Background())
+	if err == nil {
+		t.Fatalf("refresh failure was reported as success: %+v", got)
+	}
+	if got.Expires.After(first.Expires) {
+		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expires, got.Expires)
+	}
+}
@@ -8,8 +8,8 @@ package stsauth
 import (
 	"fmt"
 	"net/http"
-	"time"

+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -20,24 +20,15 @@ import (
 // scope and the gateway does not enforce a specific value.
 const defaultRegion = "us-east-1"

-// Credentials is the decoded result of AssumeRole: a SigV4 credential set bound
-// to the assumed role, with its expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
 // AssumeRole builds an aws-sdk STS client pointed at the gateway and calls
 // AssumeRole with the supplied static (instance-role) credentials, returning the
 // assumed-role credential set. httpClient must trust the gateway CA.
-func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (Credentials, error) {
+func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken, roleARN, sessionName string) (awsv2.Credentials, error) {
 	if region == "" {
 		region = defaultRegion
 	}
 	if roleARN == "" || sessionName == "" {
-		return Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
+		return awsv2.Credentials{}, fmt.Errorf("stsauth: roleARN and sessionName are required")
 	}
 	sess, err := session.NewSession(&aws.Config{
 		Region:      aws.String(region),
@@ -47,7 +38,7 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		DisableSSL:  aws.Bool(false),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("new session: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("new session: %w", err)
 	}

 	out, err := sts.New(sess).AssumeRole(&sts.AssumeRoleInput{
@@ -55,16 +46,17 @@ func AssumeRole(region, gatewayURL string, httpClient *http.Client, akid, secret
 		RoleSessionName: aws.String(sessionName),
 	})
 	if err != nil {
-		return Credentials{}, fmt.Errorf("AssumeRole: %w", err)
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole: %w", err)
 	}
 	if out.Credentials == nil {
-		return Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
+		return awsv2.Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
 	}
 	c := out.Credentials
-	return Credentials{
+	return awsv2.Credentials{
 		AccessKeyID:     aws.StringValue(c.AccessKeyId),
 		SecretAccessKey: aws.StringValue(c.SecretAccessKey),
 		SessionToken:    aws.StringValue(c.SessionToken),
-		Expiration:      aws.TimeValue(c.Expiration),
+		CanExpire:       c.Expiration != nil,
+		Expires:         aws.TimeValue(c.Expiration),
 	}, nil
 }
@@ -41,8 +41,8 @@ func TestAssumeRole_DecodesCredentials(t *testing.T) {
 		t.Errorf("bad creds: %+v", creds)
 	}
 	want, _ := time.Parse(time.RFC3339, "2031-01-02T03:04:05Z")
-	if !creds.Expiration.Equal(want) {
-		t.Errorf("Expiration = %v, want %v", creds.Expiration, want)
+	if !creds.Expires.Equal(want) {
+		t.Errorf("Expires = %v, want %v", creds.Expires, want)
 	}
 	// The request must carry the AssumeRole action and the role ARN we asked for.
 	if !strings.Contains(gotBody, "Action=AssumeRole") {
@@ -0,0 +1,29 @@
+package testutil
+
+import (
+	"testing"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// RequireAWSError asserts err carries the AWS code and exact client-facing
+// message; an empty message asserts the bare code with no message attached.
+func RequireAWSError(t *testing.T, err error, code, message string) {
+	t.Helper()
+	require.Error(t, err)
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+	assert.Equal(t, message, msg)
+}
+
+// RequireAWSCode asserts err carries the AWS code, whatever its message.
+func RequireAWSCode(t *testing.T, err error, code string) {
+	t.Helper()
+	require.Error(t, err)
+	got, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+}
@@ -167,11 +167,11 @@ func registerVpcdStub(t *testing.T, nc *nats.Conn, defaultPayload []byte) {
 	})
 }

-// SeedKV creates a KV bucket and populates it with the given entries.
+// SeedKV creates or updates a KV bucket and populates it with the given entries.
 // Returns the KV handle for further use.
 func SeedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue {
 	t.Helper()
-	kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
+	kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
 	require.NoError(t, err)
 	for k, v := range entries {
 		_, err := kv.Put(t.Context(), k, v)
@@ -0,0 +1,19 @@
+package testutil
+
+import (
+	"testing"
+
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSeedKV_ExistingBucket(t *testing.T) {
+	_, _, js := StartTestJetStream(t)
+	_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "seeded", History: 5})
+	require.NoError(t, err)
+
+	kv := SeedKV(t, js, "seeded", map[string][]byte{"k": []byte("v")})
+	entry, err := kv.Get(t.Context(), "k")
+	require.NoError(t, err)
+	require.Equal(t, []byte("v"), entry.Value())
+}
@@ -9,8 +9,7 @@ Wants=network-online.target

 [Service]
 Type=simple
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds,
-# containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
 # cloud-init at launch. Absent file is tolerated — the agent falls back to
 # its built-in defaults.
 EnvironmentFile=-/etc/spinifex-ecs/agent.env
@@ -3,8 +3,7 @@
 # IMDSv2, registers + heartbeats + polls task assignments through the AWS gateway
 # over TLS+SigV4 (no NATS), and pulls task images through containerd.
 #
-# Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-# socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.
+# Static config (gateway URL, CA, region, cluster, containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by cloud-init at launch.

 description="Spinifex ECS container-instance agent"
 command="/usr/local/bin/ecs-agent"
@@ -28,7 +27,7 @@ start_pre() {
        # shellcheck disable=SC1091
        . /etc/spinifex-ecs/agent.env
        export ECS_GATEWAY_URL ECS_GATEWAY_CA ECS_REGION ECS_CLUSTER \
-            ECS_ACCESS_KEY ECS_SECRET_KEY ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
+            ECS_IMDS_BASE ECS_CONTAINERD_SOCKET \
            ECS_HEARTBEAT_INTERVAL ECS_POLL_INTERVAL
    fi
 }
@@ -12,8 +12,9 @@ import (
 	"strings"
 	"time"

+	"github.com/aws/aws-sdk-go-v2/aws"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
 	ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )

@@ -102,7 +103,9 @@ func New(cfg Config) (*Agent, error) {
 		AgentVersion: cfg.AgentVersion,
 	}

-	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	// The cache refetches only once the real expiry passes, and a failed
+	// refresh is an error rather than an extended expiry.
+	creds := aws.NewCredentialsCache(imdscreds.NewProvider(imdsClient, cfg.IMDSBase))
 	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)

 	// A dial that fails here is not fatal and not final: the node starts the
@@ -21,17 +21,14 @@ const (

 // Config holds the static settings the agent reads at boot. Cluster identity
 // (account ID, instance ID) is discovered at runtime from IMDS, not configured;
-// only the cluster *name* the instance was launched into is static. AccessKey /
-// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
-// gateway calls with instance-role credentials from IMDS; the agent never holds
-// a NATS token.
+// only the cluster *name* the instance was launched into is static. The agent
+// SigV4-signs gateway calls with instance-role credentials from IMDS and never
+// holds a NATS token.
 type Config struct {
 	GatewayURL       string
 	GatewayCA        string
 	Region           string
 	ClusterName      string
-	AccessKey        string
-	SecretKey        string
 	IMDSBase         string
 	ContainerdSocket string
 	Heartbeat        time.Duration
@@ -53,8 +50,6 @@ func LoadConfig(envFile string) Config {
 		GatewayCA:        get("ECS_GATEWAY_CA"),
 		Region:           get("ECS_REGION"),
 		ClusterName:      get("ECS_CLUSTER"),
-		AccessKey:        get("ECS_ACCESS_KEY"),
-		SecretKey:        get("ECS_SECRET_KEY"),
 		IMDSBase:         get("ECS_IMDS_BASE"),
 		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
 	}
@@ -10,7 +10,7 @@ import (
 func TestLoadConfig_Defaults(t *testing.T) {
 	// Ensure env vars don't leak in from the host.
 	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
-		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CLUSTER", "ECS_IMDS_BASE",
 		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
 		t.Setenv(k, "")
 	}
@@ -43,7 +43,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
-	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")

 	cfg := LoadConfig(envFile)
 	if cfg.GatewayURL != "https://gw.env" {
@@ -55,9 +54,6 @@ func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
 	if cfg.ClusterName != "prod" {
 		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
 	}
-	if cfg.AccessKey != "AKIAENV" {
-		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
-	}
 }

 func TestLoadConfig_HeartbeatOverride(t *testing.T) {
@@ -1,17 +1,17 @@
 package agent

 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"time"

+	awsv2 "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"

 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/ecsgw"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )

@@ -47,26 +47,14 @@ const pollTimeout = 30 * time.Second

 // newGatewayControlPlane builds the SigV4 client signing with the instance-role
 // credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
-func newGatewayControlPlane(cfg Config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
-	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+func newGatewayControlPlane(cfg Config, creds awsv2.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, gwsign.New(creds), cfg.Region, pollTimeout)
 	if err != nil {
 		return nil, err
 	}
 	return &gatewayControlPlane{client: client}, nil
 }

-// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
-// per-call credentials hook.
-func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
-	return func(ctx context.Context) (string, string, string, error) {
-		c, err := p.Retrieve(ctx)
-		if err != nil {
-			return "", "", "", err
-		}
-		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
-	}
-}
-
 func (g *gatewayControlPlane) Register(id identity) error {
 	in := &ecs.RegisterContainerInstanceInput{
 		Cluster:                  aws.String(id.ClusterName),
@@ -8,17 +8,17 @@ import (
 	"net/http/httptest"
 	"testing"

+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 )

-// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// fixedCredsProvider is a static aws.CredentialsProvider for tests
 // that only need a valid SigV4 signature, not real IMDS rotation.
 type fixedCredsProvider struct{}

-func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
-	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+func (fixedCredsProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
 }

 func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
@@ -13,9 +13,9 @@ import (
 	"sync"
 	"time"

+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
 	"github.com/mulgadc/spinifex/internal/stsauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 )

 const (
@@ -56,7 +56,7 @@ type credRole struct {

 type credEntry struct {
 	roleARN string
-	creds   stsauth.Credentials
+	creds   aws.Credentials
 }

 // credEndpoint is the host-netns HTTP server that hands a task its IAM role
@@ -64,7 +64,7 @@ type credEntry struct {
 // mints/caches credentials on demand by assuming the role over the gateway with
 // the agent's instance credentials.
 type credEndpoint struct {
-	provider   credentials.CredentialsProvider
+	provider   aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -75,7 +75,7 @@ type credEndpoint struct {

 	// assume mints fresh credentials for a role; defaults to AssumeRole over the
 	// gateway and is overridable in tests.
-	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+	assume func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error)

 	mu         sync.Mutex
 	roles      map[string]credRole  // credID -> task role
@@ -89,7 +89,7 @@ type credEndpoint struct {
 // newCredEndpoint builds the endpoint. An empty ip/port falls back to the
 // AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
 // adds the link-local address to the dummy interface (real or fake).
-func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
 	if ip == "" {
 		ip = defaultCredEndpointIP
 	}
@@ -165,7 +165,7 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
-		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+		Expiration:      creds.Expires.UTC().Format(time.RFC3339),
 	}); err != nil {
 		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
@@ -176,12 +176,12 @@ var errUnknownCredID = fmt.Errorf("unknown credId")

 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (aws.Credentials, credRole, error) {
 	c.mu.Lock()
 	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, credRole{}, errUnknownCredID
+		return aws.Credentials{}, credRole{}, errUnknownCredID
 	}
 	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
@@ -191,7 +191,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden

 	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, role, err
+		return aws.Credentials{}, role, err
 	}

 	c.mu.Lock()
@@ -204,7 +204,7 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 // roleARN over the gateway — the execution-role path for ECR image pulls. It
 // caches the assumed set and re-assumes within the refresh margin. session scopes
 // the STS session name.
-func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+func (c *credEndpoint) AssumeProvider(roleARN, session string) aws.CredentialsProvider {
 	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
 }

@@ -216,39 +216,34 @@ type assumeProvider struct {
 	session string

 	mu     sync.Mutex
-	cached stsauth.Credentials
+	cached aws.Credentials
 }

-var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+var _ aws.CredentialsProvider = (*assumeProvider)(nil)

-func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+func (p *assumeProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if !credValid(p.cached, credRefreshMargin) {
 		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
 		if err != nil {
-			return credentials.Credentials{}, err
+			return aws.Credentials{}, err
 		}
 		p.cached = creds
 	}
-	return credentials.Credentials{
-		AccessKeyID:     p.cached.AccessKeyID,
-		SecretAccessKey: p.cached.SecretAccessKey,
-		SessionToken:    p.cached.SessionToken,
-		Expiration:      p.cached.Expiration,
-	}, nil
+	return p.cached, nil
 }

 // assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
 // against the gateway with the agent's instance credentials.
-func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 	instCreds, err := c.provider.Retrieve(ctx)
 	if err != nil {
-		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+		return aws.Credentials{}, fmt.Errorf("instance credentials: %w", err)
 	}
 	client, err := c.client()
 	if err != nil {
-		return stsauth.Credentials{}, err
+		return aws.Credentials{}, err
 	}
 	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
 		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
@@ -387,14 +382,14 @@ func (c *credEndpoint) delEndpointAddr() {
 }

 // credValid reports whether creds are usable now with margin to spare.
-func credValid(c stsauth.Credentials, margin time.Duration) bool {
+func credValid(c aws.Credentials, margin time.Duration) bool {
 	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
 		return false
 	}
-	if c.Expiration.IsZero() {
+	if c.Expires.IsZero() {
 		return true
 	}
-	return time.Now().Add(margin).Before(c.Expiration)
+	return time.Now().Add(margin).Before(c.Expires)
 }

 // sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
@@ -9,15 +9,15 @@ import (
 	"testing"
 	"time"

-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )

 // newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
 // counting calls so cache behaviour is observable. run is nil (no iface plumbing).
-func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+func newTestCredEndpoint(creds aws.Credentials) (*credEndpoint, *int32) {
 	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
 	var calls int32
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
 		atomic.AddInt32(&calls, 1)
 		return creds, nil
 	}
@@ -26,8 +26,8 @@ func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {

 func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expires: exp,
 	})
 	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")

@@ -54,11 +54,11 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 // The STS session name lands in the assumed-role ARN and in STS logs, so it is
 // built from the public task ID and never from the secret credID.
 func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c, _ := newTestCredEndpoint(aws.Credentials{})
 	var gotSession string
-	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+	c.assume = func(ctx context.Context, roleARN, session string) (aws.Credentials, error) {
 		gotSession = session
-		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+		return aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
 	}
 	c.Register("secret-cred", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -72,7 +72,7 @@ func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
 }

 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, calls := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
 	if rr.Code != http.StatusNotFound {
@@ -84,7 +84,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 }

 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
@@ -94,8 +94,8 @@ func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 }

 func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
@@ -112,8 +112,8 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {

 func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	// Expiry inside the refresh margin -> every hit re-assumes.
-	c, calls := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	c, calls := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Minute),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
@@ -126,7 +126,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 }

 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
@@ -137,9 +137,9 @@ func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 }

 func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{})
-	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
-		return stsauth.Credentials{}, context.DeadlineExceeded
+	c, _ := newTestCredEndpoint(aws.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
+		return aws.Credentials{}, context.DeadlineExceeded
 	}
 	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
@@ -150,8 +150,8 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 }

 func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{
-		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	c, _ := newTestCredEndpoint(aws.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expires: time.Now().Add(time.Hour),
 	})
 	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80
@@ -5,7 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"

-	"github.com/mulgadc/spinifex/internal/stsauth"
+	"github.com/aws/aws-sdk-go-v2/aws"

 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
@@ -30,7 +30,7 @@ func TestTaskCredID(t *testing.T) {
 // The task ID is public (it is the task ARN suffix), so naming another task's ID
 // at the credential endpoint must not yield that task's role.
 func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
-	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	c, _ := newTestCredEndpoint(aws.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
 	a := Agent{cred: c}
 	get := func(id string) int {
 		rr := httptest.NewRecorder()
@@ -5,8 +5,7 @@
 // task assignments, runs them through containerd, and reports state back over
 // the gateway.
 //
-// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
-// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// Static config (gateway URL, CA, region, cluster, containerd socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
 // (KEY=value); real env vars override it. Host identity (account, instance, AZ)
 // comes from IMDS at boot.
 package agent
@@ -7,8 +7,8 @@ import (
 	"strings"
 	"sync"

+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/mulgadc/spinifex/internal/ecrauth"
-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
 	"github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
 )

@@ -22,7 +22,7 @@ import (
 // pulled, so the agent can still register + heartbeat on a host with absent or
 // wrong ECR config.
 type ecrResolver struct {
-	creds      credentials.CredentialsProvider
+	creds      aws.CredentialsProvider
 	region     string
 	gatewayURL string
 	caPath     string
@@ -35,13 +35,13 @@ var _ runtime.Resolver = (*ecrResolver)(nil)

 // newECRResolver builds a resolver with an injected HTTP client (tests). The
 // client is used as-is and never rebuilt from caPath.
-func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+func newECRResolver(creds aws.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
 }

 // newLazyECRResolver builds a resolver that constructs its gateway HTTP client
 // from caPath on first use (production path).
-func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+func newLazyECRResolver(creds aws.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
 	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
 }

@@ -14,16 +14,16 @@ import (
 	"testing"
 	"time"

-	"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
 )

-// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+// stubCreds is a aws.CredentialsProvider returning fixed creds or error.
 type stubCreds struct {
-	c   credentials.Credentials
+	c   aws.Credentials
 	err error
 }

-func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+func (s stubCreds) Retrieve(context.Context) (aws.Credentials, error) {
 	return s.c, s.err
 }

@@ -58,9 +58,9 @@ func trusting(srv *httptest.Server) *http.Client {

 func TestECRResolver_Authorize(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))

@@ -85,7 +85,7 @@ func TestECRResolver_CredsError(t *testing.T) {

 func TestECRResolver_GatewayError(t *testing.T) {
 	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
 		t.Fatal("expected gateway error")
@@ -93,7 +93,7 @@ func TestECRResolver_GatewayError(t *testing.T) {
 }

 func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
-	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
 	// Construction with a bogus CA path must not fail — only Authorize does.
 	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
 	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
@@ -141,9 +141,9 @@ func TestIsECRRef(t *testing.T) {
 // registry, even though it lacks a ".dkr.ecr." name.
 func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
 	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
-	creds := stubCreds{c: credentials.Credentials{
+	creds := stubCreds{c: aws.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
-		Expiration: time.Now().Add(time.Hour),
+		Expires: time.Now().Add(time.Hour),
 	}}
 	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))

@@ -1,41 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/mulgadc/spinifex/internal/imdscreds"
-)
-
-// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
-// which refetches once the real expiry passes. A failed refresh is an error
-// here, not an extended expiry. Safe for concurrent use.
-type IMDSProvider struct {
-	cache *aws.CredentialsCache
-}
-
-var _ CredentialsProvider = (*IMDSProvider)(nil)
-
-// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
-// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
-// default HTTP client.
-func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
-	provider := imdscreds.NewProvider(client, base)
-	return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
-}
-
-// Retrieve returns cached credentials when still valid, else fetches fresh ones.
-func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
-	creds, err := p.cache.Retrieve(ctx)
-	if err != nil {
-		return Credentials{}, err
-	}
-	return Credentials{
-		AccessKeyID:     creds.AccessKeyID,
-		SecretAccessKey: creds.SecretAccessKey,
-		SessionToken:    creds.SessionToken,
-		Expiration:      creds.Expires,
-	}, nil
-}
@@ -1,164 +0,0 @@
-package credentials
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
-// real /latest prefix the SDK IMDS client always requests.
-func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
-	t.Helper()
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		// Real IMDS echoes the requested TTL back as a response header; the SDK
-		// client parses it from there, not the body.
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits != nil {
-			atomic.AddInt32(hits, 1)
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-func TestRetrieve_FetchesAndCaches(t *testing.T) {
-	var hits int32
-	// Kept under an hour: ec2rolecreds caps Expires to now+1h.
-	exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		"Expiration":      exp.Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	got, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("Retrieve: %v", err)
-	}
-	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
-		t.Errorf("creds mismatch: %+v", got)
-	}
-
-	// Second call well within validity must not re-hit IMDS.
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatalf("Retrieve 2: %v", err)
-	}
-	if hits != 1 {
-		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
-	}
-}
-
-func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
-	var hits int32
-	// Already expired: aws.CredentialsCache refetches once the stored
-	// credentials' real expiry has passed, regardless of ExpiryWindow.
-	srv := imdsStub(t, &hits, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Expiration":      time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := p.Retrieve(context.Background()); err != nil {
-		t.Fatal(err)
-	}
-	if hits != 2 {
-		t.Errorf("hits = %d, want 2 (expired refetch)", hits)
-	}
-}
-
-// imdsStubFailAfterFirst serves creds on the first credential request and 500s
-// on every one after it, so a test can drive a refresh that fails against a
-// credential the cache has already stored.
-func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
-	t.Helper()
-	var hits atomic.Int32
-	mux := http.NewServeMux()
-	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-		_, _ = w.Write([]byte("v2-token"))
-	})
-	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
-			_, _ = w.Write([]byte("node-role"))
-			return
-		}
-		if hits.Add(1) > 1 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(creds)
-	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// A refresh that fails must surface as an error. Left to itself the SDK's
-// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
-// key with Expires pushed 5-15 minutes out, so the agent would sign requests
-// with a credential the control plane has already rejected — indefinitely,
-// since every subsequent refresh fails the same way.
-func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
-	srv := imdsStubFailAfterFirst(t, map[string]string{
-		"Code":            "Success",
-		"AccessKeyId":     "AKIA",
-		"SecretAccessKey": "secret",
-		"Token":           "session",
-		// Already expired, so the next Retrieve must go back to IMDS.
-		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
-	})
-
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	first, err := p.Retrieve(context.Background())
-	if err != nil {
-		t.Fatalf("first Retrieve: %v", err)
-	}
-
-	got, err := p.Retrieve(context.Background())
-	if err == nil {
-		t.Fatalf("refresh failure was reported as success: %+v", got)
-	}
-	if got.Expiration.After(first.Expiration) {
-		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
-	}
-}
-
-func TestRetrieve_CancelledContext(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	if _, err := p.Retrieve(ctx); err == nil {
-		t.Fatal("expected error from cancelled context")
-	}
-}
-
-func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
-	srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
-	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
-	if _, err := p.Retrieve(context.Background()); err == nil {
-		t.Fatal("expected error when Code != Success")
-	}
-}
@@ -1,24 +0,0 @@
-// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
-// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
-// with the host's instance-role credentials, fetched from IMDS and cached until
-// near expiry.
-package credentials
-
-import (
-	"context"
-	"time"
-)
-
-// Credentials is a SigV4 credential set with an expiry.
-type Credentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
-	SessionToken    string
-	Expiration      time.Time
-}
-
-// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
-// must be safe for concurrent use.
-type CredentialsProvider interface {
-	Retrieve(ctx context.Context) (Credentials, error)
-}
@@ -80,7 +80,17 @@ func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.I
 	if err != nil {
 		return "", "", err
 	}
-	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}, caCert, caKey)
 	if err != nil {
 		return "", "", err
 	}
@@ -181,8 +191,9 @@ func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
 	return ec, nil
 }

-// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
-func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+// Mint generates an ECDSA-P256 key, issues tmpl for it with a random serial and
+// signs it with parentKey, or self-signs when parent is nil. tmpl is not modified.
+func Mint(tmpl, parent *x509.Certificate, parentKey *ecdsa.PrivateKey) (certPEM, keyPEM []byte, err error) {
 	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("generate key: %w", err)
@@ -191,19 +202,12 @@ func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns
 	if err != nil {
 		return nil, nil, fmt.Errorf("serial: %w", err)
 	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: cn},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		DNSNames:              dns,
-		IPAddresses:           ips,
+	t := *tmpl
+	t.SerialNumber = serial
+	if parent == nil {
+		parent, parentKey = &t, priv
 	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	der, err := x509.CreateCertificate(rand.Reader, &t, parent, &priv.PublicKey, parentKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("create cert: %w", err)
 	}
@@ -1,15 +1,11 @@
 package podidentity

 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"path/filepath"
@@ -83,17 +79,8 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 	if leafCurrent(certPath, keyPath, ca, spec) {
 		return nil
 	}
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return fmt.Errorf("serial: %w", err)
-	}
 	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
 		Subject:               pkix.Name{CommonName: spec.cn},
 		NotBefore:             now.Add(-time.Hour),
 		NotAfter:              now.Add(certValidity),
@@ -101,20 +88,15 @@ func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec)
 		ExtKeyUsage:           []x509.ExtKeyUsage{spec.usage},
 		BasicConstraintsValid: true,
 		IPAddresses:           spec.ips,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, ca, &priv.PublicKey, caKey)
+	}, ca, caKey)
 	if err != nil {
-		return fmt.Errorf("create certificate: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return fmt.Errorf("marshal key: %w", err)
-	}
-	// Key first: a cert without its key is never published.
-	if err := writeAtomic(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
 		return err
 	}
-	return writeAtomic(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
+	// Key first: a cert without its key is never published.
+	if err := writeAtomic(keyPath, keyPEM, 0o600); err != nil {
+		return err
+	}
+	return writeAtomic(certPath, certPEM, 0o644)
 }

 // leafCurrent reports whether the cached pair loads, matches, was signed by ca,
@@ -1,21 +1,17 @@
 package tokenwebhook

 import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
-	"encoding/pem"
 	"fmt"
-	"math/big"
 	"net"
 	"os"
 	"time"

 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
 )

 // serverTLSConfig is the webhook's loopback serving TLS config.
@@ -41,7 +37,18 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 		}
 	}

-	certPEM, keyPEM, err := generateSelfSigned()
+	now := time.Now().UTC()
+	certPEM, keyPEM, err := konnectivitycert.Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
+		DNSNames:              []string{"localhost"},
+	}, nil, nil)
 	if err != nil {
 		return tls.Certificate{}, nil, err
 	}
@@ -58,41 +65,6 @@ func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error
 	return cert, certPEM, nil
 }

-func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return nil, nil, fmt.Errorf("serial: %w", err)
-	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: "eks-token-webhook"},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
-		DNSNames:              []string{"localhost"},
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create cert: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("marshal key: %w", err)
-	}
-	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	return certPEM, keyPEM, nil
-}
-
 // writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
 // --authentication-token-webhook-config-file. It points the apiserver at the
 // loopback webhook and trusts its self-signed serving cert as the CA bundle.
@@ -14,9 +14,6 @@ package webhookcert

 import (
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
@@ -24,11 +21,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math/big"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
 )

 // Config names the webhook certificate and where it is cached.
@@ -69,13 +67,25 @@ func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err

 	if c, cerr := os.ReadFile(certPath); cerr == nil {
 		if k, kerr := os.ReadFile(keyPath); kerr == nil {
-			if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
-				return c, k, nil
+			if blk, _ := pem.Decode(c); blk != nil && len(k) > 0 {
+				if _, perr := x509.ParseCertificate(blk.Bytes); perr == nil {
+					return c, k, nil
+				}
 			}
 		}
 	}

-	certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
+	now := time.Now().UTC()
+	certPEM, keyPEM, err = konnectivitycert.Mint(&x509.Certificate{
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		DNSNames:              dnsNames,
+	}, nil, nil)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -91,51 +101,6 @@ func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err
 	return certPEM, keyPEM, nil
 }

-// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
-// a server cert, with the given common name and DNS SANs.
-func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("generate key: %w", err)
-	}
-	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	if err != nil {
-		return nil, nil, fmt.Errorf("serial: %w", err)
-	}
-	now := time.Now().UTC()
-	tmpl := x509.Certificate{
-		SerialNumber:          serial,
-		Subject:               pkix.Name{CommonName: cn},
-		NotBefore:             now.Add(-time.Hour),
-		NotAfter:              now.AddDate(10, 0, 0),
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-		DNSNames:              dnsNames,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create cert: %w", err)
-	}
-	keyDER, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, nil, fmt.Errorf("marshal key: %w", err)
-	}
-	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
-	return certPEM, keyPEM, nil
-}
-
-// decodeFirst returns the DER bytes of the first PEM block, or nil.
-func decodeFirst(pemBytes []byte) []byte {
-	blk, _ := pem.Decode(pemBytes)
-	if blk == nil {
-		return nil
-	}
-	return blk.Bytes
-}
-
 func splitCSV(s string) []string {
 	var out []string
 	for p := range strings.SplitSeq(s, ",") {
@@ -2,6 +2,7 @@ package webhookcert

 import (
 	"crypto/x509"
+	"encoding/pem"
 	"os"
 	"path/filepath"
 	"testing"
@@ -14,7 +15,11 @@ const (

 func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
 	t.Helper()
-	leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
+	blk, _ := pem.Decode(certPEM)
+	if blk == nil {
+		t.Fatal("no PEM block in cert")
+	}
+	leaf, err := x509.ParseCertificate(blk.Bytes)
 	if err != nil {
 		t.Fatalf("parse cert: %v", err)
 	}
@@ -8,6 +8,7 @@ import (

 	"github.com/aws/aws-sdk-go/service/acm"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 	"github.com/nats-io/nats.go"
 )

@@ -17,7 +18,7 @@ const natsTimeout = 30 * time.Second
 // RequestCertificate — CertificateManager.RequestCertificate.
 func RequestCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.RequestCertificateOutput, error) {
 	input := new(acm.RequestCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 	return natsmsg.NATSRequest[acm.RequestCertificateOutput](ctx, natsConn, "acm.RequestCertificate", input, natsTimeout, accountID)
@@ -26,7 +27,7 @@ func RequestCertificate(ctx context.Context, natsConn *nats.Conn, accountID stri
 // ImportCertificate — CertificateManager.ImportCertificate.
 func ImportCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.ImportCertificateOutput, error) {
 	input := new(acm.ImportCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.ImportCertificateOutput](ctx, natsConn, "acm.ImportCertificate", input, natsTimeout, accountID)
@@ -35,7 +36,7 @@ func ImportCertificate(ctx context.Context, natsConn *nats.Conn, accountID strin
 // DescribeCertificate — CertificateManager.DescribeCertificate.
 func DescribeCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.DescribeCertificateOutput, error) {
 	input := new(acm.DescribeCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.DescribeCertificateOutput](ctx, natsConn, "acm.DescribeCertificate", input, natsTimeout, accountID)
@@ -44,7 +45,7 @@ func DescribeCertificate(ctx context.Context, natsConn *nats.Conn, accountID str
 // GetCertificate — CertificateManager.GetCertificate.
 func GetCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.GetCertificateOutput, error) {
 	input := new(acm.GetCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.GetCertificateOutput](ctx, natsConn, "acm.GetCertificate", input, natsTimeout, accountID)
@@ -53,7 +54,7 @@ func GetCertificate(ctx context.Context, natsConn *nats.Conn, accountID string,
 // ListCertificates — CertificateManager.ListCertificates.
 func ListCertificates(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.ListCertificatesOutput, error) {
 	input := new(acm.ListCertificatesInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.ListCertificatesOutput](ctx, natsConn, "acm.ListCertificates", input, natsTimeout, accountID)
@@ -62,7 +63,7 @@ func ListCertificates(ctx context.Context, natsConn *nats.Conn, accountID string
 // DeleteCertificate — CertificateManager.DeleteCertificate.
 func DeleteCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.DeleteCertificateOutput, error) {
 	input := new(acm.DeleteCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.DeleteCertificateOutput](ctx, natsConn, "acm.DeleteCertificate", input, natsTimeout, accountID)
@@ -71,7 +72,7 @@ func DeleteCertificate(ctx context.Context, natsConn *nats.Conn, accountID strin
 // ListTagsForCertificate — CertificateManager.ListTagsForCertificate.
 func ListTagsForCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.ListTagsForCertificateOutput, error) {
 	input := new(acm.ListTagsForCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.ListTagsForCertificateOutput](ctx, natsConn, "acm.ListTagsForCertificate", input, natsTimeout, accountID)
@@ -80,7 +81,7 @@ func ListTagsForCertificate(ctx context.Context, natsConn *nats.Conn, accountID
 // AddTagsToCertificate — CertificateManager.AddTagsToCertificate.
 func AddTagsToCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.AddTagsToCertificateOutput, error) {
 	input := new(acm.AddTagsToCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.AddTagsToCertificateOutput](ctx, natsConn, "acm.AddTagsToCertificate", input, natsTimeout, accountID)
@@ -89,7 +90,7 @@ func AddTagsToCertificate(ctx context.Context, natsConn *nats.Conn, accountID st
 // RemoveTagsFromCertificate — CertificateManager.RemoveTagsFromCertificate.
 func RemoveTagsFromCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.RemoveTagsFromCertificateOutput, error) {
 	input := new(acm.RemoveTagsFromCertificateInput)
-	if err := unmarshalIfBody(body, input); err != nil {
+	if err := envelope.UnmarshalJSONBody(body, input); err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidParameter)
 	}
 	return natsmsg.NATSRequest[acm.RemoveTagsFromCertificateOutput](ctx, natsConn, "acm.RemoveTagsFromCertificate", input, natsTimeout, accountID)
@@ -2,7 +2,6 @@ package awsapi

 import (
 	"context"
-	"net/http/httptest"
 	"testing"

 	"github.com/mulgadc/spinifex/internal/testkit"
@@ -11,23 +10,6 @@ import (
 	"github.com/stretchr/testify/require"
 )

-func TestUnmarshalIfBody(t *testing.T) {
-	var out struct {
-		A string `json:"a"`
-	}
-	require.NoError(t, unmarshalIfBody(nil, &out))
-	require.NoError(t, unmarshalIfBody([]byte(`{"a":"x"}`), &out))
-	assert.Equal(t, "x", out.A)
-	require.Error(t, unmarshalIfBody([]byte("{bad"), &out))
-}
-
-func TestWriteJSONResponse(t *testing.T) {
-	w := httptest.NewRecorder()
-	WriteJSONResponse(w, map[string]string{"CertificateArn": "arn:aws:acm:x"})
-	assert.Equal(t, JSONContentType, w.Header().Get("Content-Type"))
-	assert.Contains(t, w.Body.String(), "CertificateArn")
-}
-
 // Invalid JSON bodies are rejected before any NATS round-trip.
 func TestOps_InvalidBodyRejected(t *testing.T) {
 	bad := []byte("{not json")
@@ -2,40 +2,3 @@
 // NATS-backed service contract. Generic gateway routing and authorization stay
 // at the ingress boundary.
 package awsapi
-
-import (
-	"encoding/json"
-	"log/slog"
-	"net/http"
-
-	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// JSONContentType is the AWS JSON 1.1 content type ACM clients expect.
-const JSONContentType = "application/x-amz-json-1.1"
-
-// WriteJSONResponse serialises obj as a 200 AWS JSON 1.1 response using the
-// SDK's own jsonutil marshaler, which encodes time.Time as epoch seconds
-// (encoding/json emits RFC3339 strings the SDK rejects).
-func WriteJSONResponse(w http.ResponseWriter, obj any) {
-	body, err := jsonutil.BuildJSON(obj)
-	if err != nil {
-		slog.Error("ACM: failed to marshal response JSON", "err", err)
-		http.Error(w, awserrors.ErrorInternalError, http.StatusInternalServerError)
-		return
-	}
-	w.Header().Set("Content-Type", JSONContentType)
-	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(body); err != nil {
-		slog.Error("ACM: failed to write response", "err", err)
-	}
-}
-
-// unmarshalIfBody decodes body into out only when non-empty.
-func unmarshalIfBody(body []byte, out any) error {
-	if len(body) == 0 {
-		return nil
-	}
-	return json.Unmarshal(body, out)
-}
@@ -10,6 +10,7 @@ import (

 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 	"github.com/nats-io/nats.go"
 )

@@ -124,6 +125,6 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return err
 	}

-	WriteJSONResponse(w, output)
+	envelope.WriteJSONResult(w, output)
 	return nil
 }
@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case isRevisionConflict(err):
+		case kvutil.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue
@@ -62,20 +62,6 @@ func (s *Service) checkTerminating(ctx context.Context, accountID string) error
 	return nil
 }

-// isRevisionConflict reports a lost CAS race to retry, including the bare
-// wrong-last-sequence APIError from Create's retry against a delete marker.
-// A private copy of domains/ec2/account's helper, which this package may not import.
-func isRevisionConflict(err error) bool {
-	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
-		return true
-	}
-	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
-		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
-			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
-	}
-	return false
-}
-
 // sealBucket returns this gateway's override bucket, or get-or-creates it
 // when quotas are disabled here. Called only after checkTerminating accepts,
 // so a refused seal never creates the bucket.
@@ -150,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case isRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"sync"
 	"testing"

@@ -433,16 +432,6 @@ func TestSealStoreErrorWritesNothing(t *testing.T) {
 	assert.Equal(t, before, lastOverrideSeq(t, js))
 }

-func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
-	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
-	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
-	assert.False(t, isRevisionConflict(nil))
-}
-
 // Clearing never writes when there is nothing active to clear: an absent key,
 // a tombstoned key, and a sealed key hidden behind a stale not-found Get all
 // leave the stream exactly where they found it.
@@ -8,6 +8,7 @@ import (

 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )

@@ -58,20 +59,6 @@ func (s *AccountSettingsServiceImpl) checkTerminating(ctx context.Context, accou
 	return nil
 }

-// isRevisionConflict reports a lost CAS race SealAccount should retry. Besides
-// the two sentinels, Create's retry against a delete marker returns a bare
-// *jetstream.APIError carrying a wrong-last-sequence code.
-func isRevisionConflict(err error) bool {
-	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
-		return true
-	}
-	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
-		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
-			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
-	}
-	return false
-}
-
 // SealAccount seals accountID's settings record so no later mutation can
 // recreate it. It never relies on a read for its claim: the only success is a
 // leader-accepted write, evidenced by the returned revision.
@@ -122,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case isRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"sync"
 	"testing"

@@ -397,16 +396,6 @@ func TestSealStoreErrorWritesNothing(t *testing.T) {
 	assert.Equal(t, before, lastSettingsSeq(t, svc))
 }

-func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
-	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
-	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
-	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
-	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
-	assert.False(t, isRevisionConflict(nil))
-}
-
 // An account IAM does not hold as TERMINATING refuses the seal before any
 // settings-KV write, whether the key is present or absent.
 func TestSealRefusedForActiveAccount(t *testing.T) {
@@ -0,0 +1,88 @@
+package accountinventory
+
+import (
+	"context"
+	"fmt"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// SpotRequestScanner reads one account's spot requests, failing on any record
+// it cannot read.
+type SpotRequestScanner interface {
+	ScanAccount(ctx context.Context, accountID string) (*ec2spot.AccountScan, error)
+}
+
+// VolumeScanner reads one account's volume documents, failing on any document
+// it cannot read.
+type VolumeScanner interface {
+	ListVolumesStrict(ctx context.Context, accountID string) ([]ec2metadata.Volume, error)
+}
+
+var (
+	_ SpotRequestScanner = (*ec2spot.SpotInstanceServiceImpl)(nil)
+	_ VolumeScanner      = (*ec2metadata.Store)(nil)
+)
+
+// SpotRequestItem is one request the account holds.
+type SpotRequestItem struct {
+	RequestID string
+	State     string
+	Active    bool
+}
+
+// VolumeItem is one volume document the account holds.
+type VolumeItem struct {
+	VolumeID            string
+	State               string
+	AttachedInstance    string
+	DeleteOnTermination bool
+}
+
+// WithCompute adds the spot-request and volume owner stores.
+func (i *Inventory) WithCompute(spot SpotRequestScanner, volumes VolumeScanner) *Inventory {
+	i.spot = spot
+	i.volumes = volumes
+	return i
+}
+
+// SpotRequestInventory returns every spot request the account holds. A scan
+// that cannot be completed is an error, never a shorter list.
+func (i *Inventory) SpotRequestInventory(ctx context.Context, accountID string) ([]SpotRequestItem, error) {
+	if i.spot == nil {
+		return nil, fmt.Errorf("spot request inventory is not configured")
+	}
+	scan, err := i.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("spot request inventory incomplete: %w", err)
+	}
+	items := make([]SpotRequestItem, 0, len(scan.Active)+len(scan.Terminal))
+	for _, r := range scan.Active {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State, Active: true})
+	}
+	for _, r := range scan.Terminal {
+		items = append(items, SpotRequestItem{RequestID: r.RequestID, State: r.State})
+	}
+	return items, nil
+}
+
+// VolumeInventory returns every volume document the account holds, including
+// those already marked deleting. An unreadable document is an error.
+func (i *Inventory) VolumeInventory(ctx context.Context, accountID string) ([]VolumeItem, error) {
+	if i.volumes == nil {
+		return nil, fmt.Errorf("volume inventory is not configured")
+	}
+	docs, err := i.volumes.ListVolumesStrict(ctx, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("volume inventory incomplete: %w", err)
+	}
+	items := make([]VolumeItem, 0, len(docs))
+	for _, v := range docs {
+		items = append(items, VolumeItem{
+			VolumeID: v.VolumeID, State: v.State, AttachedInstance: v.AttachedInstance,
+			DeleteOnTermination: v.DeleteOnTermination,
+		})
+	}
+	return items, nil
+}
@@ -0,0 +1,82 @@
+package accountinventory
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	ec2metadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2spot "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+type fakeSpot struct {
+	scan *ec2spot.AccountScan
+	err  error
+}
+
+func (f fakeSpot) ScanAccount(context.Context, string) (*ec2spot.AccountScan, error) {
+	return f.scan, f.err
+}
+
+type fakeVolumes struct {
+	docs []ec2metadata.Volume
+	err  error
+}
+
+func (f fakeVolumes) ListVolumesStrict(context.Context, string) ([]ec2metadata.Volume, error) {
+	return f.docs, f.err
+}
+
+func TestSpotRequestInventoryMarksActiveAndTerminal(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{scan: &ec2spot.AccountScan{
+		Active:   []ec2spot.AccountSpotRequest{{RequestID: "sir-1", State: "active"}},
+		Terminal: []ec2spot.AccountSpotRequest{{RequestID: "sir-2", State: "cancelled"}},
+	}}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []SpotRequestItem{
+		{RequestID: "sir-1", State: "active", Active: true},
+		{RequestID: "sir-2", State: "cancelled"},
+	}, items)
+}
+
+func TestSpotRequestInventoryScanFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(fakeSpot{err: errList}, nil)
+
+	items, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestVolumeInventoryReportsAttachmentAndDeletingDocuments(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{docs: []ec2metadata.Volume{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []VolumeItem{
+		{VolumeID: "vol-1", State: "in-use", AttachedInstance: "i-1", DeleteOnTermination: true},
+		{VolumeID: "vol-2", State: "deleting"},
+	}, items)
+}
+
+func TestVolumeInventoryListFailureIsAnErrorNotAShorterList(t *testing.T) {
+	inv := New(nil, nil, nil, nil).WithCompute(nil, fakeVolumes{err: errList})
+
+	items, err := inv.VolumeInventory(t.Context(), testAccount)
+	require.ErrorIs(t, err, errList)
+	assert.Nil(t, items)
+}
+
+func TestComputeInventoriesRefuseWhenNotConfigured(t *testing.T) {
+	inv := New(nil, nil, nil, nil)
+	_, err := inv.SpotRequestInventory(t.Context(), testAccount)
+	require.Error(t, err)
+	_, err = inv.VolumeInventory(t.Context(), testAccount)
+	require.Error(t, err)
+}
@@ -61,6 +61,8 @@ type Inventory struct {
 	groups    PlacementGroupScanner
 	templates LaunchTemplateScanner
 	tags      TagIndexScanner
+	spot      SpotRequestScanner
+	volumes   VolumeScanner
 	now       func() time.Time
 }

@@ -4,6 +4,7 @@
 package account

 import (
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"log/slog"

 	"github.com/aws/aws-sdk-go/aws"
@@ -14,12 +15,7 @@ import (
 // but default-vpc is static; defaultVPCID looks that one up, is called only
 // when it is asked for, and returns "" for an account with no default VPC.
 func DescribeAccountAttributes(input *ec2.DescribeAccountAttributesInput, defaultVPCID func() (string, error)) (*ec2.DescribeAccountAttributesOutput, error) {
-	requestedAttrs := make(map[string]bool)
-	for _, name := range input.AttributeNames {
-		if name != nil {
-			requestedAttrs[*name] = true
-		}
-	}
+	requestedAttrs := identifiers.Set(input.AttributeNames)

 	returnAll := len(requestedAttrs) == 0

@@ -151,7 +151,7 @@ var ec2Actions = map[string]ec2Action{
 		launchQuotaCheck := func() error {
 			return c.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount)))
 		}
-		reservation, err := ec2instanceapi.RunInstances(ctx, input, c.NATS, c.IAM, accountID, passRoleCheck, launchQuotaCheck, c.ExpectedNodes)
+		reservation, err := ec2instanceapi.RunInstances(ctx, input, c.NATS, c.clientTokenBucket, c.IAM, accountID, passRoleCheck, launchQuotaCheck, c.ExpectedNodes)
 		if err != nil {
 			return nil, err
 		}
@@ -436,7 +436,7 @@ var ec2Actions = map[string]ec2Action{
 		passRoleCheck := func(roleARN string) error {
 			return c.passRole(roleARN)
 		}
-		return ec2spotinstanceapi.RequestSpotInstances(ctx, input, c.NATS, c.IAM, accountID, c.AZ, passRoleCheck, c.Quota, c.ExpectedNodes)
+		return ec2spotinstanceapi.RequestSpotInstances(ctx, input, c.NATS, c.clientTokenBucket, c.IAM, accountID, c.AZ, passRoleCheck, c.Quota, c.ExpectedNodes)
 	}),
 	"DescribeSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.DescribeSpotInstanceRequestsInput, c *call, accountID string) (any, error) {
 		return ec2spotinstanceapi.DescribeSpotInstanceRequests(ctx, input, c.NATS, accountID)
@@ -4,10 +4,15 @@
 package idem

 import (
+	"context"
+	"errors"
+	"log/slog"
 	"reflect"
 	"time"

+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
+	"github.com/nats-io/nats.go/jetstream"
 )

 const (
@@ -54,3 +59,45 @@ func TokenAndParams(input any) (token, paramHash string, ok bool) {
 	clone.FieldByName(tokenField).Set(reflect.Zero(field.Type()))
 	return field.Elem().String(), idempotency.ParamHash(clone.Addr().Interface()), true
 }
+
+// Bucket returns the KV bucket that holds the token records. It is called only
+// when a request carries a token, so tokenless requests never bind it.
+type Bucket func(ctx context.Context) (jetstream.KeyValue, error)
+
+// Run runs work under the input's ClientToken, storing its result under
+// namespace. Requests without a token run unwrapped, which is what AWS does:
+// the token is how a client opts in.
+func Run[Out any](
+	ctx context.Context,
+	bucket Bucket,
+	namespace, accountID string,
+	input any,
+	work func() (Out, error),
+) (Out, error) {
+	var zero Out
+	// Hashed before work runs, so work that mutates its input cannot change what
+	// a retry of the same token hashes to.
+	token, paramHash, ok := TokenAndParams(input)
+	if !ok {
+		return work()
+	}
+
+	kv, kerr := bucket(ctx)
+	if kerr != nil {
+		// Running anyway would create the duplicate the token was sent to
+		// prevent, so this fails rather than degrading to no idempotency.
+		slog.ErrorContext(ctx, "client-token store unavailable", "namespace", namespace, "err", kerr)
+		return zero, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	store := idempotency.NewStore[Out](kv, namespace)
+	output, err := idempotency.Do(ctx, store, accountID, token, paramHash, work)
+	switch {
+	case errors.Is(err, idempotency.ErrParamMismatch):
+		return zero, errors.New(awserrors.ErrorIdempotentParameterMismatch)
+	case errors.Is(err, idempotency.ErrUnavailable):
+		slog.ErrorContext(ctx, "client-token replay unavailable", "namespace", namespace, "token", token, "err", err)
+		return zero, errors.New(awserrors.ErrorServerInternal)
+	}
+	return output, err
+}
@@ -1,11 +1,15 @@
 package idem_test

 import (
+	"context"
+	"errors"
 	"testing"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -79,3 +83,21 @@ func TestTokenAndParams_LeavesTheInputIntact(t *testing.T) {
 	require.NotNil(t, input.ClientToken)
 	assert.Equal(t, "tok", *input.ClientToken)
 }
+
+// A token the store cannot honour fails the request rather than running it
+// unprotected; a request without a token never touches the store.
+func TestRun_BucketFailure(t *testing.T) {
+	t.Parallel()
+	broken := func(context.Context) (jetstream.KeyValue, error) { return nil, errors.New("no jetstream") }
+	ran := false
+	work := func() (string, error) { ran = true; return "ok", nil }
+
+	_, err := ec2idemapi.Run(t.Context(), broken, "", "acct",
+		&ec2.RunInstancesInput{ClientToken: aws.String("tok")}, work)
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	assert.False(t, ran)
+
+	out, err := ec2idemapi.Run(t.Context(), broken, "", "acct", &ec2.RunInstancesInput{}, work)
+	require.NoError(t, err)
+	assert.Equal(t, "ok", out)
+}
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log/slog"

 	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -43,7 +42,9 @@ func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *
 			if !ok {
 				return nil, errors.New(awserrors.ErrorInternalError)
 			}
-			output, err := runEC2Idempotent(ctx, action, c, accountID, input, func() (Out, error) {
+			// Namespaced by action: one bucket holds every EC2 action's records, and
+			// without the prefix one action's payload could be decoded as another's Out.
+			output, err := ec2idemapi.Run(ctx, c.clientTokenBucket, action, accountID, input, func() (Out, error) {
 				return handler(ctx, input, c, accountID)
 			})
 			if err != nil {
@@ -53,43 +54,3 @@ func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *
 		},
 	}
 }
-
-// runEC2Idempotent runs work under the input's ClientToken. Requests without one
-// run unwrapped, which is what AWS does: the token is how a client opts in.
-func runEC2Idempotent[In, Out any](
-	ctx context.Context,
-	action string,
-	c *call,
-	accountID string,
-	input *In,
-	work func() (Out, error),
-) (Out, error) {
-	var zero Out
-	// Hashed before the handler runs, so a handler that mutates its input cannot
-	// change what a retry of the same token hashes to.
-	token, paramHash, ok := ec2idemapi.TokenAndParams(input)
-	if !ok {
-		return work()
-	}
-
-	kv, kerr := c.clientTokenBucket(ctx)
-	if kerr != nil {
-		// Running anyway would create the duplicate the token was sent to
-		// prevent, so this fails rather than degrading to no idempotency.
-		slog.ErrorContext(ctx, "client-token store unavailable", "action", action, "err", kerr)
-		return zero, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// Namespaced by action: one bucket holds every EC2 action's records, and
-	// without the prefix one action's payload could be decoded as another's Out.
-	store := idempotency.NewStore[Out](kv, action)
-	output, err := idempotency.Do(ctx, store, accountID, token, paramHash, work)
-	switch {
-	case errors.Is(err, idempotency.ErrParamMismatch):
-		return zero, errors.New(awserrors.ErrorIdempotentParameterMismatch)
-	case errors.Is(err, idempotency.ErrUnavailable):
-		slog.ErrorContext(ctx, "client-token replay unavailable", "action", action, "token", token, "err", err)
-		return zero, errors.New(awserrors.ErrorServerInternal)
-	}
-	return output, err
-}
@@ -0,0 +1,151 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// accountInstanceRecordsSubject serves every non-terminated instance record an
+// account owns from the shared store, whichever node holds the instance.
+const accountInstanceRecordsSubject = "ec2.DescribeAccountInstanceRecords"
+
+// DescribeAccountInstancesStrict lists every non-terminated instance the
+// account owns, or returns a retryable ServiceUnavailable. Existence comes from
+// the shared record store, so an instance no answering node holds is still
+// listed with its recorded state; node answers are joined on for live state.
+// Anything that could make the list shorter than the truth is an error: a
+// configured node that did not answer, a frame from a node outside nodeIDs, a
+// record the owner could not project, or a store that did not answer.
+func DescribeAccountInstancesStrict(ctx context.Context, natsConn *nats.Conn, nodeIDs []string, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) {
+	if len(nodeIDs) == 0 {
+		slog.ErrorContext(ctx, "DescribeInstances: strict listing refused, configured node set unknown")
+		return nil, incompleteListingError()
+	}
+	cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout}
+	for _, opt := range opts {
+		opt(&cfg)
+	}
+
+	jsonData, err := json.Marshal(&ec2.DescribeInstancesInput{})
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "marshal describe input: %v", err)
+	}
+
+	records, ok := queryInstanceBucket(ctx, natsConn, accountInstanceRecordsSubject, jsonData, accountID)
+	if !ok {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, instance record store did not answer")
+		return nil, incompleteListingError()
+	}
+
+	// Collect to the deadline with no responder grace, so a reply from a node
+	// outside the set that lands after the configured nodes is still seen.
+	frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, natsmsg.GatherOpts{
+		Timeout:            cfg.fanoutTimeout,
+		AccountID:          accountID,
+		ExpectedResponders: len(nodeIDs),
+		Mode:               natsmsg.CollectUntilDeadline,
+	})
+	if err != nil {
+		return nil, err
+	}
+	live, complete := judgeIdentityCompleteness(ctx, frames, sum, nodeIDs)
+	if stray := unconfiguredNodes(frames, sum, nodeIDs); len(stray) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, frame from a node outside the configured set",
+			"unconfigured_nodes", stray)
+		return nil, incompleteListingError()
+	}
+	if !complete {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, fan-out incomplete", "configured_nodes", len(nodeIDs))
+		return nil, incompleteListingError()
+	}
+
+	joined, nodeOnly, malformed := joinLiveOntoRecords(records, live)
+	if malformed {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, a reservation or instance has no usable identity")
+		return nil, incompleteListingError()
+	}
+	if len(nodeOnly) > 0 {
+		slog.WarnContext(ctx, "DescribeInstances: strict listing refused, node reports instances with no record",
+			"instances", nodeOnly)
+		return nil, incompleteListingError()
+	}
+	return &ec2.DescribeInstancesOutput{Reservations: joined}, nil
+}
+
+// unconfiguredNodes returns the node IDs that answered, with a success or an
+// error, but are not in nodeIDs.
+func unconfiguredNodes(frames []natsmsg.Frame, sum natsmsg.Summary, nodeIDs []string) []string {
+	var stray []string
+	note := func(id string) {
+		if id != "" && !slices.Contains(nodeIDs, id) && !slices.Contains(stray, id) {
+			stray = append(stray, id)
+		}
+	}
+	for _, frame := range frames {
+		note(frame.NodeID)
+	}
+	for id := range sum.Responders {
+		note(id)
+	}
+	slices.Sort(stray)
+	return stray
+}
+
+// joinLiveOntoRecords starts from the recorded instances and replaces each with
+// the answering node's view of it. Instances only a node reports are returned
+// by ID: a lost record write can cause one, and it must not be dropped. A nil
+// reservation, instance or instance ID is reported as malformed, never skipped.
+func joinLiveOntoRecords(records, live []*ec2.Reservation) ([]*ec2.Reservation, []string, bool) {
+	type entry struct {
+		reservation *ec2.Reservation
+		instance    *ec2.Instance
+	}
+	byID := map[string]*entry{}
+	var order []string
+	malformed := false
+	add := func(reservations []*ec2.Reservation) {
+		for _, res := range reservations {
+			if res == nil {
+				malformed = true
+				continue
+			}
+			for _, inst := range res.Instances {
+				if inst == nil || aws.StringValue(inst.InstanceId) == "" {
+					malformed = true
+					continue
+				}
+				id := aws.StringValue(inst.InstanceId)
+				if existing, ok := byID[id]; ok {
+					existing.instance = inst
+					continue
+				}
+				byID[id] = &entry{reservation: res, instance: inst}
+				order = append(order, id)
+			}
+		}
+	}
+	add(records)
+	recorded := len(order)
+	add(live)
+	nodeOnly := slices.Clone(order[recorded:])
+
+	out := make([]*ec2.Reservation, 0, len(order))
+	for _, id := range order {
+		e := byID[id]
+		out = append(out, &ec2.Reservation{
+			ReservationId: e.reservation.ReservationId,
+			OwnerId:       e.reservation.OwnerId,
+			Groups:        e.reservation.Groups,
+			Instances:     []*ec2.Instance{e.instance},
+		})
+	}
+	return mergeReservationsByID(out), nodeOnly, malformed
+}
@@ -0,0 +1,229 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func strictInstance(id, state string) *ec2.Instance {
+	return &ec2.Instance{InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)}}
+}
+
+func strictReservations(instances ...*ec2.Instance) *ec2.DescribeInstancesOutput {
+	out := &ec2.DescribeInstancesOutput{}
+	for _, inst := range instances {
+		out.Reservations = append(out.Reservations, &ec2.Reservation{
+			ReservationId: aws.String("r-" + aws.StringValue(inst.InstanceId)),
+			OwnerId:       aws.String("123456789012"),
+			Instances:     []*ec2.Instance{inst},
+		})
+	}
+	return out
+}
+
+func subscribeAccountRecords(t *testing.T, nc *nats.Conn, data []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(accountInstanceRecordsSubject, func(msg *nats.Msg) { _ = msg.Respond(data) })
+	require.NoError(t, err)
+}
+
+const strictAccount = "123456789012"
+
+func strictList(nc *nats.Conn, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
+	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(2*time.Second))
+}
+
+// Existence comes from the record store, so an instance no node holds is still
+// listed, with the state the record carries.
+func TestDescribeAccountInstancesStrict_UnheldRecordIsReturnedWithItsRecordedState(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-unheld", "stopped"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	inst := out.Reservations[0].Instances[0]
+	assert.Equal(t, "i-unheld", aws.StringValue(inst.InstanceId))
+	assert.Equal(t, "stopped", aws.StringValue(inst.State.Name))
+}
+
+// A node's answer replaces the recorded view of the instance it holds.
+func TestDescribeAccountInstancesStrict_NodeAnswersAreJoinedOntoRecords(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "pending"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.NoError(t, err)
+	states := map[string]string{}
+	for _, res := range out.Reservations {
+		for _, inst := range res.Instances {
+			states[aws.StringValue(inst.InstanceId)] = aws.StringValue(inst.State.Name)
+		}
+	}
+	assert.Equal(t, map[string]string{"i-held": "running"}, states)
+}
+
+// An instance a node reports with no record can come from a lost record write.
+// Returning it or dropping it would both answer for an unproven account.
+func TestDescribeAccountInstancesStrict_NodeOnlyInstanceFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-held", "running"))))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", mustMarshalDescribeOutput(t,
+		strictReservations(strictInstance("i-held", "running"), strictInstance("i-node-only", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	require.Error(t, err)
+	assert.Nil(t, out)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
+}
+
+// A frame from a node outside the configured set means the expected set is
+// wrong, and a listing built on it could omit that node's instances.
+func TestDescribeAccountInstancesStrict_FrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running"))))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A stray node that answers with an error is still a node outside the set, and
+// its instances are unaccounted for.
+func TestDescribeAccountInstancesStrict_ErrorFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// The configured nodes answer at once, so a reply that lands later but inside
+// the fan-out deadline must still be seen.
+func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	late := mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running")))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(600*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = late
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-stray")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+// A record with no instance ID cannot be listed or counted, so it must fail
+// the read rather than vanish from it.
+func TestDescribeAccountInstancesStrict_RecordWithoutInstanceIDFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations(&ec2.Instance{State: &ec2.InstanceState{Name: aws.String("stopped")}})))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_ConfiguredNodeSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1", "node-2")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreErrorFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_RecordStoreSilentFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnknownNodeSetFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc)
+
+	assertIncompleteListing(t, out, err)
+}
+
+func TestDescribeAccountInstancesStrict_UnattributedFrameFails(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeAccountRecords(t, nc, mustMarshalDescribeOutput(t, strictReservations()))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "", []byte(`{}`))
+	nc.Flush()
+
+	out, err := strictList(nc, "node-1")
+
+	assertIncompleteListing(t, out, err)
+}
@@ -80,5 +80,5 @@ func DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceA
 	// terraform retries cleanly.
 	slog.WarnContext(ctx, "DescribeInstanceAttribute: instance absent or no responses",
 		"instance_id", *input.InstanceId, "received", sum.Received, "expected", expectedNodes)
-	return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", *input.InstanceId)
 }
@@ -184,7 +184,7 @@ func TestDescribeInstanceAttribute_AllNodesNotFound(t *testing.T) {

 	_, err = DescribeInstanceAttribute(context.Background(), input, nc, 2, "123456789012")
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-missing' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 // TestDescribeInstanceAttribute_ClientErrorPropagates ensures that a
@@ -254,7 +254,7 @@ func TestDescribeInstanceAttribute_NoResponders(t *testing.T) {
 	duration := time.Since(start)

 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-nowhere' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 	// Must not block the caller past the 3 s deadline.
 	assert.Less(t, duration, 4*time.Second)
 }
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
 	"sort"
@@ -74,12 +75,7 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
 		}
 	}

-	requestedTypes := make(map[string]bool)
-	for _, it := range input.InstanceTypes {
-		if it != nil {
-			requestedTypes[*it] = true
-		}
-	}
+	requestedTypes := identifiers.Set(input.InstanceTypes)

 	var finalInstanceTypes []*ec2.InstanceTypeInfo
 	if showCapacity {
@@ -126,7 +126,7 @@ func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesI
 		missing := missingInstanceIDs(reservations, input.InstanceIds)
 		if len(missing) > 0 {
 			if complete {
-				return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+				return nil, awserrors.IDsNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", missing)
 			}
 			// An incomplete sweep cannot prove absence, so a missing id is
 			// unresolved, not gone. NotFound would be a false negative and an
@@ -0,0 +1,63 @@
+package instance
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Pins today: the completeness judge merges the reservations of a frame from a
+// node outside the configured set and still reports the listing complete. The
+// stray node is never named, so a silent unconfigured node cannot be told from
+// one that does not exist.
+func TestCurrentBehaviour_JudgeIdentityCompleteness_FrameFromUnconfiguredNodeIsMerged(t *testing.T) {
+	t.Parallel()
+	stray := mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{
+		Reservations: []*ec2.Reservation{{ReservationId: aws.String("r-stray"), Instances: []*ec2.Instance{{InstanceId: aws.String("i-on-stray")}}}},
+	})
+	frames := []natsmsg.Frame{
+		{NodeID: "node-1", Data: []byte(`{}`)},
+		{NodeID: "node-stray", Data: stray},
+	}
+
+	reservations, complete := judgeIdentityCompleteness(context.Background(), frames, natsmsg.Summary{}, []string{"node-1"})
+
+	assert.True(t, complete, "CURRENT: a node outside nodeIDs does not make the listing incomplete")
+	require.Len(t, reservations, 1)
+	assert.Equal(t, "i-on-stray", *reservations[0].Instances[0].InstanceId,
+		"CURRENT: the unconfigured node's instances are merged into the result")
+}
+
+// Pins today: the fan-out counts every identified frame toward the expected
+// responders, so a stray frame can end collection before a configured node has
+// answered. The listing then refuses, which fails closed, but for a node that
+// was only slow.
+func TestCurrentBehaviour_DescribeAccountInstancesComplete_StrayFrameConsumesAResponderSlot(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	subscribeEmptyInstanceBuckets(t, nc)
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
+	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-stray", []byte(`{}`))
+	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
+		time.AfterFunc(500*time.Millisecond, func() {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = []byte(`{}`)
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-2")
+			_ = msg.RespondMsg(reply)
+		})
+	})
+	require.NoError(t, err)
+	nc.Flush()
+
+	output, err := DescribeAccountInstancesComplete(context.Background(), nc, []string{"node-1", "node-2"}, "123456789012",
+		WithFanoutTimeout(3*time.Second))
+
+	assertIncompleteListing(t, output, err)
+}
@@ -448,11 +448,13 @@ func TestDescribeInstancesChecked_ExplicitIDNotFound_CompleteSweep(t *testing.T)
 	require.NoError(t, err)
 	nc.Flush()

-	input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}}
+	input := &ec2.DescribeInstancesInput{InstanceIds: aws.StringSlice([]string{"i-doesnotexist0000000", "i-real", "i-doesnotexist0000001"})}
 	_, err = DescribeInstancesChecked(context.Background(), input, nc, 1, nil, "123456789012")

-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "err = %v", err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, code)
+	assert.Equal(t, "The instance IDs 'i-doesnotexist0000000, i-doesnotexist0000001' do not exist", message)
 }

 // TestDescribeInstances_ExplicitIDNotFound_StaysSilent guards the internal
@@ -624,8 +626,7 @@ func TestDescribeInstancesChecked_Identity_AllNodesAnswer_NotFoundAsserted(t *te
 	_, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012",
 		WithFanoutTimeout(2*time.Second))

-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound), "err = %v", err)
 }

 // A node answering with a decodable error envelope never becomes a valid
@@ -684,7 +685,7 @@ func TestDescribeInstancesChecked_Identity_NilReservationsIsValidResponder(t *te
 		WithFanoutTimeout(2*time.Second))

 	require.Error(t, err, "a nil-Reservations node is a complete, valid responder, so the sweep is complete and NotFound must fire")
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound), "err = %v", err)
 }

 // An empty nodeIDs falls back to the legacy expectedNodes path, which the
@@ -47,7 +47,7 @@ func GetConsoleOutput(ctx context.Context, input *ec2.GetConsoleOutputInput, nat
 	if err != nil {
 		// No daemon subscription or timeout: stopped/terminated/non-existent instances all surface as NotFound.
 		if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) {
-			return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", *input.InstanceId)
 		}
 		return nil, fmt.Errorf("failed to get console output: %w", err)
 	}
@@ -52,7 +52,7 @@ func GetPasswordData(ctx context.Context, input *ec2.GetPasswordDataInput, natsC
 	if err != nil {
 		// No daemon subscription or timeout: stopped/terminated/non-existent instances all surface as NotFound.
 		if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) {
-			return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", *input.InstanceId)
 		}
 		return nil, fmt.Errorf("failed to get password data: %w", err)
 	}
@@ -120,7 +120,7 @@ func TestGetPasswordData_NoResponders(t *testing.T) {

 	require.Error(t, err)
 	assert.Nil(t, output)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-nosubscriber' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestGetPasswordData_Timeout(t *testing.T) {
@@ -140,7 +140,7 @@ func TestGetPasswordData_Timeout(t *testing.T) {

 	require.Error(t, err)
 	assert.Nil(t, output)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-slow' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestGetPasswordData_DaemonError(t *testing.T) {
@@ -82,7 +82,7 @@ func AssociateIamInstanceProfile(ctx context.Context, input *ec2.AssociateIamIns
 	assoc, err := natsmsg.NATSRequest[ec2.IamInstanceProfileAssociation](ctx, natsConn, subject, command, fanOutTimeout, accountID)
 	if err != nil {
 		if errors.Is(err, nats.ErrNoResponders) {
-			return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", *input.InstanceId)
 		}
 		return nil, err
 	}
@@ -293,7 +293,7 @@ func TestAssociateIamInstanceProfile_NoResponders(t *testing.T) {
 	// a raw NATS timeout.
 	_, err := AssociateIamInstanceProfile(context.Background(), in, nc, svc, testGwAccountID, nil)
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-no-daemon' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestAssociateIamInstanceProfile_Success(t *testing.T) {
@@ -87,7 +87,7 @@ func RebootInstances(ctx context.Context, input *ec2.RebootInstancesInput, natsC
 			case state != bucketStateAbsent:
 				return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
 			case queryOK:
-				return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+				return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 			default:
 				return nil, errors.New(awserrors.ErrorServerInternal)
 			}
@@ -88,7 +88,7 @@ func TestRebootInstances_NeverExistedIsNotFound(t *testing.T) {

 	_, err := RebootInstances(context.Background(), input, nc, "123456789012")
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-d2d5fe795b6b57882' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestRebootInstances_BucketsUnreachableIsServerError(t *testing.T) {
@@ -8,7 +8,6 @@ import (
 	"errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"log/slog"
 	"math"
 	"slices"
 	"strings"
@@ -16,6 +15,7 @@ import (

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
 	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
@@ -126,12 +126,13 @@ func hasPinnedPrivateIP(input *ec2.RunInstancesInput) bool {

 // RunInstances validates input, resolves any IAM instance profile (normalising
 // to ARN for daemons), enforces iam:PassRole, then dispatches via NATS.
-// When ClientToken is set, wraps the launch in idempotency via the KV store.
+// When ClientToken is set, wraps the launch in idempotency via tokenBucket,
+// which may be nil only when no ClientToken is supplied.
 // iamSvc may be nil only when no IamInstanceProfile is supplied.
 // passRoleCheck may be nil to skip PassRole enforcement.
 // launchQuotaCheck may be nil to skip quota enforcement; when present it runs
 // after validation and PassRole authorization, but before any launch dispatch.
-func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
+func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, tokenBucket ec2idemapi.Bucket, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
 	// Expand a referenced launch template before validation so validation, quota,
 	// and downstream routing all see the effective (post-merge) parameters.
 	if err = ExpandLaunchTemplate(ctx, natsConn, input, accountID); err != nil {
@@ -142,21 +143,10 @@ func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *n
 		return reservation, err
 	}

-	token := aws.StringValue(input.ClientToken)
-	if token == "" {
-		return runInstancesInner(ctx, input, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
-	}
-
-	// ClientToken set: dedup concurrent/retried launches; store failure is fatal
-	// to avoid a double launch on retry.
-	store, serr := getClientTokenStore(ctx, natsConn)
-	if serr != nil {
-		slog.ErrorContext(ctx, "RunInstances: client-token store unavailable", "err", serr)
-		return reservation, errors.New(awserrors.ErrorServerInternal)
-	}
-	// Hash before any mutation so the same token+params always matches.
-	paramHash := clientTokenParamHash(input)
-	return runInstancesWithClientToken(ctx, store, accountID, token, paramHash, func() (ec2.Reservation, error) {
+	// Unnamespaced: every other EC2 action namespaces its keys by action, but
+	// RunInstances predates that, and re-keying live records would make a retry
+	// re-launch instead of replay. Hashed here, after template expansion.
+	return ec2idemapi.Run(ctx, tokenBucket, "", accountID, input, func() (ec2.Reservation, error) {
 		return runInstancesInner(ctx, input, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
 	})
 }
@@ -347,7 +347,7 @@ func TestParseRunInstances(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 			// For validation tests, we can pass nil conn since validation happens before NATS call
-			response, err := RunInstances(context.Background(), test.input, nil, nil, "123456789012", nil, nil, 1)
+			response, err := RunInstances(context.Background(), test.input, nil, nil, nil, "123456789012", nil, nil, 1)

 			// Use assert to check if the error is as expected
 			assert.Equal(t, test.want, err)
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
 	"time"
@@ -102,7 +103,7 @@ func startLiveInstance(ctx context.Context, natsConn *nats.Conn, instanceID, acc

 	if responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data); parseErr != nil {
 		slog.ErrorContext(ctx, "StartInstances: owner returned error", "instance_id", instanceID, "code", *responseError.Code)
-		return nil, false, startRefusalError(responseError)
+		return nil, false, startRefusalError(responseError, msg.Header)
 	}

 	slog.InfoContext(ctx, "StartInstances: restarted via owner node", "instance_id", instanceID, "response", string(msg.Data))
@@ -135,18 +136,25 @@ func startStoppedInstance(ctx context.Context, natsConn *nats.Conn, instanceID,

 	if responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data); parseErr != nil {
 		slog.ErrorContext(ctx, "StartInstances: Daemon returned error", "instance_id", instanceID, "code", *responseError.Code)
-		return nil, startRefusalError(responseError)
+		return nil, startRefusalError(responseError, msg.Header)
 	}

 	slog.InfoContext(ctx, "StartInstances: Command sent successfully", "instance_id", instanceID, "response", string(msg.Data))
 	return newStateChange(instanceID, 0, "pending", 80, "stopped"), nil
 }

-// startRefusalError rebuilds the daemon's error, keeping the message it sent.
-// Reconstructing from the code alone renders the generic ErrorLookup text
-// instead, which replaces the name of the node holding the volume with
-// boilerplate about attaching volumes -- the one fact the operator needed.
-func startRefusalError(responseError ec2.ResponseError) error {
+// startRefusalError rebuilds the daemon's error, keeping the message it sent,
+// which names the node holding the volume, and wrapping ErrVolumeHeldElsewhere
+// when the reply's header says that is why the start was refused.
+func startRefusalError(responseError ec2.ResponseError, header nats.Header) error {
+	err := startRefusalCodeError(responseError)
+	if header.Get(natsmsg.VolumeHeldElsewhereHeader) == "true" {
+		return fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere, err)
+	}
+	return err
+}
+
+func startRefusalCodeError(responseError ec2.ResponseError) error {
 	var code, message string
 	if responseError.Code != nil {
 		code = *responseError.Code
@@ -3,11 +3,14 @@ package instance
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"testing"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -161,3 +164,33 @@ func TestStartInstances_MixedSuccessAndFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, "InvalidInstanceID.NotFound", err.Error())
 }
+
+// The AWS body says only IncorrectState, so the header is the one thing that
+// tells a held volume apart from any other refusal in that code.
+func TestStartInstances_VolumeHeldElsewhereHeader(t *testing.T) {
+	t.Parallel()
+	for name, held := range map[string]bool{"with header": true, "without header": false} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			_, nc := startTestNATSServer(t)
+			const id = "i-0abc0000000000002"
+			_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(id), func(msg *nats.Msg) {
+				reply := nats.NewMsg(msg.Reply)
+				reply.Data = awserrors.GenerateErrorPayloadWithMessage(awserrors.ErrorIncorrectState, "volume vol-1 is published to node-a")
+				if held {
+					reply.Header.Set(natsmsg.VolumeHeldElsewhereHeader, "true")
+				}
+				_ = msg.RespondMsg(reply)
+			})
+			require.NoError(t, err)
+
+			_, err = StartInstances(t.Context(), &ec2.StartInstancesInput{InstanceIds: []*string{aws.String(id)}}, nc, "123456789012")
+			require.Error(t, err)
+			assert.Equal(t, held, errors.Is(err, ec2v1.ErrVolumeHeldElsewhere))
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorIncorrectState, code)
+			assert.Equal(t, "volume vol-1 is published to node-a", message)
+		})
+	}
+}
@@ -73,7 +73,7 @@ func StopInstances(ctx context.Context, input *ec2.StopInstancesInput, natsConn
 			case state == bucketStateTerminated:
 				return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
 			case queryOK:
-				return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+				return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 			default:
 				return nil, errors.New(awserrors.ErrorServerInternal)
 			}
@@ -124,7 +124,7 @@ func TestStopInstances_NeverExistedIsNotFound(t *testing.T) {
 	}, nc, "123456789012")

 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-d2d5fe795b6b57882' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 	assert.Nil(t, output)
 }

@@ -220,6 +220,6 @@ func TestStopInstances_MixedBatchWithUnknownIDFails(t *testing.T) {
 	}, nc, "123456789012")

 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-587b21c4147f9d9d8' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 	assert.Nil(t, output)
 }
@@ -115,7 +115,7 @@ func TerminateInstances(ctx context.Context, input *ec2.TerminateInstancesInput,
 				}
 				if queryOK && state == bucketStateAbsent && daemonReportedAbsent {
 					slog.InfoContext(ctx, "TerminateInstances: Instance does not exist", "instance_id", instanceID)
-					return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+					return nil, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 				}
 				slog.ErrorContext(ctx, "TerminateInstances: cannot confirm instance state",
 					"instance_id", instanceID, "bucket_state", state, "query_ok", queryOK, "terminate_err", terminateErr)
@@ -291,7 +291,7 @@ func TestTerminateInstances_NeverExistedIsNotFound(t *testing.T) {
 	}, nc, "123456789012")

 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-d419ad3c922169e6e' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 	assert.Nil(t, output)
 }

@@ -380,6 +380,6 @@ func TestTerminateInstances_MixedBatchWithUnknownIDFails(t *testing.T) {
 	}, nc, "123456789012")

 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-0ee1ff203142536a7' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 	assert.Nil(t, output)
 }
@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// ClientTokenStore is the RunInstances idempotency store: the first caller owns
-// the launch, duplicates replay its reservation or poll for it.
-type ClientTokenStore = idempotency.Store[ec2.Reservation]
-
-var (
-	ctStore        *ClientTokenStore
-	ctOnce         sync.Once
-	errCTStoreInit error
-)
-
-// getClientTokenStore lazily initialises the process-wide client-token store via
-// sync.Once. Process-wide because RunInstances is a free function with no
-// instance to hang it on.
-func getClientTokenStore(ctx context.Context, nc *nats.Conn) (*ClientTokenStore, error) {
-	ctOnce.Do(func() {
-		js, err := jetstream.New(nc)
-		if err != nil {
-			errCTStoreInit = fmt.Errorf("clienttoken jetstream: %w", err)
-			return
-		}
-		// The bind happens once per process, so it must not inherit the first
-		// caller's cancellation: a client that disconnects mid-open would poison
-		// the store for every later launch. Deadline-free, so the open falls back
-		// to the JetStream API's own timeout.
-		ctStore, errCTStoreInit = newClientTokenStore(context.WithoutCancel(ctx), js)
-	})
-	return ctStore, errCTStoreInit
-}
-
-// newClientTokenStore takes the bucket unnamespaced. Every other EC2 action
-// namespaces its keys by action, but RunInstances predates that: re-keying live
-// records would make a retry re-launch instead of replay.
-func newClientTokenStore(ctx context.Context, js jetstream.JetStream) (*ClientTokenStore, error) {
-	return idempotency.OpenStore[ec2.Reservation](ctx, js, ec2idemapi.KVBucket, ec2idemapi.TTL)
-}
-
-// clientTokenParamHash hashes the request excluding ClientToken, so the same
-// params always produce the same hash. Must run before any input mutation.
-func clientTokenParamHash(input *ec2.RunInstancesInput) string {
-	clone := *input
-	clone.ClientToken = nil
-	return idempotency.ParamHash(&clone)
-}
-
-// runInstancesWithClientToken wraps a launch in ClientToken idempotency and maps
-// the token errors onto the AWS ones RunInstances returns. Extracted for
-// unit-testability.
-func runInstancesWithClientToken(
-	ctx context.Context,
-	store *ClientTokenStore,
-	accountID, token, paramHash string,
-	launch func() (ec2.Reservation, error),
-) (ec2.Reservation, error) {
-	res, err := idempotency.Do(ctx, store, accountID, token, paramHash, launch)
-	switch {
-	case errors.Is(err, idempotency.ErrParamMismatch):
-		return ec2.Reservation{}, errors.New(awserrors.ErrorIdempotentParameterMismatch)
-	case errors.Is(err, idempotency.ErrUnavailable):
-		slog.ErrorContext(ctx, "RunInstances: client-token claim failed", "token", token, "err", err)
-		return ec2.Reservation{}, errors.New(awserrors.ErrorServerInternal)
-	}
-	return res, err
-}
@@ -1,155 +1,55 @@
 package instance

 import (
-	"errors"
+	"context"
 	"testing"
+	"time"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

 const ctTestAccount = "111122223333"

-func newTestClientTokenStore(t *testing.T) *ClientTokenStore {
-	t.Helper()
+// RunInstances records predate per-action namespacing: they live at the bare
+// account/token key, hashed over the request minus its token. A record written
+// that way must still replay, or a retry across an upgrade would re-launch.
+func TestRunInstances_ClientTokenReplaysUnnamespacedRecord(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
-	store, err := newClientTokenStore(t.Context(), testutil.NewJetStream(t, nc))
+	kv, err := idempotency.OpenBucket(t.Context(), testutil.NewJetStream(t, nc), "ec2-tokens", time.Minute)
 	require.NoError(t, err)
-	return store
-}
+	bucket := func(context.Context) (jetstream.KeyValue, error) { return kv, nil }

-// The claim/replay/abort mechanics live in spinifex/foundation/lifecycle/idempotency and are tested
-// there. What is EC2's own is the param hash, the launch orchestration around
-// the store, and the AWS error a mismatch maps onto.
+	const tok = "rt-1"
+	input := defaults
+	input.ClientToken = aws.String(tok)
+	unTokened := input
+	unTokened.ClientToken = nil
+	hash := idempotency.ParamHash(&unTokened)

-// clientTokenParamHash ignores ClientToken (same params, different token →
-// same hash) but reflects a real parameter change.
-func TestClientTokenParamHash_IgnoresTokenReflectsParams(t *testing.T) {
-	t.Parallel()
-	base := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-123"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		ClientToken:  aws.String("tok-A"),
-	}
-	sameParamsDiffToken := *base
-	sameParamsDiffToken.ClientToken = aws.String("tok-B")
-	diffParams := *base
-	diffParams.InstanceType = aws.String("t3.large")
-
-	assert.Equal(t, clientTokenParamHash(base), clientTokenParamHash(&sameParamsDiffToken),
-		"token must not affect the param hash")
-	assert.NotEqual(t, clientTokenParamHash(base), clientTokenParamHash(&diffParams),
-		"a real param change must change the hash")
-}
-
-// The hash is taken from a copy, so hashing must not clear the caller's token.
-func TestClientTokenParamHash_LeavesTheInputAlone(t *testing.T) {
-	t.Parallel()
-	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-123"), ClientToken: aws.String("tok-A")}
-
-	clientTokenParamHash(input)
-
-	assert.Equal(t, "tok-A", aws.StringValue(input.ClientToken),
-		"hashing must not strip the token the launch still needs")
-}
-
-// A completed token replays its reservation and must NOT invoke the launcher.
-func TestRunInstancesWithClientToken_ReplaySkipsLaunch(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-1", "h"
+	store := idempotency.NewStore[ec2.Reservation](kv, "")
 	_, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash)
 	require.NoError(t, err)
 	require.True(t, owned)
 	require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, hash, ec2.Reservation{ReservationId: aws.String("r-x")}))
+	_, err = kv.Get(t.Context(), idempotency.Key(ctTestAccount, tok))
+	require.NoError(t, err, "record must sit at the unprefixed key")

-	launched := false
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		launched = true
-		return ec2.Reservation{}, nil
-	})
+	// A nil NATS connection: replay must not reach the launch path.
+	replay := input
+	res, err := RunInstances(t.Context(), &replay, nil, bucket, nil, ctTestAccount, nil, nil, 1)
 	require.NoError(t, err)
-	assert.False(t, launched, "replay must not launch")
 	assert.Equal(t, "r-x", aws.StringValue(res.ReservationId))
-}

-// The owner launches once and finalizes; a duplicate replays without launching.
-func TestRunInstancesWithClientToken_OwnerLaunchesOnceThenReplay(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-2", "h"
-	launches := 0
-	launch := func() (ec2.Reservation, error) {
-		launches++
-		return ec2.Reservation{ReservationId: aws.String("r-own")}, nil
-	}
-
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch)
-	require.NoError(t, err)
-	assert.Equal(t, "r-own", aws.StringValue(res.ReservationId))
-
-	res2, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch)
-	require.NoError(t, err)
-	assert.Equal(t, "r-own", aws.StringValue(res2.ReservationId))
-	assert.Equal(t, 1, launches, "duplicate must replay, not relaunch")
-}
-
-// A launch failure aborts the token so a retry re-launches.
-func TestRunInstancesWithClientToken_LaunchFailureAborts(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok, hash = "rt-3", "h"
-
-	_, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		return ec2.Reservation{}, errors.New("no capacity")
-	})
-	require.Error(t, err)
-
-	relaunched := false
-	res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) {
-		relaunched = true
-		return ec2.Reservation{ReservationId: aws.String("r-retry")}, nil
-	})
-	require.NoError(t, err)
-	assert.True(t, relaunched, "after abort the retry must launch")
-	assert.Equal(t, "r-retry", aws.StringValue(res.ReservationId))
-}
-
-// Token reuse with different params maps to the AWS IdempotentParameterMismatch
-// error code and never launches.
-func TestRunInstancesWithClientToken_ParamMismatchMapsAWSError(t *testing.T) {
-	t.Parallel()
-	store := newTestClientTokenStore(t)
-	const tok = "rt-4"
-	_, owned, err := store.Claim(t.Context(), ctTestAccount, tok, "hA")
-	require.NoError(t, err)
-	require.True(t, owned)
-	require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, "hA", ec2.Reservation{ReservationId: aws.String("r")}))
-
-	launched := false
-	_, err = runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, "hB", func() (ec2.Reservation, error) {
-		launched = true
-		return ec2.Reservation{}, nil
-	})
+	changed := input
+	changed.InstanceType = aws.String("t3.large")
+	_, err = RunInstances(t.Context(), &changed, nil, bucket, nil, ctTestAccount, nil, nil, 1)
 	require.EqualError(t, err, awserrors.ErrorIdempotentParameterMismatch)
-	assert.False(t, launched)
-}
-
-// getClientTokenStore binds the process-wide store once and returns the same
-// instance on subsequent calls.
-func TestGetClientTokenStore_BindsOnce(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	s1, err := getClientTokenStore(t.Context(), nc)
-	require.NoError(t, err)
-	require.NotNil(t, s1)
-	s2, err := getClientTokenStore(t.Context(), nc)
-	require.NoError(t, err)
-	assert.Same(t, s1, s2, "store binds once")
 }
@@ -448,7 +448,7 @@ func extractClientError(results []nodeLaunchResult) error {
 		if r.Err == nil {
 			continue
 		}
-		code, ok := awserrors.ResolveErrorCode(r.Err)
+		code, msg, ok := awserrors.ResolveErrorDetail(r.Err)
 		if !ok {
 			continue
 		}
@@ -462,6 +462,9 @@ func extractClientError(results []nodeLaunchResult) error {
 			awserrors.ErrorInvalidGroupNotFound,
 			awserrors.ErrorInvalidParameterValue,
 			awserrors.ErrorSecurityGroupsPerInterfaceLimitExceeded:
+			if msg != "" {
+				return awserrors.Errorf(code, "%s", msg)
+			}
 			return errors.New(code)
 		}
 	}
@@ -424,6 +424,19 @@ func TestExtractClientError_NestedWrappedCode(t *testing.T) {
 	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
 }

+func TestExtractClientError_KeepsMessage(t *testing.T) {
+	t.Parallel()
+	inner := awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "architecture mismatch")
+	results := []nodeLaunchResult{
+		{NodeID: "node-1", Err: fmt.Errorf("launch on node-1: %w", inner)},
+	}
+
+	code, msg, ok := awserrors.ResolveErrorDetail(extractClientError(results))
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+	assert.Equal(t, "architecture mismatch", msg)
+}
+
 func TestExtractClientError_JoinedCode(t *testing.T) {
 	t.Parallel()
 	joined := errors.Join(assert.AnError, errors.New(awserrors.ErrorInvalidGroupNotFound))
@@ -817,7 +830,7 @@ func TestRunInstances_SingleInstanceDistributes(t *testing.T) {
 		MaxCount:     aws.Int64(1),
 	}

-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 1)
 	assert.Equal(t, "i-single", aws.StringValue(reservation.Instances[0].InstanceId))
@@ -1370,7 +1383,7 @@ func TestRunInstances_ClusterPlacementGroupRouting(t *testing.T) {
 		},
 	}

-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 2)
 }
@@ -1419,7 +1432,7 @@ func TestRunInstances_MultiInstanceUsesDistribution(t *testing.T) {
 		MaxCount:     aws.Int64(2),
 	}

-	reservation, err := RunInstances(context.Background(), input, nc, nil, "test-account", nil, nil, 1)
+	reservation, err := RunInstances(context.Background(), input, nc, nil, nil, "test-account", nil, nil, 1)
 	require.NoError(t, err)
 	assert.Len(t, reservation.Instances, 2)
 	assert.True(t, statusQueried, "multi-instance launch should query node status")
@@ -82,7 +82,7 @@ func (h *Handler) Dispatch(w http.ResponseWriter, inv dispatch.Invocation) error
 		return err
 	}
 	// AWS refuses a bad tag resource type even on a DryRun request.
-	if err := validateTagSpecificationTypes(action, input); err != nil {
+	if err := validateTagSpecifications(action, input); err != nil {
 		return err
 	}
 	if dryRunRequested(input) {
@@ -14,6 +14,7 @@ import (

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -62,7 +63,7 @@ func ValidateRequestSpotInstancesInput(input *ec2.RequestSpotInstancesInput) err
 // ClientToken passed through, same vCPU quota gate), then builds and persists one
 // active/fulfilled SpotInstanceRequest per launched instance. On a launch failure
 // (including InsufficientInstanceCapacity) it returns the error and persists nothing.
-func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, iamSvc ec2instanceapi.InstanceProfiles, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota LaunchQuota, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
+func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, tokenBucket ec2idemapi.Bucket, iamSvc ec2instanceapi.InstanceProfiles, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota LaunchQuota, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
 	var output ec2.RequestSpotInstancesOutput

 	if err := ValidateRequestSpotInstancesInput(input); err != nil {
@@ -80,7 +81,7 @@ func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesIn

 	// RunInstances normalises runInput in place (e.g. instance profile to ARN),
 	// so the launch spec echoed back is built from runInput afterwards.
-	reservation, err := ec2instanceapi.RunInstances(ctx, runInput, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
+	reservation, err := ec2instanceapi.RunInstances(ctx, runInput, natsConn, tokenBucket, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes)
 	if err != nil {
 		return output, err
 	}
@@ -125,7 +125,7 @@ func TestRequestSpotInstances_QuotaGateRunsBeforeDispatchAndLaunch(t *testing.T)
 		InstanceCount: aws.Int64(1),
 	}

-	_, err = RequestSpotInstances(context.Background(), input, nc, nil, "test-account", "ap-southeast-2a", nil, quota, 1)
+	_, err = RequestSpotInstances(context.Background(), input, nc, nil, nil, "test-account", "ap-southeast-2a", nil, quota, 1)
 	require.NoError(t, err)

 	quotaIdx := order.indexOf("quota-check")
@@ -5,6 +5,7 @@ import (
 	"math"
 	"reflect"
 	"slices"
+	"strings"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -52,15 +53,26 @@ var ec2TaggableTypes = map[string]taggableTypes{
 	"RunInstances":                    taggable("instance", "volume", "network-interface", "spot-instances-request"),
 }

-// validateTagSpecificationTypes refuses a TagSpecifications entry whose
-// ResourceType the action cannot tag, rather than silently dropping its tags.
-func validateTagSpecificationTypes(action string, input any) error {
+// validateTagSpecifications refuses a TagSpecifications entry whose ResourceType
+// the action cannot tag, or a tag key with the reserved aws: prefix. Only
+// customer requests come through here, so system callers may still set aws: tags.
+func validateTagSpecifications(action string, input any) error {
 	allowed, ok := ec2TaggableTypes[action]
 	if !ok {
 		return nil
 	}
 	for _, spec := range tagSpecifications(input) {
-		if spec == nil || spec.ResourceType == nil {
+		if spec == nil {
+			continue
+		}
+		for _, tag := range spec.Tags {
+			// AWS reserves the prefix in any case.
+			if key := aws.StringValue(tag.Key); strings.HasPrefix(strings.ToLower(key), "aws:") {
+				return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+					"Value ( %s ) for parameter key is invalid. Tag keys starting with 'aws:' are reserved for internal use", key)
+			}
+		}
+		if spec.ResourceType == nil {
 			continue
 		}
 		if rt := aws.StringValue(spec.ResourceType); !slices.Contains(allowed.types, rt) {
@@ -84,7 +84,7 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 			if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
 				return output, errors.New(awserrors.ErrorIncorrectInstanceState)
 			}
-			return output, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return output, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 		}
 		return output, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -110,7 +110,7 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 			if force {
 				return forceDetach(ctx, natsConn, accountID, volumeID, instanceID)
 			}
-			return output, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return output, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 		}
 		return output, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -152,7 +152,7 @@ func TestDetachVolume_WithoutForceStillReportsTheMissingInstance(t *testing.T) {
 		InstanceId: aws.String("i-gone"),
 	}, nc, "000000000001")
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.EqualError(t, err, "The instance ID 'i-gone' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 // The gateway's deadline must outlast the daemon's, or a detach that is still
@@ -69,7 +69,7 @@ func AttachNetworkInterface(ctx context.Context, input *ec2.AttachNetworkInterfa
 		slog.ErrorContext(ctx, "AttachNetworkInterface: NATS request failed",
 			"instanceId", instanceID, "eniId", eniID, "err", err)
 		if errors.Is(err, nats.ErrNoResponders) {
-			return output, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return output, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 		}
 		return output, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -69,7 +69,7 @@ func DetachNetworkInterface(ctx context.Context, input *ec2.DetachNetworkInterfa
 		slog.ErrorContext(ctx, "DetachNetworkInterface: NATS request failed",
 			"instanceId", instanceID, "attachmentId", attachmentID, "err", err)
 		if errors.Is(err, nats.ErrNoResponders) {
-			return output, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+			return output, awserrors.IDNotFound(awserrors.ErrorInvalidInstanceIDNotFound, "instance", instanceID)
 		}
 		return output, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -92,7 +92,7 @@ func TestAttachNetworkInterface_NoResponders(t *testing.T) {
 		InstanceId:         aws.String("i-no-responder"),
 		DeviceIndex:        aws.Int64(0),
 	}, nc, testAccountID)
-	assert.EqualError(t, err, awserrors.ErrorInvalidInstanceIDNotFound)
+	assert.EqualError(t, err, "The instance ID 'i-no-responder' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestAttachNetworkInterface_Success(t *testing.T) {
@@ -245,7 +245,7 @@ func TestDetachNetworkInterface_NoResponders(t *testing.T) {
 	_, err := DetachNetworkInterface(context.Background(), &ec2.DetachNetworkInterfaceInput{
 		AttachmentId: aws.String("eni-attach-1"),
 	}, nc, testAccountID)
-	assert.EqualError(t, err, awserrors.ErrorInvalidInstanceIDNotFound)
+	assert.EqualError(t, err, "The instance ID 'i-no-daemon' does not exist: "+awserrors.ErrorInvalidInstanceIDNotFound)
 }

 func TestDetachNetworkInterface_Success(t *testing.T) {
@@ -158,12 +158,7 @@ var describeEIGWValidFilters = map[string]bool{
 func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx context.Context, input *ec2.DescribeEgressOnlyInternetGatewaysInput, accountID string) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
 	var egressOnlyIGWs []*ec2.EgressOnlyInternetGateway

-	eigwIDs := make(map[string]bool)
-	for _, id := range input.EgressOnlyInternetGatewayIds {
-		if id != nil {
-			eigwIDs[*id] = true
-		}
-	}
+	eigwIDs := awsidentifiers.Set(input.EgressOnlyInternetGatewayIds)

 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeEIGWValidFilters)
 	if err != nil {
@@ -572,19 +572,9 @@ var describeAddressesValidFilters = map[string]bool{

 // DescribeAddresses lists Elastic IPs with optional filtering by allocation ID.
 func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.DescribeAddressesInput, accountID string) (*ec2.DescribeAddressesOutput, error) {
-	allocIDs := make(map[string]bool)
-	for _, id := range input.AllocationIds {
-		if id != nil {
-			allocIDs[*id] = true
-		}
-	}
+	allocIDs := awsidentifiers.Set(input.AllocationIds)

-	publicIPs := make(map[string]bool)
-	for _, ip := range input.PublicIps {
-		if ip != nil {
-			publicIPs[*ip] = true
-		}
-	}
+	publicIPs := awsidentifiers.Set(input.PublicIps)

 	parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeAddressesValidFilters)
 	if err != nil {
@@ -3,7 +3,6 @@ package guestmetadata
 import (
 	"context"
 	"log/slog"
-	"sync"
 	"time"

 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -14,50 +13,6 @@ import (
 // little and keeps the untokened path off the NATS fan-out.
 const v1AllowTTL = 30 * time.Second

-type v1AllowEntry struct {
-	allowed bool
-	expires time.Time
-}
-
-// v1AllowCache memoises whether an instance permits IMDSv1. It exists for
-// amplification control: the untokened path is reachable without credentials, so
-// resolving HttpTokens per request would let any guest drive unbounded
-// DescribeInstances fan-outs.
-type v1AllowCache struct {
-	mu      sync.Mutex
-	entries map[string]v1AllowEntry
-}
-
-func newV1AllowCache() *v1AllowCache {
-	return &v1AllowCache{entries: make(map[string]v1AllowEntry)}
-}
-
-func (c *v1AllowCache) get(instanceID string, now time.Time) (bool, bool) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	e, ok := c.entries[instanceID]
-	if !ok || now.After(e.expires) {
-		return false, false
-	}
-	return e.allowed, true
-}
-
-func (c *v1AllowCache) put(instanceID string, allowed bool, now time.Time) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.entries[instanceID] = v1AllowEntry{allowed: allowed, expires: now.Add(v1AllowTTL)}
-}
-
-func (c *v1AllowCache) sweep(now time.Time) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	for id, e := range c.entries {
-		if now.After(e.expires) {
-			delete(c.entries, id)
-		}
-	}
-}
-
 // imdsV1Allowed reports whether this ENI's instance was launched or modified
 // with HttpTokens=optional, which is the only case where an untokened GET is
 // served. Defaults closed: an unattached ENI, a lookup failure or a missing
@@ -66,7 +21,7 @@ func (s *IMDSServiceImpl) imdsV1Allowed(ctx context.Context, eni *eniFacts) bool
 	if eni.instanceID == "" {
 		return false
 	}
-	if allowed, ok := s.v1Allow.get(eni.instanceID, s.now()); ok {
+	if allowed, ok := s.v1Allow.get(eni.instanceID); ok {
 		return allowed
 	}

@@ -79,6 +34,6 @@ func (s *IMDSServiceImpl) imdsV1Allowed(ctx context.Context, eni *eniFacts) bool

 	// Cached even on failure: without that, a guest could replay a failing
 	// lookup to keep the fan-out running.
-	s.v1Allow.put(eni.instanceID, allowed, s.now())
+	s.v1Allow.put(eni.instanceID, allowed)
 	return allowed
 }
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"net/http"
 	"testing"
-	"time"

 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/stretchr/testify/assert"
@@ -107,36 +106,6 @@ func TestIMDSv1_LookupFailureDeniesAndCaches(t *testing.T) {
 	assert.Equal(t, 1, res.calls, "a failing lookup must not be retried per request")
 }

-// An expired entry is re-resolved, so flipping HttpTokens takes effect without a
-// vpcd restart.
-func TestIMDSv1_CacheExpires(t *testing.T) {
-	c := newV1AllowCache()
-	now := time.Unix(1_700_000_000, 0).UTC()
-
-	c.put("i-1", true, now)
-	allowed, ok := c.get("i-1", now.Add(v1AllowTTL-time.Second))
-	require.True(t, ok)
-	assert.True(t, allowed)
-
-	_, ok = c.get("i-1", now.Add(v1AllowTTL+time.Second))
-	assert.False(t, ok, "entry must expire after the TTL")
-}
-
-// sweep drops expired entries so the map cannot grow without bound across
-// instance churn.
-func TestIMDSv1_CacheSweep(t *testing.T) {
-	c := newV1AllowCache()
-	now := time.Unix(1_700_000_000, 0).UTC()
-
-	c.put("i-old", true, now)
-	c.put("i-new", true, now.Add(v1AllowTTL))
-	c.sweep(now.Add(v1AllowTTL + time.Second))
-
-	_, ok := c.get("i-old", now.Add(v1AllowTTL+time.Second))
-	assert.False(t, ok)
-	assert.Len(t, c.entries, 1, "only the live entry survives the sweep")
-}
-
 // An ENI with no attached instance denies without attempting a lookup.
 func TestIMDSv1_UnattachedENIDenies(t *testing.T) {
 	res := v1Resolver(ec2.HttpTokensStateOptional)
@@ -150,10 +150,12 @@ func withTapConn(h http.Handler, eni *eniFacts, setTTL connTTLSetter) http.Handl
 // clock. The per-tap responder is left nil — the HTTP handler is exercised directly.
 func newTestService(res eniResolver, fIAM profileLookup, assumer stsAssumer) (*IMDSServiceImpl, time.Time) {
 	now := time.Unix(1_700_000_000, 0).UTC()
+	v1Allow := newTTLCache[bool](v1AllowTTL)
+	v1Allow.now = func() time.Time { return now }
 	return &IMDSServiceImpl{
 		resolver:       res,
 		tokens:         newTokenStore(),
-		v1Allow:        newV1AllowCache(),
+		v1Allow:        v1Allow,
 		creds:          newCredCache(assumer),
 		iam:            fIAM,
 		roleMiss:       newRoleMissLogger(func() time.Time { return now }),
@@ -151,3 +151,14 @@ func (c *ttlCache[V]) put(key string, v V) {
 	defer c.mu.Unlock()
 	c.m[key] = ttlCacheEntry[V]{val: v, expiresAt: c.now().Add(c.ttl)}
 }
+
+// sweep evicts every entry that has expired as of now.
+func (c *ttlCache[V]) sweep(now time.Time) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for k, e := range c.m {
+		if now.After(e.expiresAt) {
+			delete(c.m, k)
+		}
+	}
+}
@@ -197,3 +197,20 @@ func TestTTLCache_ExpiresEntries(t *testing.T) {
 	_, ok = c.get("k")
 	assert.False(t, ok)
 }
+
+// sweep drops expired entries so the map cannot grow without bound across
+// instance churn.
+func TestTTLCache_SweepDropsExpired(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	c := newTTLCache[bool](time.Minute)
+	c.now = func() time.Time { return now }
+
+	c.put("old", true)
+	now = now.Add(time.Minute)
+	c.put("new", true)
+	c.sweep(now.Add(time.Second))
+
+	assert.Len(t, c.m, 1, "only the live entry survives the sweep")
+	_, ok := c.m["new"]
+	assert.True(t, ok)
+}
@@ -65,9 +65,11 @@ type eniResolver interface {
 // responder per local primary-ENI tap, each serving the shared mux with the
 // tap's ENI identity.
 type IMDSServiceImpl struct {
-	resolver       eniResolver
-	tokens         *tokenStore
-	v1Allow        *v1AllowCache
+	resolver eniResolver
+	tokens   *tokenStore
+	// v1Allow memoises HttpTokens per instance: the untokened path needs no
+	// credentials, so resolving it per request would allow unbounded fan-outs.
+	v1Allow        *ttlCache[bool]
 	creds          *credCache
 	iam            profileLookup
 	roleMiss       *roleMissLogger
@@ -169,7 +171,7 @@ func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer
 			lookup:   &localInstanceLookup{local: localState, records: records},
 		},
 		tokens:         newTokenStore(),
-		v1Allow:        newV1AllowCache(),
+		v1Allow:        newTTLCache[bool](v1AllowTTL),
 		creds:          newCredCache(sts),
 		iam:            iamSvc,
 		roleMiss:       newRoleMissLogger(time.Now),
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Julian Sommer	3f2fff3542	chore: suppress preflight build script output	2026-11-04 11:41:09 +10:00
Julian Sommer	769bb8014a	test: stub the /dev/kvm probe so QEMU argv tests run on hosts without KVM	2026-11-04 11:37:01 +10:00
Josh Siviour	4d23698d28	feat(ec2): complete-or-error account inventories for instances, spot requests and volumes (#1544 ) * test: pin instance inventory gaps G1-G3 as current behaviour * feat(ec2): strict account instance inventory from the record store * feat(ec2): strict spot-request and volume inventories * fix(ec2): strict instance inventory fails on node-only instances * fix(ec2): strict instance inventory sees error and late strays and rejects malformed records	2026-11-04 11:19:28 +10:00
Julian Sommer	5ae6ae7234	fix: refuse ELBv2 health check, IpAddressType, pool and SSL policy type values outside the model with ValidationError (#1562 )	2026-11-04 11:54:35 +11:00
Julian Sommer	2b51f83c26	refactor: sign ECS agent gateway calls with gwsign and drop the bespoke credential types (#1563 ) The ecs-agent now holds an aws.CredentialsProvider (a CredentialsCache over imdscreds.NewProvider) and ecsgw.New takes a *gwsign.Signer, matching eksgw, rdsgw and elbv2. imdscreds.Fetch and stsauth.AssumeRole return aws.Credentials, so agents/ecs/credentials and the field-copy adapters are removed.	2026-11-04 11:50:49 +11:00
Julian Sommer	845bbb89c0	refactor(admin): delete ParsePredastoreHostIDFromConfig, which only its own tests called (#1561 )	2026-11-04 11:49:28 +11:00
Julian Sommer	57a55caf26	refactor(installer): render sizes and validate subnet masks in the UI through install.HumanSize and install.ValidDottedMask (#1560 )	2026-11-04 11:49:17 +11:00
Julian Sommer	e04d710d0d	refactor(eks): open the per-account bucket through one accountBucket helper (#1559 )	2026-11-04 11:49:03 +11:00
Julian Sommer	9c4b7755b0	refactor(eks): run the access entry, add-on and nodegroup tag CAS updates through kvutil.Update (#1558 )	2026-11-04 11:42:01 +11:00
Julian Sommer	2cdf71f2ce	refactor(qmp): delete the unused command response map and query-block reply envelope (#1557 )	2026-11-04 11:41:47 +11:00
Julian Sommer	83b6cfd3ea	refactor(vm): cache the host virtualization stack with sync.OnceValue (#1556 )	2026-11-04 11:38:35 +11:00
Julian Sommer	92b2d7d925	refactor(eks): mint every EKS agent certificate through one konnectivitycert.Mint (#1555 )	2026-11-04 11:36:58 +11:00
Julian Sommer	81b76172ff	refactor: delete the ECS agent's unused ECS_ACCESS_KEY/ECS_SECRET_KEY config (#1554 )	2026-11-04 11:36:45 +11:00
Julian Sommer	35ae09dd6c	refactor(ec2): build describe ID lookup sets with one identifiers.Set helper (#1553 )	2026-11-04 11:36:31 +11:00
Julian Sommer	bfc4737bf2	refactor(vpc): delete unused Purpose constants, the legacy external-type map and the external IPAM aliases (#1552 )	2026-11-04 11:34:26 +11:00
Julian Sommer	6e51d46bb4	refactor(imds): back the IMDSv1 allow cache with the generic TTL cache, which gains sweep (#1551 )	2026-11-04 11:34:15 +11:00
Julian Sommer	701686704f	refactor(natgw): delete unused PublishAddEvent and test-only GetNatGateway (#1550 )	2026-11-04 11:34:03 +11:00
Julian Sommer	25c7567d79	refactor: share one kvutil.IsRevisionConflict instead of five private conflict checks (#1549 )	2026-11-04 11:30:57 +11:00
Julian Sommer	ba2a418b3d	refactor(iam): share policy attach, inline policy and tag helpers across users, groups, roles and other taggable records (#1548 )	2026-11-04 11:29:14 +11:00
Julian Sommer	08b52169c3	refactor(awserrors): delete the unwired WithFields error fields and the test-only ValidErrorCode and HasErrorCode (#1547 )	2026-11-04 11:28:57 +11:00
Julian Sommer	ad60a7260f	refactor: share AWS error assertions and SeedKV through internal/testkit instead of per-package copies (#1546 )	2026-11-04 11:28:43 +11:00
Julian Sommer	295a4c0040	refactor: collect sorted map keys with slices.Sorted(maps.Keys) in dispatch, SQS, Route 53 and cloud fakes (#1545 )	2026-11-04 11:28:25 +11:00
Julian Sommer	893ac710d6	refactor: share the assumed-role principal type and the caller projection through ingress dispatch (#1543 )	2026-11-04 11:28:12 +11:00
Julian Sommer	9a9ffeccc9	refactor: route RunInstances ClientToken through the shared EC2 idempotency wrapper (#1542 )	2026-11-04 11:27:55 +11:00
Julian Sommer	66fcda9b49	refactor: evaluate EC2 tag-key and tag-value filters only through MatchesTags (#1541 )	2026-11-04 11:27:41 +11:00
Julian Sommer	60eb047721	refactor: send ECR, ACM, EKS, ECS, bedrock and tagging results through one AWS JSON 1.1 writer (#1540 )	2026-11-04 11:19:36 +11:00
Julian Sommer	934110e828	refactor: read the kvstore last sequence through the shared leader message get, honouring the JetStream API prefix and domain (#1539 )	2026-11-04 11:17:54 +11:00
Julian Sommer	8db1b7f63e	refactor: page EKS and RDS listings through foundation/aws/paging and reject foreign EKS tokens (#1538 )	2026-11-04 11:17:39 +11:00
Julian Sommer	b255007fb2	fix: page GetSecurityGroupsForVpc with resume tokens so a deleted group does not shift the next page (#1537 )	2026-11-04 11:15:50 +11:00
Bryn Mailer	52423d3573	feat(uefi): delete a terminated instance's variable store and report unowned stores (#1502 ) * feat(uefi): delete a terminated instance's variable store through its store record and report unowned stores Terminate, on both the running and stopped paths, moves the store record to deleting and stamps a uefi teardown entry, then returns. The cluster-wide uefi-store reaper deletes the objects under the instance prefix and then the record, so the deletion converges even after the terminated record's TTL has expired. CompareAndSet refuses to return a deleting record to live. spx admin volumes orphans now also lists instance prefixes with no owner, image prefixes with no AMI document, and store records with no instance record. It deletes nothing. * refactor(uefi): read instance state for the store reaper and orphan report through daemon capabilities The reaper and the orphan report named the raw instance-state records, buckets and key prefixes, which ADR-0007:S6 forbids a new consumer to do. They now ask TerminateDecided, OpenTeardowns, MarkTeardownDone and KnownInstanceIDs, each stating its absent-record and failure behaviour. * fix(uefi): keep a stopped instance's store when its terminate left the stopped record TerminateDecided read the terminated record first, so a stopped terminate that wrote it but failed to remove the stopped record let the reaper delete the store of an instance that can still start. The live record now decides, and the reaper no longer marks a live store deleting while one exists.	2026-11-04 10:14:27 +10:00
Bryn Mailer	d6a9bc00ef	feat(uefi): launch a UEFI instance from its variable store pointer (#1500 ) * feat(uefi): launch a UEFI instance from its variable store pointer A launch whose record points at a variable store keeps the live copy at <DataDir>/uefi/<instance-id>/vars.fd and opens it as pflash unit 1 with format=raw. It reuses the copy when seeded.json beside it names the pointer, and otherwise gets the object, decompresses it, checks it is exactly the node's VARS size and writes vars.fd then seeded.json by temporary file and rename. A failed get is retryable like a mount; a missing or malformed object is not. A store whose firmware arch or VARS size differs from the node's fails the launch before any volume is mounted, with a Server.InternalError state reason and a log naming both firmwares. A different CODE hash boots, and is logged and counted. A UEFI instance with no pointer and no EFI volume, or with both, is refused with InternalError before its record is changed, in launch, Start and the claimed-instance start. An instance whose variables are on an EFI volume still launches through it. * test(uefi): separate the embedded store from the injected error in the failing-get fake * test(uefi): call startQEMU with its one argument after the dev merge	2026-11-04 10:13:55 +10:00
Bryn Mailer	020319c44a	fix(ec2): classify a volume held elsewhere by the provider code, not lease text (#1496 ) * fix(ec2): classify a volume held elsewhere by the provider code, not lease text The instance service and the EKS cluster reconciler recognised a start refused because another node holds a volume only by viperblockd's lease text, so a provider that words the refusal differently was reported as ServerInternal and counted against the reconciler's restart budget. The instance service now matches the EBS provider's volume_held_elsewhere code, on both the stopped-instance claim and the live-owner start. The owning node marks such a refusal with an X-Volume-Held-Elsewhere reply header, the ec2.start forwarder relays it, and StartInstances wraps ec2v1.ErrVolumeHeldElsewhere around the rebuilt error, which the reconciler matches with errors.Is. The AWS code stays IncorrectState. The lease text stays as a fallback for replies that carry no code: the legacy ebs.mount reply, a publish below schema version 3 (where volume_in_use also means a read-only mismatch), and an owner node that predates the header. * fix(daemon): relay every header of the owner's start reply on both forwarder paths The ec2.start forwarder copied only the held-elsewhere header onto a relayed reply, and its unsealed-writes branch answered with the body alone, so any other header the owning node set was lost in the relay. Both relay branches now go through relayStartReply, which forwards the owner's reply body and every header unchanged. * test(daemon): pin that only a held volume gets the held-elsewhere header The owner's reply header is the one signal that a start was refused because another node holds a volume, and every other start refusal in the same IncorrectState code must reach the caller without it. * refactor(ec2): classify a volume held elsewhere only by the provider code Every node sends provider requests at the one contract version, at which every provider refuses a volume held by another node with the volume_held_elsewhere code. The lease-text fallbacks in the instance service and the EKS cluster reconciler matched replies that no longer exist, so they are removed: a start is a held-volume refusal only when the mount error carries the provider code, or when the start error wraps ec2v1.ErrVolumeHeldElsewhere, which StartInstances rebuilds from the owner's header. The ec2instance.ErrVolumeHeldElsewhere alias is removed, so the sentinel has one name, ec2v1.ErrVolumeHeldElsewhere. The reconciler's budget test now proves an IncorrectState without the sentinel still counts, whether its message is unrelated or carries the lease text. * fix(ec2): log a held-volume start refusal at Warn with the instance ID A start refused because another node holds a volume is returned as IncorrectState, a client error, yet was logged at Error without the instance it concerned. It is now logged at Warn, and every vmMgr.Start failure log names the instance.	2026-11-04 10:13:23 +10:00
Julian Sommer	9c73e2c523	fix: fail instance-role signing when an expired IMDS key cannot be refreshed (#1536 )	2026-11-04 11:12:59 +11:00
Julian Sommer	3e588fdcb6	test: delete the never-called AssertPredastoreObjectLifecycle e2e helper (#1535 )	2026-11-04 11:12:40 +11:00
Julian Sommer	29b8788fec	refactor: delete the unused vpc.ValidatePoolConfig and its IP comparison helper (#1534 )	2026-11-04 11:12:26 +11:00
Julian Sommer	acaa22fd25	refactor: delete the unwired gpu Manager.MarkMIGFailed and its tests (#1533 )	2026-11-04 11:12:12 +11:00
Julian Sommer	1bda358f0b	fix: accept the private-ip-address filters on DescribeInstances (#1529 )	2026-11-04 11:11:56 +11:00
Julian Sommer	637f68c62b	refactor: drop redundant helpers in DHCP leases, OVN router lookups and Route 53 tag sets (#1532 )	2026-11-04 11:11:22 +11:00
Julian Sommer	e2bd1f523a	fix: name the caller, action and repository in ECR registry policy denials (#1531 )	2026-11-04 11:11:10 +11:00
Julian Sommer	4e8c182e2f	fix: name the resource, not the cluster, in the default EKS ResourceNotFoundException message (#1530 )	2026-11-04 11:06:04 +11:00
Julian Sommer	4f5049d6da	fix: refuse DetachInternetGateway with DependencyViolation while an elastic IP is mapped in the VPC (#1523 )	2026-11-04 11:05:27 +11:00
Julian Sommer	98bde73228	fix: accept the tag-key and tag-value filters in DescribeVolumes and DescribeSnapshots (#1528 )	2026-11-04 11:00:26 +11:00
Julian Sommer	5645b49a61	fix: return LoadBalancerNotFound from DescribeLoadBalancers for unknown load balancer ARNs (#1527 )	2026-11-04 10:59:58 +11:00
Julian Sommer	4851a9f8e4	fix: report Attachment.InstanceOwnerId on described network interfaces, as amazon-elb for system VMs (#1526 )	2026-11-04 10:57:38 +11:00
Julian Sommer	7af40f3411	fix: refuse RunInstances when the AMI architecture differs from the instance type (#1525 )	2026-11-04 10:57:22 +11:00
Julian Sommer	56658c1647	fix: return counts, statistics and empty lists on every ECS cluster response and honour DescribeClusters include (#1518 )	2026-11-04 10:56:04 +11:00
Julian Sommer	795cf8de13	fix: return HTTP 500 for InternalFailure so SDKs treat it as a server fault (#1524 )	2026-11-04 10:54:05 +11:00
Julian Sommer	b9c412f385	fix: reject RegisterTargets for an instance that does not exist with ValidationError (#1520 )	2026-11-04 10:47:31 +11:00
Julian Sommer	e7c1ccf5ad	fix: name the missing instance ids in InvalidInstanceID.NotFound from CreateTags and DescribeInstances (#1522 )	2026-11-04 10:44:31 +11:00
Julian Sommer	f53b7dab03	fix: refuse reserved aws: tag keys in TagSpecifications on EC2 create calls (#1521 )	2026-11-04 10:41:13 +11:00
Julian Sommer	23aed4a08d	fix(elbv2): validate tags on AddTags and the four create calls instead of silently dropping them (#1519 ) Tags are checked against the ELBv2 model's key and value length and pattern (ValidationError), repeated keys are refused with DuplicateTagKeys, and more than 50 tags on a resource with TooManyTags, counting a resource's existing tags on AddTags. A tag with no Value is stored as an empty value and a tag with no Key is refused rather than dropped.	2026-11-04 10:40:46 +11:00
Julian Sommer	21500f18c3	refactor(cli): delete the unread admin init --host and global --host, --nats-token and --nats-subject flags (#1516 )	2026-11-04 10:36:16 +11:00
Julian Sommer	a94038b358	refactor: share the stream invocation record, route action list and e2e key-pair helpers (#1515 ) Ochre's two streaming pumps now record through one streamRecordCtx.record and frame upstream faults through one frameWriter.writeUpstreamFault. The EKS and Lambda route tables list their actions through rest.Router.Actions, toDetails converts toSummary's identically-shaped struct, Lambda's environment response uses aws.StringMap, and the e2e suites call harness.DiscoverNanoInstanceType and harness.EnsureKeyPair directly.	2026-11-04 10:23:10 +11:00
Julian Sommer	1f2dc13c5e	refactor(installer): delete unused ImportablePools, Config.PlaneBridge, RolesForDisks and Disk.Stable (#1514 )	2026-11-04 10:22:22 +11:00