test(e2e): attach and detach a volume on a stopped instance, as AWS allows (#1508)

* test: pin guest poweroff handling and stopped-instance volume state

* test: gofmt the poweroff and stopped-volume tests

* test: pin start after detach, one-poll poweroff, reattach of released volumes and the delete refusal

* test: separate the embedded mounter from the recorder fields

* fix: end a guest poweroff as a stop and keep stopped instances' data volumes attached

A clean QEMU exit after the heartbeat saw the guest power off now stops the
instance instead of waiting for crash detection. Unmount no longer releases
data volumes. DetachVolume on a stopped instance edits the record, then
releases the volume. Starting an instance stopped on an older build drops data
volumes that are attached elsewhere or deleted and fails on a missing boot
volume.

* fix: keep the stopped-instance detach inside the instance service file and format start reconcile

* docs: describe guest poweroff, stopped-instance volumes and detach on a stopped instance

* fix: allow detaching the root volume of a stopped instance and refuse a start with no root volume as AWS does

* feat: attach volumes to a stopped instance and fix root-detach start fixtures

* docs: describe attach and root detach on a stopped instance

* test: Unmount seals only and leaves every attached volume in-use

Changed with the person's explicit approval, relayed by dev-design on 11-03.
EBS root and data volumes persist across stop on AWS, so stop and crash
unmount no longer release any attached volume.

* docs: note that the EFI variable store cannot be detached from a stopped instance

* test: unmount contract tests expect no volume state write on stop

* test: order imports in stopped-volume daemon tests

* refactor: drop the unused volume state updater from the volume mounter

.github/workflows/e2e-baremetal.yml

@@ -71,9 +71,16 @@ jobs:
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
       SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
+      # A cancelled job's root steps outlive it, since the runner cannot reach
+      # them once it kills sudo, and would keep writing here and to the cache.
+      # A rootfs killed before bin/bash exists is one the build refuses to reuse.
       - name: Clean Workspace
         run: |
+          sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
           sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ -d "${BM_ISO_CACHE}/rootfs" ] && [ ! -x "${BM_ISO_CACHE}/rootfs/bin/bash" ]; then
+            sudo rm -rf --one-file-system "${BM_ISO_CACHE}/rootfs"
+          fi
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
@@ -113,10 +120,15 @@ jobs:
             "$GITHUB_WORKSPACE" \
             "${RUNNER_TEMP}/spinifex-tests.tar" \
             "${RUNNER_TEMP}/bm-bins"
+          # Built here, as the runner user, so the PXE build finds it current.
+          make -C "$GITHUB_WORKSPACE" build-microvm-image
 
       - name: Build PXE Artifacts
         run: |
-          sudo make -C mulga/scripts/iso-builder \
+          # -o build-spinifex: the binaries come from Build Artifacts, and a
+          # rebuild here would write root-owned files into the workspace.
+          sudo systemd-run --scope --collect --quiet --unit=spinifex-bm-pxe -- \
+            make -C mulga/scripts/iso-builder -o build-spinifex \
             build-rootfs build-squashfs build-initramfs \
             CACHE="${BM_ISO_CACHE}" \
             SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
@@ -124,6 +136,10 @@ jobs:
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
             VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
+          if root_owned="$(find "$GITHUB_WORKSPACE" -user root -print -quit)" && [ -n "$root_owned" ]; then
+            echo "::error::the PXE build wrote root-owned files into the workspace, e.g. $root_owned"
+            exit 1
+          fi
 
       - name: Write ePDU credentials
         run: |
@@ -195,13 +211,18 @@ jobs:
           done
           export BM_TEST_TIMEOUT
           echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
-          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
+          sudo -E systemd-run --scope --collect --quiet --unit=spinifex-bm-e2e -- \
+            mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
             --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
             --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
             2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
 
+      - name: Stop this job's root steps
+        if: always()
+        run: sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
+
       - name: Generate JUnit report
         if: always()
         run: |

architecture/layering/allowlist_test.go

@@ -15,9 +15,6 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/key", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/launchtemplate", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/placementgroup", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},

contracts/ec2/v1/README.md

@@ -23,7 +23,7 @@ subject shape are wire compatibility. Callers and receivers must use this
 package rather than rebuild those details. `commands_test.go` preserves the
 version-1 JSON shape and subject form.
 
-This package owns shared command payloads and the drain-volume response. It
+This package owns shared command payloads. It
 does not define every action's reply body, authorize the request, store EC2
 state or implement NATS transport. Those remain responsibilities of the EC2
 domain and its runtime composition.

contracts/ec2/v1/commands.go

@@ -29,7 +29,6 @@ type EC2InstanceCommand struct {
 	Attributes                EC2CommandAttributes       `json:"attributes"`
 	AttachVolumeData          *AttachVolumeData          `json:"attach_volume_data,omitempty"`
 	DetachVolumeData          *DetachVolumeData          `json:"detach_volume_data,omitempty"`
-	DrainVolumeData           *DrainVolumeData           `json:"drain_volume_data,omitempty"`
 	AttachENIData             *AttachENIData             `json:"attach_eni_data,omitempty"`
 	DetachENIData             *DetachENIData             `json:"detach_eni_data,omitempty"`
 	IamProfileAssociationData *IamProfileAssociationData `json:"iam_profile_association_data,omitempty"`
@@ -45,7 +44,6 @@ type EC2CommandAttributes struct {
 	StartInstance               bool `json:"start_instance"`
 	AttachVolume                bool `json:"attach_volume"`
 	DetachVolume                bool `json:"detach_volume"`
-	DrainVolume                 bool `json:"drain_volume,omitempty"`
 	RebootInstance              bool `json:"reboot_instance"`
 	AttachENI                   bool `json:"attach_eni"`
 	DetachENI                   bool `json:"detach_eni"`
@@ -69,33 +67,6 @@ type DetachVolumeData struct {
 	Force    bool   `json:"force,omitempty"`
 }
 
-// DrainVolumeData carries parameters for a drain-volume command, which flushes
-// the volume's in-flight writes to S3 before a snapshot reads them. The command
-// is addressed to the instance the volume is attached to because the drain
-// socket only exists on the node hosting it.
-type DrainVolumeData struct {
-	VolumeID string `json:"volume_id"`
-}
-
-const (
-	// DrainVolumeStatusDrained is the ack a node returns once the volume's
-	// in-flight writes have reached S3.
-	DrainVolumeStatusDrained = "drained"
-
-	// DrainVolumeStatusNotRunning is the ack a node returns when it still holds
-	// the instance but the VM is not running, so no process is writing to the
-	// volume and its sealed checkpoint is already current. An attachment record
-	// outlives the writer — stop deliberately leaves boot volumes attached — so
-	// the caller cannot tell this from the record alone.
-	DrainVolumeStatusNotRunning = "not-running"
-)
-
-// DrainVolumeResponse is the reply to a drain-volume command.
-type DrainVolumeResponse struct {
-	VolumeID string `json:"volume_id"`
-	Status   string `json:"status"`
-}
-
 // AttachENIData carries parameters for an attach-network-interface command.
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`

contracts/ec2/v1/commands_test.go

@@ -16,7 +16,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			StartInstance:               true,
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			RebootInstance:              true,
 			AttachENI:                   true,
 			DetachENI:                   true,
@@ -28,7 +27,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
-		DrainVolumeData:           &DrainVolumeData{VolumeID: "vol-3"},
 		AttachENIData:             &AttachENIData{NetworkInterfaceID: "eni-1", DeviceIndex: 2},
 		DetachENIData:             &DetachENIData{AttachmentID: "eni-attach-1", Force: true},
 		IamProfileAssociationData: &IamProfileAssociationData{InstanceProfileArn: "arn:aws:iam::123456789012:instance-profile/example"},
@@ -47,7 +45,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"start_instance": true,
 			"attach_volume": true,
 			"detach_volume": true,
-			"drain_volume": true,
 			"reboot_instance": true,
 			"attach_eni": true,
 			"detach_eni": true,
@@ -59,7 +56,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
-		"drain_volume_data": {"volume_id": "vol-3"},
 		"attach_eni_data": {"network_interface_id": "eni-1", "device_index": 2},
 		"detach_eni_data": {"attachment_id": "eni-attach-1", "force": true},
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
@@ -69,16 +65,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 	}`, string(data))
 }
 
-func TestDrainVolumeResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(DrainVolumeResponse{
-		VolumeID: "vol-1",
-		Status:   DrainVolumeStatusDrained,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"volume_id":"vol-1","status":"drained"}`, string(data))
-	require.Equal(t, "not-running", DrainVolumeStatusNotRunning)
-}
-
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/operator/v1/admin.go

@@ -84,8 +84,8 @@ type ListAccountsResponse struct {
 }
 
 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing. QuiescedScope is filled only for a
-// non-finalizing run, and holds as of EvaluatedAt: a write after that is not caught.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only by a run
+// that guarantees a final inventory re-read, and holds as of EvaluatedAt.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`

contracts/viperblockd/legacy/v1/README.md

@@ -51,19 +51,6 @@ request/reply replies.
 removed its local state and export. These routes and JSON shapes are deployed
 legacy compatibility behaviour, not a generic EBS-provider interface.
 
-## Fence route
-
-`VolumeFencedSubject(node)` is `ebs.<node>.fenced`, or `ebs.fenced` with no
-node for a single-node daemon. The Viperblock adapter publishes
-`VolumeFencedEvent` on it after tearing down an export whose volume lease
-moved to another node; the local daemon subscribes to stop the guest that
-was using it. Node-addressed, like mount and unmount: the guest is local, and
-no other node's daemon has anything to stop.
-
-The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
-The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
-Repeats are harmless, since fencing a guest that is already fenced changes nothing.
-
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVolumeAbandonSubject_BothForms pins the abandon route the same way the
-// fenced route is pinned: node-addressed, with an empty-node fallback for a
-// single-node daemon.
+// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
+// with an empty-node fallback for a single-node daemon.
 func TestVolumeAbandonSubject_BothForms(t *testing.T) {
 	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
 	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))

contracts/viperblockd/legacy/v1/fence.go

@@ -1,29 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeFencedSubject is where a fenced volume is announced so the daemon on
-// this node can stop the guest that was using it. Node-addressed: the guest is
-// local, and every other node's daemon has nothing to do about it.
-func VolumeFencedSubject(node string) string {
-	if node == "" {
-		return "ebs.fenced"
-	}
-	return "ebs." + node + ".fenced"
-}
-
-// VolumeFencedEvent announces that a node tore down a volume's export because
-// the lease moved. Winner is who holds it now, for the operator reading the
-// guest's stop reason.
-type VolumeFencedEvent struct {
-	Volume string `json:"volume"`
-	Node   string `json:"node"`
-	Winner string `json:"winner"`
-	Reason string `json:"reason"`
-}
-
-// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
-// ends the sender's retries; Instance names the guest stopped, if one was, and
-// Error says why the daemon could not act on the event.
-type VolumeFencedAck struct {
-	Instance string `json:"instance,omitempty"`
-	Error    string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -1,54 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeFencedSubject_IsNodeAddressed pins the routing. A fenced guest is on
-// the node that lost the volume, so a subject without the node in it would reach
-// daemons with nothing to stop and, on a queue group, miss the one that has.
-func TestVolumeFencedSubject_IsNodeAddressed(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.fenced", viperblocklegacyv1.VolumeFencedSubject("node-a"))
-	assert.Equal(t, "ebs.fenced", viperblocklegacyv1.VolumeFencedSubject(""),
-		"a single-node daemon has no node name, and still has to hear its own fences")
-}
-
-// TestVolumeFencedEvent_JSON pins the exact field spelling of the fenced
-// event, published by viperblockd and consumed by the daemon.
-func TestVolumeFencedEvent_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","node":"node-a","winner":"node-b","reason":"volume lease moved to node-b while this node had the volume open"}`
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-1",
-		Node:   "node-a",
-		Winner: "node-b",
-		Reason: "volume lease moved to node-b while this node had the volume open",
-	}
-
-	marshaled, err := json.Marshal(event)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeFencedEvent
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, event, decoded)
-}
-
-// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
-// viperblockd's retries of a fence notice.
-func TestVolumeFencedAck_JSON(t *testing.T) {
-	golden := `{"instance":"i-1","error":"bad event"}`
-	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
-
-	marshaled, err := json.Marshal(ack)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
-	require.NoError(t, err)
-	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
-}

docs/DESIGN.md

@@ -112,14 +112,14 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
+3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
 6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
 
 ```go
-// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
-reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
+// Volume mounting through the EBS provider contract (runtime/daemon/vm_adapters.go)
+published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: req.Name, NodeID: a.node})
 
 // QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
 cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64

docs/compute/launching-instances/README.md

@@ -23,6 +23,7 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
+- [Stopping and Data Volumes](#stopping-and-data-volumes)
 - [Modify Instance Attributes](#modify-instance-attributes)
 - [Instance Metadata Options](#instance-metadata-options)
 - [Spot Instances](#spot-instances)
@@ -73,6 +74,31 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
 aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 ```
 
+## Stopping and Data Volumes
+
+A guest that powers itself off (`poweroff`, `shutdown -h`) goes `stopping` then `stopped`, the same as `stop-instances`.
+It is not restarted automatically.
+`InstanceInitiatedShutdownBehavior=terminate` is not honoured yet: a guest shutdown always stops the instance.
+
+Data volumes stay `in-use` and attached to a stopped instance, and `start-instances` mounts them again.
+While the instance is stopped, `attach-volume` of one of its data volumes to another instance returns `VolumeInUse`, and `delete-volume` is refused.
+
+`detach-volume` works on a stopped instance, with or without `--instance-id`.
+The volume becomes `available` and the next start does not attach it.
+The root volume of a stopped instance can be detached too, as on AWS.
+The internal EFI variable store is not a customer volume: it never appears in `describe-volumes` or `BlockDeviceMappings`, and detaching it is refused with `OperationNotPermitted`.
+The instance then cannot start: `start-instances` returns `InvalidParameterValue` with "Instance does not have a volume attached at root (/dev/sda1)".
+
+`attach-volume` works on a stopped instance too.
+The volume must be `available` and in the instance's availability zone, otherwise the call returns `VolumeInUse` or `InvalidVolume.ZoneMismatch`.
+The attachment is recorded on the stopped instance and the volume becomes `in-use` straight away.
+Attaching at the root device name (for example `/dev/sda1`) makes the volume the root again, and the next `start-instances` boots from it.
+
+An instance stopped by an earlier build had its data volumes released to `available` while its record still listed them.
+On start, a listed data volume that was deleted or is now attached to another instance is dropped with a warning in the daemon log, and the start continues.
+A listed volume that is still `available` and unattached is attached again.
+A missing root volume fails the start with the same `InvalidParameterValue` answer.
+
 ## Modify Instance Attributes
 
 Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
@@ -135,6 +161,30 @@ aws ec2 get-console-output --instance-id $INSTANCE_ID \
   --query 'Output' --output text | base64 -d
 ```
 
+Like AWS, the call returns the most recent 64 KiB of the console.
+While an instance is shutting down after a terminate request, the call reads the renamed file described below.
+Once the instance has left the node's running set, which includes a stopped or terminated instance, the node no longer answers and the call returns `InvalidInstanceID.NotFound`.
+AWS still lists a terminated instance for a while after termination, so a caller may expect this call to work then; it does not here, a known gap.
+Whether the API should answer for a stopped instance is not settled here; only the file on the host is kept.
+
+### Where console logs live and how big they get
+
+The serial console of each instance is written on its host to `/run/spinifex/console-<instance-id>.log`.
+`/run` is memory-backed, so these files are cleared when the host reboots, including any left by older releases.
+The daemon keeps them bounded.
+A file over 4 MiB is cut back in place to its newest 1 MiB, and the guest keeps writing while it happens.
+The console is opened in append mode, so a reboot or auto-restart of the guest adds to the same file instead of starting over.
+A live `console-<instance-id>.log` is deleted only when it has not changed for an hour and no QEMU process on the host has it open, so a stopped instance keeps its console for an hour and a quiet running guest keeps it indefinitely.
+If the daemon cannot read the open files of every QEMU process on the host, it deletes nothing on that pass and logs a warning.
+This also clears the file of an instance that was stopped and then terminated, or that was recovered on another node, within about an hour of its last write.
+When an instance is terminated, including a failed first launch and the system instances behind load balancers, queues and EKS control planes, its file is renamed to `console-<instance-id>.log.terminated`.
+The daemon removes a `.terminated` file an hour after its last write.
+
+Known limits:
+
+- The log shipper follows only `console-*.log`, so it stops following a file when it is renamed on terminate, and the last few seconds of output that it had not yet read can be lost.
+- The instance password retrieval reads only the live file name, so it does not find a console renamed after terminate.
+
 ## Instance Types
 
 List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.

docs/containers/eks/README.md

@@ -59,6 +59,9 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
+- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -377,6 +380,15 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
+**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
+
+```bash
+aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
+aws iam delete-instance-profile --instance-profile-name <role-name>
+```
+
+**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
+
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures

docs/package-boundary/eks-cluster-lifecycle.md

@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste
 
 ## Gaps
 
-1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.

docs/vultr-integration/README.md

@@ -265,7 +265,12 @@ If this returns a list of available instance types, your installation is working
 
 Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
 
-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+Log in to node 1, and follow [Setting Up Your Cluster](/docs/setting-up-your-cluster) there to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance:
+
+```bash
+./deploy-vultr.sh ssh
+```
+
 An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
 
 ## 8. Test resiliency

internal/testkit/drainsocket.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,10 +29,11 @@ func SocketTempDir(t *testing.T) string {
 // StartDrainSocket stands in for the NBD plugin's drain socket, at the path the
 // plugin serves it on the node hosting volumeID ({dataDir}/viperblock/{volumeID}
 // /snapshot.sock). Every connection is answered with ack ("OK\n" for a drain
-// that reached S3, "ERR\n" for one that did not) and closed.
-func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) {
+// that reached S3, "ERR\n" for one that did not) and closed. connections
+// reports how many drains have been asked for so far.
+func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) (connections func() int) {
 	t.Helper()
-	serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
+	return serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
 }
 
 // StartBlockingDrainSocket stands in for a plugin whose flush is still running:
@@ -43,7 +45,7 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 
 	acceptedCh := make(chan struct{})
 	releaseCh := make(chan struct{})
-	serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
+	_ = serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
 
 	var once sync.Once
 	t.Cleanup(func() { once.Do(func() { close(releaseCh) }) })
@@ -52,8 +54,9 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 }
 
 // serveDrainSocket answers every connection with ack, optionally closing
-// accepted on the first one and waiting for release before it replies.
-func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) {
+// accepted on the first one and waiting for release before it replies. The
+// returned func counts the connections accepted so far.
+func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) func() int {
 	t.Helper()
 
 	dir := filepath.Join(dataDir, "viperblock", volumeID)
@@ -63,6 +66,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = ln.Close() })
 
+	var count atomic.Int64
 	go func() {
 		var once sync.Once
 		for {
@@ -70,6 +74,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			if err != nil {
 				return // listener closed by the cleanup above
 			}
+			count.Add(1)
 			if accepted != nil {
 				once.Do(func() { close(accepted) })
 			}
@@ -80,4 +85,5 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			_ = conn.Close()
 		}
 	}()
+	return func() int { return int(count.Load()) }
 }

scripts/images/eks-node/eks-node-role.sh

@@ -159,6 +159,7 @@ case "${ROLE}" in
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     server-join)
@@ -178,12 +179,14 @@ case "${ROLE}" in
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     agent)
         log "configuring agent role"
         svc_enable k3s-agent
         svc_start k3s-agent
+        svc_optional mulga-eks-log-forward
         ;;
     *)
         die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"

scripts/images/eks-node/eks-node-role_test.sh

@@ -146,6 +146,15 @@ for INIT in openrc systemd; do
     )"
     check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
     check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
@@ -196,6 +205,15 @@ for INIT in openrc systemd; do
     # published the cluster-identical artifacts, and a join re-publish would
     # only race the bootstrap bus. This is deliberate, not an oversight.
     check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
+    check "server-join(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
     check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
@@ -207,8 +225,39 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+    check "agent(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): log-forward is enabled after k3s-agent" "yes" "$(
+        awk -v k="$(enable_cmd "${INIT}" k3s-agent)" -v f="$(enable_cmd "${INIT}" mulga-eks-log-forward)" '
+            index($0, k) == 1 { kn = NR }
+            index($0, f) == 1 { fn = NR }
+            END { print (kn > 0 && fn > kn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
+
     check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
+    # A broken log forwarder must never stop a node joining.
+    for FAILING in enable start; do
+        run_case "${INIT}" agent_logfwd_${FAILING}_fail
+        printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
+        if [ "${FAILING}" = enable ]; then
+            STUB_FAIL="$(enable_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
+        else
+            STUB_FAIL="$(start_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
+        fi
+        invoke
+        unset STUB_FAIL
+        check "agent(${INIT}): log-forward ${FAILING} failure still exits 0" 0 "${rc}"
+        check "agent(${INIT}): log-forward ${FAILING} failure still enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still starts k3s-agent" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
+    done
+
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred
     printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"

scripts/images/eks-node/manifest.conf

@@ -45,6 +45,8 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
 scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward \
+scripts/images/eks-node/mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-log-forward.initd

@@ -0,0 +1,30 @@
+#!/sbin/openrc-run
+# mulga-eks-log-forward — copies k3s warnings and errors to the serial console
+# so they reach the Elasticsearch sink. Best effort: nothing depends on it and
+# a failure here never affects the node.
+
+description="Forward k3s warnings and errors to the serial console"
+command="/bin/sh"
+command_background="yes"
+supervisor="supervise-daemon"
+respawn_delay=5
+pidfile="/run/mulga-eks-log-forward.pid"
+output_log="/dev/null"
+error_log="/dev/null"
+
+depend() {
+    after k3s k3s-agent
+}
+
+start_pre() {
+    role=$(cat /etc/spinifex-eks/role 2>/dev/null || true)
+    if [ "${role}" = "agent" ]; then
+        log=/var/log/k3s-agent.log
+        FORWARD_PREFIX="[k3s-agent]"
+    else
+        log=/var/log/k3s.log
+        FORWARD_PREFIX="[k3s]"
+    fi
+    export FORWARD_PREFIX
+    command_args="-c 'tail -n +1 -F ${log} 2>/dev/null | /usr/local/sbin/mulga-eks-log-forward'"
+}

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# mulga-eks-log-forward — copy the warnings and errors of a k3s log to the
+# serial console, where the host picks them up for the Elasticsearch sink.
+#
+# Reads the tailed log on stdin. Lines that matter are redacted first, then
+# cut to FORWARD_MAX_BYTES, then rate limited: at most FORWARD_RATE_PER_MIN a
+# minute reach the console and the rest are reported as one "N lines dropped"
+# line. Every console write has a timeout, so a stuck console loses lines
+# instead of stalling the tail or k3s.
+#
+# Env: FORWARD_CONSOLE (default /dev/console), FORWARD_PREFIX, FORWARD_MAX_BYTES
+# (512), FORWARD_RATE_PER_MIN (60), FORWARD_WRITE_TIMEOUT seconds (2) and
+# FORWARD_CLOCK_CMD (default: seconds of uptime, read without forking).
+
+CONSOLE=${FORWARD_CONSOLE:-/dev/console}
+PREFIX=${FORWARD_PREFIX:-[k3s]}
+MAX_BYTES=${FORWARD_MAX_BYTES:-512}
+RATE=${FORWARD_RATE_PER_MIN:-60}
+WRITE_TIMEOUT=${FORWARD_WRITE_TIMEOUT:-2}
+CLOCK_CMD=${FORWARD_CLOCK_CMD:-}
+
+now() {
+    if [ -n "${CLOCK_CMD}" ]; then
+        sh -c "${CLOCK_CMD}"
+    else
+        read -r up _ < /proc/uptime && echo "${up%%.*}"
+    fi
+}
+
+# emit TEXT: one console line, abandoned after WRITE_TIMEOUT. Returns non-zero
+# when the console did not take it.
+emit() {
+    timeout "${WRITE_TIMEOUT}" sh -c 'printf "%s\n" "$1" >> "$2"' sh "$1" "${CONSOLE}" 2>/dev/null
+}
+
+# Filter, redact and truncate in one awk pass so lines that do not matter cost
+# no fork. Redaction runs before the cut so a secret straddling the limit
+# cannot leave a readable prefix behind.
+select_lines() {
+    LC_ALL=C awk -v max="${MAX_BYTES}" '
+    {
+        line = $0
+        gsub(/\r/, "", line)
+        keep = 0
+        if (line ~ /level=(warning|error)/) keep = 1
+        else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
+        else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
+        if (!keep) next
+
+        gsub(/[Bb]earer +[A-Za-z0-9._~+\/=-]+/, "Bearer [REDACTED]", line)
+        gsub(/K10[0-9a-f]+(::[A-Za-z0-9-]+:[^ "\047]+)?/, "[REDACTED]", line)
+        gsub(/eyJ[A-Za-z0-9_.-]+/, "[REDACTED]", line)
+        gsub(/(AKIA|ASIA)[A-Z0-9]+/, "[REDACTED]", line)
+        gsub(/X-Amz-Signature=[^& "\047]*/, "X-Amz-Signature=[REDACTED]", line)
+        gsub(/X-Amz-Credential=[^& "\047]*/, "X-Amz-Credential=[REDACTED]", line)
+        gsub(/X-Amz-Security-Token=[^& "\047]*/, "X-Amz-Security-Token=[REDACTED]", line)
+        gsub(/[^[:print:]\t]/, " ", line)
+        print substr(line, 1, max)
+    }'
+}
+
+win_start=0
+sent=0
+dropped=0
+
+flush_dropped() {
+    [ "${dropped}" -gt 0 ] || return 0
+    emit "${PREFIX} ${dropped} lines dropped"
+    dropped=0
+}
+
+select_lines | {
+    while IFS= read -r line; do
+        t=$(now) || t=0
+        if [ "${win_start}" -eq 0 ] || [ $((t - win_start)) -ge 60 ]; then
+            flush_dropped
+            win_start=${t}
+            sent=0
+            [ "${win_start}" -eq 0 ] && win_start=1
+        fi
+        if [ "${sent}" -ge "${RATE}" ]; then
+            dropped=$((dropped + 1))
+            continue
+        fi
+        sent=$((sent + 1))
+        emit "${PREFIX} ${line}" || dropped=$((dropped + 1))
+    done
+    flush_dropped
+}
+exit 0

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -0,0 +1,190 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-log-forward.sh: the line filter,
+# redaction, truncation, rate cap and blocked-console behaviour. The script
+# reads the tailed log on stdin and writes to FORWARD_CONSOLE, which the test
+# points at a temp file (or a reader-less FIFO), so no root or tty is needed.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-log-forward_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-log-forward.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+check() { # check <desc> <expected> <actual>
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        echo "FAIL: $1"
+        echo "  expected: [$2]"
+        echo "  actual:   [$3]"
+    fi
+}
+
+# forward <name>: feeds stdin to the script with a per-case console file and a
+# fixed clock, leaving the forwarded text in ${OUT}.
+forward() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}"
+    OUT="${CASE}/console"
+    : > "${OUT}"
+    FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" \
+        FORWARD_CLOCK_CMD="echo 1000" sh "${SCRIPT}" 2>/dev/null || true
+}
+forwarded() { grep -c . "${OUT}" || true; }
+
+# --- filter: lines that must pass ---
+forward filter_pass <<'EOF'
+time="2026-11-03T03:00:00Z" level=warning msg="slow response from apiserver"
+time="2026-11-03T03:00:01Z" level=error msg="failed to renew"
+W1103 03:00:02.123456    1234 reflector.go:561] watch of *v1.Pod ended
+E1103 03:00:03.123456    1234 kubelet.go:2400] node not found
+I1103 03:00:04.123456    1234 kubelet_node_status.go:73] Node status update failed
+I1103 03:00:05.123456    1234 controller.go:193] Failed to update lease
+I1103 03:00:06.123456    1234 kubelet.go:2200] PLEG is not healthy
+time="2026-11-03T03:00:07Z" level=info msg="Event: NodeNotReady kubelet"
+EOF
+check "filter: all eight interesting lines pass" "8" "$(forwarded)"
+check "filter: lines carry the prefix" "8" "$(grep -c '^\[k3s-agent\] ' "${OUT}" || true)"
+
+# --- filter: lines that must be dropped ---
+forward filter_drop <<'EOF'
+time="2026-11-03T03:00:00Z" level=info msg="Starting k3s agent"
+time="2026-11-03T03:00:01Z" level=debug msg="tick"
+I1103 03:00:02.123456    1234 server.go:100] Version: v1.32.5
+I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
+EOF
+check "filter: info and debug lines are dropped" "0" "$(forwarded)"
+
+# --- redaction: one case per secret shape ---
+redact() { # redact <name> <line> <secret-that-must-not-appear>
+    forward "redact_$1" <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="$2"
+EOF
+    check "redact(${1}): line still forwarded" "1" "$(forwarded)"
+    if grep -qF "$3" "${OUT}"; then
+        check "redact(${1}): secret removed" "absent" "present"
+    else
+        check "redact(${1}): secret removed" "absent" "absent"
+    fi
+    check "redact(${1}): marker present" "1" "$(grep -c 'REDACTED' "${OUT}" || true)"
+}
+redact bearer "request failed Authorization: Bearer abc123SECRETvalue456" "abc123SECRETvalue456"
+redact k3s_token "join failed token K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" "s3cr3tPassw0rd"
+redact jwt "bad token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.c2lnbmF0dXJlLXNlY3JldA" "eyJhbGciOiJIUzI1NiJ9"
+redact akia "credentials AKIAIOSFODNN7EXAMPLE rejected" "AKIAIOSFODNN7EXAMPLE"
+redact asia "credentials ASIAIOSFODNN7EXAMPLE rejected" "ASIAIOSFODNN7EXAMPLE"
+redact amz_signature "GET /obj?X-Amz-Signature=0123456789abcdef0123456789abcdef&X-Amz-Expires=60 failed" "0123456789abcdef0123456789abcdef"
+redact amz_credential "GET /obj?X-Amz-Credential=ZZZsecretscope%2F20261103%2Fap%2Fs3%2Faws4_request failed" "ZZZsecretscope"
+
+# --- truncation: text over 512 bytes is cut ---
+LONG=$(awk 'BEGIN { for (i = 0; i < 2000; i++) printf "a"; }')
+forward truncate <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="${LONG}"
+EOF
+check "truncate: still forwarded" "1" "$(forwarded)"
+BODY=$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1)
+check "truncate: body is exactly 512 bytes" "512" "$(printf '%s' "${BODY}" | wc -c | tr -d ' ')"
+check "truncate: keeps the start of the line" "time=\"2026-11-03T03:00:00Z\"" "$(printf '%s' "${BODY}" | cut -c1-27)"
+
+# --- redaction runs before the cut: a secret over the limit leaves no prefix ---
+# The pad puts the 512-byte cut right after the bare marker (AKIA, eyJ, K10).
+# A cut fragment that long matches no redaction pattern, so redacting after the
+# cut would leave it in the output.
+HEAD='time="2026-11-03T03:00:00Z" level=error msg="'
+straddle() { # straddle <name> <secret> <bytes of the secret kept by the cut>
+    PAD=$(awk -v n=$((512 - ${#HEAD} - 1 - $3)) 'BEGIN { for (i = 0; i < n; i++) printf "x"; }')
+    forward "straddle_$1" <<EOF
+${HEAD}${PAD} $2 tail"
+EOF
+    check "straddle(${1}): line forwarded" "1" "$(forwarded)"
+    check "straddle(${1}): cut keeps the marker length" "$3" "$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1 | awk -v p="${PAD} " '{ print length($0) - index($0, p) - length(p) + 1 }')"
+    if grep -qF "$(printf '%s' "$2" | cut -c1-"$3")" "${OUT}"; then
+        check "straddle(${1}): no fragment survives the cut" "absent" "present"
+    else
+        check "straddle(${1}): no fragment survives the cut" "absent" "absent"
+    fi
+}
+straddle jwt "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2ln" 3
+straddle akia "AKIAIOSFODNN7EXAMPLE" 4
+straddle k3s_token "K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" 3
+
+# --- rate cap: 60 a minute, then one summary of what was dropped ---
+gen() { # gen <count>
+    awk -v n="$1" 'BEGIN { for (i = 0; i < n; i++) printf "time=\"t\" level=error msg=\"loop %d\"\n", i }'
+}
+gen 100 > "${WORK}/burst"
+forward rate_cap < "${WORK}/burst"
+check "rate cap: 60 lines forwarded plus one summary" "61" "$(forwarded)"
+check "rate cap: one dropped summary" "1" "$(grep -c '40 lines dropped' "${OUT}" || true)"
+check "rate cap: first line kept" "1" "$(grep -c 'loop 0"' "${OUT}" || true)"
+check "rate cap: line past the cap is not forwarded" "0" "$(grep -c 'loop 60"' "${OUT}" || true)"
+
+# --- rate cap: a new minute starts a new window and reports the old drops ---
+# The clock command advances 60 s every 70 calls, so lines 0-69 share a window
+# (10 dropped) and lines 70-74 land in the next one.
+CASE="${WORK}/rate_window"
+mkdir -p "${CASE}"
+OUT="${CASE}/console"
+: > "${OUT}"
+echo 0 > "${CASE}/calls"
+cat > "${CASE}/clock" <<EOF
+#!/bin/sh
+n=\$(cat "${CASE}/calls")
+echo \$((n + 1)) > "${CASE}/calls"
+echo \$((1000 + n / 70 * 60))
+EOF
+chmod +x "${CASE}/clock"
+gen 75 | FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="${CASE}/clock" sh "${SCRIPT}" 2>/dev/null || true
+check "window: 60 + 5 forwarded plus one summary" "66" "$(forwarded)"
+check "window: summary names the 10 dropped" "1" "$(grep -c '10 lines dropped' "${OUT}" || true)"
+check "window: new window forwards its lines" "1" "$(grep -c 'loop 74"' "${OUT}" || true)"
+
+# --- blocked console: lines are dropped, the forwarder neither stalls nor fails ---
+CASE="${WORK}/blocked"
+mkdir -p "${CASE}"
+mkfifo "${CASE}/console"
+gen 3 > "${CASE}/in"
+(
+    FORWARD_CONSOLE="${CASE}/console" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="echo 1000" \
+        FORWARD_WRITE_TIMEOUT=1 sh "${SCRIPT}" < "${CASE}/in"
+) &
+PID=$!
+i=0
+while kill -0 "${PID}" 2>/dev/null && [ "${i}" -lt 20 ]; do
+    sleep 1
+    i=$((i + 1))
+done
+if kill -0 "${PID}" 2>/dev/null; then
+    kill "${PID}" 2>/dev/null || true
+    check "blocked console: forwarder finished" "finished" "stalled"
+else
+    check "blocked console: forwarder finished" "finished" "finished"
+fi
+rc=0
+wait "${PID}" || rc=$?
+check "blocked console: exit 0" "0" "${rc}"
+
+# --- the service is wired in, and nothing depends on it ---
+INITD="${SCRIPT_DIR}/mulga-eks-log-forward.initd"
+check "initd ships" "yes" "$([ -f "${INITD}" ] && echo yes || echo no)"
+check "manifest installs the initd" "1" "$(grep -c 'mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+check "manifest installs the script" "1" "$(grep -c 'mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+DEPS=0
+for f in "${SCRIPT_DIR}"/*.initd; do
+    [ "${f}" = "${INITD}" ] && continue
+    if grep -E '^[[:space:]]*(need|use|want)[[:space:]].*mulga-eks-log-forward' "${f}" >/dev/null 2>&1; then
+        DEPS=$((DEPS + 1))
+    fi
+done
+check "nothing depends on the forwarder" "0" "${DEPS}"
+if [ -f "${INITD}" ]; then
+    check "initd does not need anything it could block boot on" "0" "$(grep -cE '^[[:space:]]*need[[:space:]]' "${INITD}" || true)"
+fi
+
+echo "mulga-eks-log-forward_test: ${PASS} passed, ${FAIL} failed"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -45,10 +45,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
     /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
-    /etc/init.d/eks-pod-identity-webhook
+    /etc/init.d/eks-pod-identity-webhook /etc/init.d/mulga-eks-log-forward
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
     /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-node-gc /usr/local/sbin/mulga-eks-log-forward
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

spinifex/domains/acm/awsapi/registration.go

@@ -3,14 +3,11 @@ package awsapi
 import (
 	"context"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/nats-io/nats.go"
@@ -66,7 +63,7 @@ var Actions = map[string]Handler{
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -81,25 +78,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -118,7 +101,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ACM dispatch: failed to read body", "err", err)
 		return err
@@ -144,16 +127,3 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	WriteJSONResponse(w, output)
 	return nil
 }
-
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}

spinifex/domains/acm/awsapi/registration_test.go

@@ -58,18 +58,6 @@ func dispatchWith(t *testing.T, deps Deps, inv dispatch.Invocation) (*httptest.R
 	return w, NewRegistration(deps).Dispatch(w, inv)
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		"CertificateManager.ListCertificates": "ListCertificates",
-		"a.b.ListCertificates":                "ListCertificates",
-		"ListCertificates":                    "ListCertificates",
-		"CertificateManager.":                 "",
-		"":                                    "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // TestActionsScopeTableIsExhaustive is what stops the next ACM action being
 // added with a silent account-wide grant. It asserts both directions, so a
 // scope left behind by a deleted or renamed action fails too.

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -49,11 +49,13 @@ type PlacementGroupOwner interface {
 	CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*ec2placementgroup.AccountCleanup, error)
 }
 
-// LaunchTemplateOwner is the launch-template owner's account scan and residue
-// cleanup; live templates are never deleted by account cleanup.
+// LaunchTemplateOwner is the launch-template owner's account scan, residue
+// cleanup and live-template delete; a run deletes live templates only when its
+// request asks and the consumer check completed.
 type LaunchTemplateOwner interface {
 	ScanAccount(ctx context.Context, accountID string) (*ec2launchtemplate.AccountScan, error)
 	CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*ec2launchtemplate.AccountCleanup, error)
+	DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*ec2launchtemplate.AccountTemplateDeletes, error)
 }
 
 // TagIndexOwner lists and clears the account's tag-index entries.
@@ -153,6 +155,11 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	var keyRun *keyPairRun
 	var groupRun *placementGroupRun
 	var templateRun *launchTemplateRun
+	// Live templates go first when asked: a version that names a key pair or
+	// group blocks it, so the reference read must see what remains.
+	if req.DeleteLiveTemplates && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, true)
+	}
 	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
 		refs := c.readReferences(ctx, acct)
 		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
@@ -162,8 +169,8 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
 		}
 	}
-	if slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
-		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	if templateRun == nil && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, false)
 	}
 
 	// Tags are listed before the owner re-scans, so a resource created in

spinifex/domains/ec2/accountcleanup/cleanup_live_templates_test.go

@@ -0,0 +1,159 @@
+package accountcleanup
+
+//test:in-package — shares the owner fixtures and the injected clock.
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeConsumers answers what nodegroups name; a non-nil err is returned with
+// whatever partial answer it also holds.
+type fakeConsumers struct {
+	ids, names []string
+	err        error
+}
+
+func (f *fakeConsumers) AccountTemplateConsumers(context.Context, string) (*TemplateConsumers, error) {
+	out := &TemplateConsumers{TemplateIDs: map[string]bool{}, TemplateNames: map[string]bool{}}
+	markAll(out.TemplateIDs, f.ids)
+	markAll(out.TemplateNames, f.names)
+	return out, f.err
+}
+
+var _ TemplateConsumerReader = (*fakeConsumers)(nil)
+
+// liveDelete runs a cleanup that deletes live templates, with consumers as the
+// nodegroup reader (nil means none supplied).
+func (o *owners) liveDelete(t *testing.T, consumers TemplateConsumerReader) *Result {
+	t.Helper()
+	readers := o.readers()
+	readers.Consumers = consumers
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, CorrelationRef: "ref-1", DeleteLiveTemplates: true})
+	require.NoError(t, err)
+	return res
+}
+
+func (o *owners) templateNamingKeyAndGroup(t *testing.T, name, key, group string) string {
+	t.Helper()
+	ltID := o.createTemplate(t, testAccount, name)
+	_, err := o.templates.CreateLaunchTemplateVersion(t.Context(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(ltID),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			KeyName:   aws.String(key),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String(group)},
+		},
+	}, testAccount)
+	require.NoError(t, err)
+	return ltID
+}
+
+func (o *owners) liveTemplateIDs(t *testing.T) []string {
+	t.Helper()
+	scan, err := o.templates.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	return scan.LiveTemplateIDs
+}
+
+// A template no nodegroup names is deleted in the same run as the key pair and
+// group its version names, which the reference check no longer sees.
+func TestCleanupPlatformDeletesAnUnnamedTemplateAndWhatItNames(t *testing.T) {
+	o := newOwners(t)
+	key := o.importKey(t, testAccount, "tkey")
+	group := o.createGroup(t, testAccount, "tgroup")
+	lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+
+	res := o.liveDelete(t, &fakeConsumers{})
+	assert.ElementsMatch(t, []Item{
+		item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+		item(key, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(group, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, ""),
+	}, res.Items)
+	assert.Empty(t, o.liveTemplateIDs(t))
+}
+
+// A template a nodegroup names, by ID or by name, is kept and blocked, and so
+// are the key pair and group its versions name.
+func TestCleanupPlatformKeepsATemplateANodegroupNames(t *testing.T) {
+	for name, consumers := range map[string]*fakeConsumers{
+		"by id":   {},
+		"by name": {names: []string{"web"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			group := o.createGroup(t, testAccount, "tgroup")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+			other := o.createTemplate(t, testAccount, "free")
+			if name == "by id" {
+				consumers.ids = []string{lt}
+			}
+
+			res := o.liveDelete(t, consumers)
+			assert.ElementsMatch(t, []Item{
+				item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(other, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+				item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(group, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonLaunchTemplateReference),
+			}, res.Items)
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A consumer read that fails, returns a partial answer with an error, or is not
+// supplied deletes no template, key pair or group, and the kind is incomplete.
+func TestCleanupPlatformUnreadableConsumersDeleteNoTemplate(t *testing.T) {
+	cases := map[string]TemplateConsumerReader{
+		"error":        &fakeConsumers{err: errInjected},
+		"partial":      &fakeConsumers{ids: []string{"lt-0other"}, err: errInjected},
+		"not supplied": nil,
+	}
+	for name, consumers := range cases {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "none")
+
+			res := o.liveDelete(t, consumers)
+			assert.Contains(t, res.Items, item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceCheckFailed))
+			assert.Contains(t, res.Items, item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference))
+			assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A template only a DELETING nodegroup still names is as blocked as any other:
+// the reader reports every status, so the cleanup never sees a status to trust.
+func TestCleanupPlatformKeepsATemplateADeletingNodegroupStillNames(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "draining")
+
+	res := o.liveDelete(t, &fakeConsumers{ids: []string{lt}})
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}
+
+// Without the request flag a supplied consumer reader changes nothing: live
+// templates are never deleted, as the exercise relies on.
+func TestCleanupPlatformLiveTemplatesSurviveWithoutTheRequestFlag(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "web")
+	readers := o.readers()
+	readers.Consumers = &fakeConsumers{}
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+
+	res := run(t, c)
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -204,13 +204,18 @@ type launchTemplateRun struct {
 	before  *ec2launchtemplate.AccountScan
 	cleaned *ec2launchtemplate.AccountCleanup
 	did     attempts
+	// deleteLive is true when this run tried to delete live templates.
+	deleteLive bool
 }
 
-func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time) *launchTemplateRun {
-	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts()}
+func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time, deleteLive bool) *launchTemplateRun {
+	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts(), deleteLive: deleteLive}
 	if before, err := c.templates.ScanAccount(ctx, acct); err == nil {
 		run.before = before
 	}
+	if deleteLive {
+		c.deleteLiveTemplates(ctx, acct, run)
+	}
 	cleaned, err := c.templates.CleanupAccountResidue(ctx, acct, settledBefore)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: launch-template residue cleanup failed", "err", err)
@@ -218,10 +223,50 @@ func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settled
 		return run
 	}
 	run.cleaned = cleaned
-	keepAll(run.did.kept, cleaned.KeptTemplateIDs, ReasonReferenceUnknown)
+	for _, id := range cleaned.KeptTemplateIDs {
+		if run.did.kept[id] == "" {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
+	}
 	return run
 }
 
+// deleteLiveTemplates deletes the live templates no consumer names. Without a
+// complete consumer answer nothing is deleted and every live template is kept
+// as unchecked, so an unreadable consumer never reads as "no references".
+func (c *Cleanup) deleteLiveTemplates(ctx context.Context, acct string, run *launchTemplateRun) {
+	var consumers *TemplateConsumers
+	if c.refs.Consumers != nil {
+		var err error
+		consumers, err = c.refs.Consumers.AccountTemplateConsumers(ctx, acct)
+		if err != nil {
+			slog.WarnContext(ctx, "Platform cleanup: template consumer check failed", "err", err)
+			consumers = nil
+		}
+	}
+	if consumers == nil {
+		if run.before != nil {
+			keepAll(run.did.kept, run.before.LiveTemplateIDs, ReasonReferenceCheckFailed)
+		}
+		run.kind.incomplete(ReasonReferenceCheckFailed)
+		return
+	}
+	deletes, err := c.templates.DeleteLiveTemplates(ctx, acct, func(id, name string) string {
+		if consumers.names(id, name) {
+			return ReasonLaunchTemplateReference
+		}
+		return ""
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: live template delete failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return
+	}
+	markAll(run.did.deleted, deletes.DeletedTemplateIDs)
+	markAll(run.did.failed, deletes.FailedTemplateIDs)
+	maps.Copy(run.did.kept, deletes.Referenced)
+}
+
 func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *launchTemplateRun, res *Result) ownedIDs {
 	defer c.finishKind(run.kind)
 	after, err := c.templates.ScanAccount(ctx, acct)
@@ -250,8 +295,10 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 	}
 	// Live templates are never deleted here, so every one still present is
 	// blocked on the unknown EKS nodegroup reference, whenever it appeared.
-	for _, id := range afterLive {
-		run.did.kept[id] = ReasonReferenceUnknown
+	if !run.deleteLive {
+		for _, id := range afterLive {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
 	}
 	for _, id := range sortedUnion(beforeLive, kept, afterLive) {
 		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)

spinifex/domains/ec2/accountcleanup/references.go

@@ -35,17 +35,38 @@ type TemplateReferenceReader interface {
 	AccountTemplateReferences(ctx context.Context, accountID string) (*ec2launchtemplate.TemplateReferences, error)
 }
 
+// TemplateConsumers are the IDs and names of launch templates that records
+// outside EC2, such as EKS nodegroups, still name. They select what to keep and
+// are never reported.
+type TemplateConsumers struct {
+	TemplateIDs   map[string]bool
+	TemplateNames map[string]bool
+}
+
+// TemplateConsumerReader reads which launch templates records outside EC2 name,
+// every status included, complete or an error. The consumer's owner answers it;
+// EC2 never reads those records.
+type TemplateConsumerReader interface {
+	AccountTemplateConsumers(ctx context.Context, accountID string) (*TemplateConsumers, error)
+}
+
+func (c *TemplateConsumers) names(id, name string) bool {
+	return c.TemplateIDs[id] || c.TemplateNames[name]
+}
+
 var (
 	_ SpotReferenceReader     = (*ec2spotinstance.SpotInstanceServiceImpl)(nil)
 	_ TemplateReferenceReader = (*ec2launchtemplate.LaunchTemplateServiceImpl)(nil)
 )
 
 // ReferenceReaders are the checks run before key pairs and placement groups
-// are deleted. A nil reader counts as a failed check.
+// are deleted. A nil reader counts as a failed check. Consumers is read only
+// by a request that deletes live templates, where nil is also a failed check.
 type ReferenceReaders struct {
 	Instances InstanceReferenceReader
 	Spot      SpotReferenceReader
 	Templates TemplateReferenceReader
+	Consumers TemplateConsumerReader
 }
 
 func newReferences() *References {

spinifex/domains/ec2/accountcleanup/types.go

@@ -10,6 +10,11 @@ type Request struct {
 
 	// Kinds limits the run to these platform kinds. Empty means all three.
 	Kinds []string `json:"kinds,omitempty"`
+
+	// DeleteLiveTemplates lets the run delete live launch templates that no
+	// nodegroup names, using the TemplateConsumerReader. Without it live
+	// templates are never deleted.
+	DeleteLiveTemplates bool `json:"delete_live_templates,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1
@@ -88,7 +93,8 @@ const (
 	ReasonInstanceReference = "instance-reference"
 	// ReasonSpotRequestReference: an open or active spot request names it.
 	ReasonSpotRequestReference = "spot-request-reference"
-	// ReasonLaunchTemplateReference: a version of a live launch template names it.
+	// ReasonLaunchTemplateReference: a version of a live launch template names a
+	// key pair or group, or a nodegroup names a template.
 	ReasonLaunchTemplateReference = "launch-template-reference"
 	// ReasonReferenceUnknown: a record EC2 cannot read may name it, so it is
 	// never deleted by account cleanup.

spinifex/domains/ec2/awsapi/registration.go

@@ -1,8 +1,6 @@
 package awsapi
 
 import (
-	"net/http"
-
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 )
 
@@ -11,7 +9,7 @@ import (
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveQueryAction,
 		Dispatch:      NewHandler(deps).Dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
 		Traits:        dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeEC2},
@@ -24,13 +22,3 @@ func NewRegistration(deps Deps) dispatch.Registration {
 func OperationInventory() dispatch.Inventory {
 	return dispatch.Inventory{Registered: ActionNames()}
 }
-
-// resolveAction is the one Query Action parser telemetry and dispatch share.
-// It leaves the body readable for the dispatcher and answers "" on any failure.
-func resolveAction(r *http.Request) string {
-	args, err := readQueryArgs(r)
-	if err != nil {
-		return ""
-	}
-	return args["Action"]
-}

spinifex/domains/ec2/awsapi/registration_test.go

@@ -1,11 +1,6 @@
-//test:in-package — drives the unexported resolveAction with the registration.
-
 package awsapi
 
 import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -27,29 +22,3 @@ func TestNewRegistration_DeclaresTheEC2Service(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(reg))
 }
-
-func TestResolveAction(t *testing.T) {
-	post := func(body string) *http.Request {
-		r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
-		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		return r
-	}
-
-	assert.Equal(t, "DescribeVpcs", resolveAction(post("Action=DescribeVpcs&Version=2016-11-15")))
-	assert.Empty(t, resolveAction(post("Version=2016-11-15")))
-
-	// The body stays readable for the dispatcher after the action is resolved.
-	r := post("Action=DescribeVpcs")
-	require.Equal(t, "DescribeVpcs", resolveAction(r))
-	assert.Equal(t, "DescribeVpcs", resolveAction(r))
-}
-
-func TestResolveAction_UnreadableBody(t *testing.T) {
-	r := httptest.NewRequest(http.MethodPost, "/", errReader{})
-	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	assert.Empty(t, resolveAction(r))
-}
-
-type errReader struct{}
-
-func (errReader) Read([]byte) (int, error) { return 0, http.ErrBodyReadAfterClose }

spinifex/domains/ec2/awsapi/request.go

@@ -1,9 +1,7 @@
 package awsapi
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
@@ -12,7 +10,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -26,25 +23,11 @@ const (
 	ec2ServicePrincipal = "ec2.amazonaws.com"
 )
 
-// readQueryArgs parses the Query body and rewinds it, so the same bytes can be
-// read again. Any read or parse failure is a malformed query string.
-func readQueryArgs(r *http.Request) (map[string]string, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, err
-	}
-	r.Body = io.NopCloser(bytes.NewReader(body))
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return query.ParseArgs(string(body))
-}
-
 // Dispatch parses, authorizes and runs one EC2 request, then writes the XML
 // response. A returned error is an awserrors code the gateway renders.
 func (h *Handler) Dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	queryArgs, err := readQueryArgs(r)
+	queryArgs, err := dispatch.ReadArgs(r)
 	if err != nil {
 		slog.Debug("EC2: malformed query string", "err", err)
 		return errors.New(awserrors.ErrorMalformedQueryString)

spinifex/domains/ec2/awsapi/volume/AttachVolume.go

@@ -49,6 +49,12 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 		device = *input.Device
 	}
 
+	// The shared record decides: a stopped instance has no QMP to plug into, so
+	// its record is edited and the volume claimed there instead.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return attachToStopped(ctx, natsConn, accountID, input)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -97,6 +103,24 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 	return output, nil
 }
 
+// attachToStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const attachToStoppedSubject = "ec2.AttachVolumeToStopped"
+
+// attachToStopped asks the instance service to add the volume to the stopped
+// instance's record; the volume is claimed in-use by the same call.
+func attachToStopped(ctx context.Context, natsConn *nats.Conn, accountID string, input *ec2.AttachVolumeInput) (ec2.VolumeAttachment, error) {
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, attachToStoppedSubject, input, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolume: stopped-instance attach failed", "volumeId", aws.StringValue(input.VolumeId), "instanceId", aws.StringValue(input.InstanceId), "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}
+
 // isStoppedInstance checks the shared KV (via ec2.DescribeStoppedInstances) to
 // determine whether instanceID exists as a stopped instance.
 func isStoppedInstance(ctx context.Context, instanceID string, natsConn *nats.Conn, accountID string) bool {

spinifex/domains/ec2/awsapi/volume/AttachVolume_stopped_test.go

@@ -0,0 +1,55 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The record decides the path: a stopped instance has no QMP to plug into, so
+// the attach goes to the instance service and ec2.cmd.<id> is never used.
+func TestAttachVolume_StoppedInstanceUsesTheRecordPath(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		attachToStoppedSubject: func(m *nats.Msg) {
+			log.add(attachToStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"),
+				Device: aws.String("/dev/sdf"), State: aws.String("attached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := AttachVolume(context.Background(), &ec2.AttachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+		Device:     aws.String("/dev/sdf"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "attached", aws.StringValue(out.State))
+	assert.Equal(t, []string{attachToStoppedSubject}, log.get(),
+		"only the record path runs for a stopped instance")
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume.go

@@ -74,6 +74,12 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 
 	force := input.Force != nil && *input.Force
 
+	// The shared record decides: a stopped instance has no QMP to unplug from,
+	// so its record is edited first and the volume released after.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return detachFromStopped(ctx, natsConn, accountID, volumeID, instanceID, device)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -131,6 +137,10 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 // running the instance and so cannot help when that host is the problem.
 const forceDetachSubject = "ec2.ForceDetachVolume"
 
+// detachFromStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const detachFromStoppedSubject = "ec2.DetachVolumeFromStopped"
+
 // forceDetach releases a volume whose instance has no responder.
 //
 // A volume outlives the instance it was attached to whenever teardown failed
@@ -153,3 +163,27 @@ func forceDetach(ctx context.Context, natsConn *nats.Conn, accountID, volumeID,
 	}
 	return *attachment, nil
 }
+
+// detachFromStopped removes the volume from the stopped instance's record and
+// then releases the volume, in that order, so a failed release leaves the
+// record clean and the volume still held rather than the reverse.
+func detachFromStopped(ctx context.Context, natsConn *nats.Conn, accountID, volumeID, instanceID, device string) (ec2.VolumeAttachment, error) {
+	in := &ec2.DetachVolumeInput{VolumeId: &volumeID, InstanceId: &instanceID}
+	if device != "" {
+		in.Device = &device
+	}
+	if _, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, detachFromStoppedSubject, in, 30*time.Second, accountID); err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: stopped-instance record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, forceDetachSubject,
+		&ec2.DetachVolumeInput{VolumeId: &volumeID, Force: aws.Bool(true)}, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: release after record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume_stopped_test.go

@@ -0,0 +1,115 @@
+package volume
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type subjectLog struct {
+	mu   sync.Mutex
+	seen []string
+}
+
+func (l *subjectLog) add(s string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.seen = append(l.seen, s)
+}
+
+func (l *subjectLog) get() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.seen...)
+}
+
+func respondJSON(t *testing.T, msg *nats.Msg, v any) {
+	t.Helper()
+	data, err := json.Marshal(v)
+	require.NoError(t, err)
+	require.NoError(t, msg.Respond(data))
+}
+
+// The record decides the path. A stopped instance has no QMP to unplug from, so
+// its record is edited and the volume released, and ec2.cmd.<id> is never used.
+func TestDetachVolume_StoppedInstanceEditsTheRecordThenReleasesTheVolume(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		detachFromStoppedSubject: func(m *nats.Msg) {
+			log.add(detachFromStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), Device: aws.String("/dev/sdf"),
+			})
+		},
+		forceDetachSubject: func(m *nats.Msg) {
+			log.add(forceDetachSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), State: aws.String("detached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "detached", aws.StringValue(out.State))
+	assert.Equal(t, []string{detachFromStoppedSubject, forceDetachSubject}, log.get(),
+		"record first, then the volume; the host path is never used for a stopped instance")
+}
+
+// NoResponders alone must not be read as "stopped": a dead or partitioned node
+// gives the same answer, and editing the record then would race a live QEMU.
+func TestDetachVolume_NoRespondersAloneDoesNotMeanStopped(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	for _, subject := range []string{detachFromStoppedSubject, forceDetachSubject} {
+		sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
+			log.add(subject)
+			_ = m.Respond(nil)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	sub, err := nc.Subscribe("ec2.DescribeStoppedInstances", func(m *nats.Msg) {
+		respondJSON(t, m, &ec2.DescribeInstancesOutput{})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	_, err = DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-silent-node"),
+	}, nc, "000000000001")
+
+	require.Error(t, err)
+	assert.Empty(t, log.get(), "no record edit and no control-plane release without a stopped record")
+}

spinifex/domains/ec2/image/drain_test.go

@@ -1,140 +0,0 @@
-package image
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const drainImageInstanceID = "i-image-drain-host"
-
-// setupDrainImageService mirrors domains/ec2/snapshot's setupDrainService: a
-// reachable (but rejecting) Predastore host so a drained volume falls through
-// into the real snapshot path and fails fast, a data dir with no local drain
-// socket, and a NATS connection to route the drain to the hosting node.
-func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
-	t.Helper()
-
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
-	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
-	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: testBucket,
-		},
-	}
-	svc := NewImageServiceImplWithConfig(cfg, store, nc)
-	return svc, store, nc
-}
-
-func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, testBucket, volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-func drainImageResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-func drainedImageAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-func awaitImageDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// CreateImageFromInstance's IsRunning path must drain the volume's hosting node
-// before it reads the live checkpoint — the same guarantee ec2.CreateSnapshot
-// has. Without this, a checkpoint predating a guest-triggered GPT rewrite
-// (e.g. growpart on first boot) can be captured silently.
-func TestSnapshotRunningVolume_DrainsAttachedVolume(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-drain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-drain", "in-use", drainImageInstanceID)
-	got := drainImageResponder(t, nc, drainImageInstanceID, drainedImageAck(t, "vol-image-drain"))
-
-	err := svc.snapshotRunningVolume("vol-image-drain", "snap-image-drain", testAccountID)
-	// The rejecting Predastore stub fails the snapshot itself; what matters
-	// here is that the drain reached the hosting node first.
-	require.Error(t, err)
-
-	command := awaitImageDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainImageInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-image-drain", command.DrainVolumeData.VolumeID)
-}
-
-// An attached volume whose host reports the drain failed must fail
-// CreateImage rather than register an AMI built from a stale checkpoint.
-func TestSnapshotRunningVolume_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-nodrain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-nodrain", "in-use", drainImageInstanceID)
-	drainImageResponder(t, nc, drainImageInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.snapshotRunningVolume("vol-image-nodrain", "snap-image-nodrain", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// An unattached (stopped-instance) volume takes the no-drain path: the
-// checkpoint Close() left behind is already current.
-func TestSnapshotRunningVolume_AvailableSkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-available", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-available", "available", "")
-	svc.natsConn = nil
-
-	// No responder exists and the socket is never dialled: proceeding straight
-	// to the (rejecting) snapshot path proves the drain was skipped, not stuck.
-	err := svc.snapshotRunningVolume("vol-image-available", "snap-image-available", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -2,16 +2,20 @@ package image
 
 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -96,15 +100,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a
-// stopped-volume snapshot under a provider is created via provider.CreateSnapshot
+// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
+// root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -118,22 +122,22 @@ func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID)
+	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -222,3 +226,98 @@ func TestCreateImageFromInstance_SetsEnaSupportFromInstance(t *testing.T) {
 	require.NotNil(t, ami.EnaSupport)
 	assert.True(t, *ami.EnaSupport)
 }
+
+// createUnavailableProvider refuses every CreateSnapshot as retry-later.
+type createUnavailableProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
+}
+
+// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
+// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
+func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
+	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+}
+
+// createInUseProvider refuses every CreateSnapshot as volume in use.
+type createInUseProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
+}
+
+// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
+// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
+	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, doc.VolumeID)
+	assert.Contains(t, message, "Detach")
+}
+
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// An image of a running instance is snapshotted by the provider, which drains
+// a buffered volume itself, so CreateImage neither dials the drain socket nor
+// asks the instance's host to drain.
+func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *testing.T) {
+	const instanceID = "i-running-drain"
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	_, nc := testutil.StartTestNATS(t)
+	svc.natsConn = nc
+	svc.config.DataDir = testutil.SocketTempDir(t)
+	counting := &countingSnapshotProvider{EBSProvider: provider}
+	svc.SetEBSProvider(counting)
+
+	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
+	doc.State, doc.AttachedInstance = "in-use", instanceID
+	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, doc.VolumeID, "OK\n")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(instanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(instanceID), Name: aws.String("running-ami")},
+		RootVolumeID: doc.VolumeID,
+		IsRunning:    true,
+	}, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, out.ImageId)
+
+	assert.Equal(t, int32(1), counting.creates.Load(), "the provider must take the snapshot")
+	assert.Zero(t, drains(), "CreateImage must not dial the drain socket")
+	assert.Zero(t, commands.Load(), "CreateImage must not ask the instance's host to drain")
+}

spinifex/domains/ec2/image/service_impl.go

@@ -38,7 +38,7 @@ type CreateImageParams struct {
 	Input         *ec2.CreateImageInput
 	RootVolumeID  string
 	SourceImageID string
-	IsRunning     bool // true = use live checkpoint (instance still running), false = use numbered checkpoint from Close
+	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -69,9 +69,7 @@ func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store {
 	return s.metadata
 }
 
-// NewImageServiceImpl creates a new daemon-side image service. natsConn is used
-// to drain a running instance's volume (routed to the node hosting it) before
-// CreateImageFromInstance reads its live checkpoint.
+// NewImageServiceImpl creates a new daemon-side image service.
 func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceImpl {
 	store := objectstore.NewS3ObjectStoreFromConfig(
 		cfg.Predastore.Host,
@@ -99,8 +97,7 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
 }
 
 // NewImageServiceImplWithConfig creates an image service with an explicit
-// config and NATS connection (for testing the running-volume drain path,
-// which needs config.DataDir/Predastore and a NATS connection to route to).
+// config, object store and NATS connection (for testing).
 func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn) *ImageServiceImpl {
 	return &ImageServiceImpl{
 		config:     cfg,
@@ -447,15 +444,9 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume (live via NATS or offline from S3)
-	var snapshotErr error
-	if params.IsRunning {
-		snapshotErr = s.snapshotRunningVolume(params.RootVolumeID, snapshotID, accountID)
-	} else {
-		snapshotErr = s.snapshotStoppedVolume(params.RootVolumeID, snapshotID, accountID)
-	}
-	if snapshotErr != nil {
-		return nil, snapshotErr
+	// Step 1: Snapshot root volume
+	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
+		return nil, err
 	}
 
 	// Step 2: Read source volume config for size
@@ -521,51 +512,21 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}, nil
 }
 
-// snapshotRunningVolume creates a snapshot of a running instance's volume by
-// draining it to the node hosting it (so any writes buffered there reach S3),
-// then reading the live checkpoint written by nbdkit on every NBD Flush. This
-// shares domains/ec2/snapshot's DrainVolume with ec2.CreateSnapshot so the two
-// live-snapshot paths cannot diverge on when a drain is required.
-func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID string) error {
+// snapshotVolume snapshots the root volume through the EBS provider, whether
+// or not the instance is running. A provider that buffers writes drains the
+// volume inside its own CreateSnapshot, so no drain is issued here.
+func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
 	if s.config == nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
 	if err != nil {
-		slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Flush the writes still buffered by whichever node serves the volume, so
-	// the snapshot below is current rather than an arbitrarily stale one
-	// predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An
-	// attached volume that cannot be drained fails here rather than silently
-	// producing an image from a stale checkpoint.
-	if err := ec2snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil {
-		slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume")
-}
-
-// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the
-// volume is stopped/detached, so nothing is buffered on a serving node.
-// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening
-// a second one, so it handles the mounted case itself.
-func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume")
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
 }
 
 // createSnapshot delegates snapshot creation for volMeta to the EBS provider.
@@ -580,7 +541,7 @@ func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetada
 	})
 	if err != nil || created == nil {
 		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
+		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
 	}
 	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
 	return nil

spinifex/domains/ec2/image/service_impl_test.go

@@ -2551,56 +2551,19 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
 
-func TestSnapshotRunningVolume_VolumeConfigMissing(t *testing.T) {
+func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := &ImageServiceImpl{
 		config:     &config.Config{},
 		store:      store,
 		bucketName: testBucket,
 	}
-	err := svc.snapshotRunningVolume("vol-missing", "snap-r1", testAccountID)
+	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotRunningVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running1", 10)
-
-	err := svc.snapshotRunningVolume("vol-running1", "snap-r2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotStoppedVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
+func TestSnapshotVolume_BackendInitFails(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 	}))
@@ -2620,7 +2583,7 @@ func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped1", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped1", "snap-s2", testAccountID)
+	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -2646,60 +2609,7 @@ func imageS3MockServer(t *testing.T) *httptest.Server {
 	}))
 }
 
-func TestSnapshotRunningVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-run-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-enckey", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-enckey", "snap-run-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotRunningVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-lsf", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-lsf", "snap-run-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
+func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -2722,12 +2632,12 @@ func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
+func TestSnapshotVolume_LoadStateFails(t *testing.T) {
 	srv := imageS3MockServer(t)
 	defer srv.Close()
 
@@ -2747,7 +2657,7 @@ func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }

spinifex/domains/ec2/instance/service_impl.go

@@ -2700,6 +2700,12 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 	// failure of this launch.
 	instance.FirstLaunch = false
 
+	if err := s.reconcileStartVolumes(ctx, instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: volumes cannot be reconciled", "instanceId", instance.ID, "err", err)
+		undo()
+		return err
+	}
+
 	instanceType, ok := s.resourceMgr.InstanceTypes()[instance.InstanceType]
 	if !ok {
 		slog.ErrorContext(ctx, "launchClaimedInstance: instance type not available on this node",
@@ -3674,3 +3680,153 @@ func (s *InstanceServiceImpl) DescribeInstanceStatus(ctx context.Context, input
 	slog.InfoContext(ctx, "DescribeInstanceStatus completed", "count", len(statuses))
 	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: statuses}, nil
 }
+
+// DetachVolumeFromStopped removes a data volume from a stopped instance's
+// record, so a later start no longer attaches it. It does not touch the
+// volume's own state; the caller releases that once the record is clean. The
+// device the volume held comes back on the attachment.
+func (s *InstanceServiceImpl) DetachVolumeFromStopped(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: stopped store not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	req, ok := findVolumeRequest(stored, volumeID)
+	if !ok {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+	if req.EFI {
+		return nil, errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+
+	var removed bool
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		removed = removeVolumeFromRecord(v, volumeID)
+	})
+	if err != nil {
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "DetachVolumeFromStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if !removed {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(req.DeviceName),
+		State:      aws.String(ec2.VolumeAttachmentStateDetaching),
+	}, nil
+}
+
+// AttachVolumeToStopped adds an available volume to a stopped instance's
+// record, so the next start mounts it. The volume is claimed in-use first and
+// released again if the record edit fails, so a failure never leaves the record
+// naming a volume another instance could take.
+func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *ec2.AttachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil || s.metadata == nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: stores not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to read volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if meta.TenantID != accountID {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+	}
+	if meta.State != "available" {
+		return nil, errors.New(awserrors.ErrorVolumeInUse)
+	}
+	if meta.AvailabilityZone != "" && s.config != nil && s.config.AZ != "" && meta.AvailabilityZone != s.config.AZ {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeZoneMismatch)
+	}
+
+	device, boot, err := stoppedAttachDevice(stored, aws.StringValue(input.Device))
+	if err != nil {
+		return nil, err
+	}
+
+	meta.State = "in-use"
+	meta.AttachedInstance = instanceID
+	meta.DeviceName = device
+	meta.AttachedAt = time.Now()
+	if err := s.metadata.PutVolume(ctx, meta); err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to claim volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	var conflict error
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		// The device was chosen against a snapshot, so it is checked again
+		// against the record the edit is applied to.
+		if _, _, cerr := stoppedAttachDevice(v, device); cerr != nil {
+			conflict = cerr
+			return
+		}
+		addVolumeToRecord(v, volumeID, device, boot)
+	})
+	if err == nil && conflict != nil {
+		err = conflict
+	}
+	if err != nil {
+		meta.State = "available"
+		meta.AttachedInstance = ""
+		meta.DeviceName = ""
+		meta.AttachedAt = time.Time{}
+		if rerr := s.metadata.PutVolume(ctx, meta); rerr != nil {
+			slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to release volume after a failed record edit", "volumeId", volumeID, "err", rerr)
+		}
+		if conflict != nil {
+			return nil, conflict
+		}
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "AttachVolumeToStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(device),
+		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
+	}, nil
+}

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -0,0 +1,54 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountRecorder notes which volumes a launch asked to mount and then fails the
+// launch, which is the earliest point that shows what the start would attach.
+type mountRecorder struct {
+	refusingMounter
+
+	mu      sync.Mutex
+	mounted []string
+}
+
+func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, req := range v.EBSRequests.Requests {
+		r.mounted = append(r.mounted, req.Name)
+	}
+	return errors.New("stop after recording")
+}
+
+func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+	return nil
+}
+
+// A volume detached while the instance was stopped must not come back when the
+// instance is started: the start path reads the same record the detach edited.
+func TestStartStoppedInstance_DoesNotReattachAVolumeDetachedWhileStopped(t *testing.T) {
+	id := "i-detach-then-start"
+	stopped := stoppedWithVolumes("acc", id)
+	stopped.InstanceType = "t3.micro"
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), "acc")
+	require.NoError(t, err)
+	require.Error(t, f.start(t, id), "the recorder fails the launch after the mount")
+
+	assert.Equal(t, []string{"vol-root", "vol-other"}, rec.mounted,
+		"the detached volume is not part of the start")
+}

spinifex/domains/ec2/instance/start_volumes.go

@@ -0,0 +1,86 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// reconcileStartVolumes drops the data volumes a claimed instance can no longer
+// have, and fails the start when its boot volume is gone.
+func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
+	if s.metadata == nil {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	requests := slices.Clone(instance.EBSRequests.Requests)
+	instance.EBSRequests.Mu.Unlock()
+
+	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+		return noRootVolumeError(instance)
+	}
+
+	var drop []string
+	for _, req := range requests {
+		meta, err := s.metadata.GetVolume(ctx, instance.AccountID, req.Name)
+		missing := err != nil && objectstore.IsNoSuchKeyError(err)
+		if err != nil && !missing {
+			return fmt.Errorf("read volume %s: %w", req.Name, err)
+		}
+		if req.Boot {
+			if missing {
+				slog.ErrorContext(ctx, "start: root volume no longer exists", "instanceId", instance.ID, "volumeId", req.Name)
+				return noRootVolumeError(instance)
+			}
+			continue
+		}
+		if req.EFI {
+			continue
+		}
+		if missing {
+			slog.WarnContext(ctx, "start: dropping deleted data volume", "instanceId", instance.ID, "volumeId", req.Name)
+			drop = append(drop, req.Name)
+			continue
+		}
+		if meta.AttachedInstance != "" && meta.AttachedInstance != instance.ID {
+			slog.WarnContext(ctx, "start: dropping data volume attached to another instance",
+				"instanceId", instance.ID, "volumeId", req.Name, "attachedTo", meta.AttachedInstance)
+			drop = append(drop, req.Name)
+		}
+	}
+	if len(drop) == 0 {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+		return slices.Contains(drop, r.Name)
+	})
+	instance.EBSRequests.Mu.Unlock()
+	if instance.Instance != nil {
+		instance.Instance.BlockDeviceMappings = slices.DeleteFunc(instance.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && slices.Contains(drop, aws.StringValue(m.Ebs.VolumeId))
+		})
+	}
+	return nil
+}
+
+// noRootVolumeError is what AWS answers when an instance is started with
+// nothing attached at its root device name.
+func noRootVolumeError(instance *vm.VM) error {
+	device := ebsmetadata.RootDeviceName
+	if instance.Instance != nil && aws.StringValue(instance.Instance.RootDeviceName) != "" {
+		device = aws.StringValue(instance.Instance.RootDeviceName)
+	}
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Instance does not have a volume attached at root (%s)", device)
+}

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -0,0 +1,125 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const startVolumesAccount = "000000000011"
+
+func startVolumesService(t *testing.T, volumes ...ebsmetadata.Volume) *InstanceServiceImpl {
+	t.Helper()
+	store := objectstore.NewMemoryObjectStore()
+	meta := ebsmetadata.NewStore(store, testRootBucket)
+	for _, v := range volumes {
+		v.TenantID = startVolumesAccount
+		require.NoError(t, meta.PutVolume(context.Background(), v))
+	}
+	return &InstanceServiceImpl{metadata: meta}
+}
+
+func startingInstance(id string, names ...string) *vm.VM {
+	v := &vm.VM{ID: id, AccountID: startVolumesAccount, Status: vm.StateStopped, Instance: &ec2.Instance{InstanceId: aws.String(id)}}
+	for i, n := range names {
+		boot := i == 0
+		device := "/dev/sd" + string(rune('f'+i))
+		if boot {
+			device = "/dev/sda1"
+		}
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
+			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
+		})
+	}
+	return v
+}
+
+func requestedNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+func mappedNames(v *vm.VM) []string {
+	var out []string
+	for _, m := range v.Instance.BlockDeviceMappings {
+		out = append(out, aws.StringValue(m.Ebs.VolumeId))
+	}
+	return out
+}
+
+// An instance stopped on an older build had its data volumes released, and they
+// may since have been attached elsewhere or deleted; starting must not take a
+// volume that is no longer its own.
+func TestReconcileStartVolumes_DropsVolumesThatAreNoLongerTheInstances(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-elsewhere", State: "in-use", AttachedInstance: "i-someone-else"},
+		ebsmetadata.Volume{VolumeID: "vol-free", State: "available"},
+		ebsmetadata.Volume{VolumeID: "vol-mine", State: "in-use", AttachedInstance: "i-old"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-elsewhere", "vol-gone", "vol-free", "vol-mine")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	want := []string{"vol-root", "vol-free", "vol-mine"}
+	assert.Equal(t, want, requestedNames(inst),
+		"attached-elsewhere and deleted volumes are dropped; free and own ones stay")
+	assert.Equal(t, want, mappedNames(inst), "the API view stops listing them too")
+}
+
+func TestReconcileStartVolumes_MissingBootVolumeFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "available"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+}
+
+// A root volume detached while the instance was stopped leaves no boot request
+// in the record; AWS refuses the start with the same answer.
+func TestReconcileStartVolumes_NoRootVolumeAttachedFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: "i-old"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	require.True(t, removeVolumeFromRecord(inst, "vol-root"))
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+	assert.Equal(t, []string{"vol-data"}, requestedNames(inst), "a refused start changes nothing")
+}
+
+// An instance stopped on an older build had its data volumes released to
+// available with no attachment; those are still the instance's own and a
+// start must bring them back.
+func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-released", State: "available"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-released")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
+	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
+}

spinifex/domains/ec2/instance/stopped_attach_test.go

@@ -0,0 +1,180 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const stoppedAttachAZ = "ap-southeast-2a"
+
+type stoppedAttachFixture struct {
+	svc    *InstanceServiceImpl
+	stored *vm.VM
+	id     string
+}
+
+func newStoppedAttachFixture(t *testing.T, volumes ...ebsmetadata.Volume) *stoppedAttachFixture {
+	t.Helper()
+	id := "i-stopped-attach"
+	svc := startVolumesService(t, volumes...)
+	stored := startingInstance(id, "vol-root", "vol-data")
+	svc.config = &config.Config{AZ: stoppedAttachAZ}
+	svc.vmMgr = mgrWith(map[string]*vm.VM{})
+	svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	return &stoppedAttachFixture{svc: svc, stored: stored, id: id}
+}
+
+func (f *stoppedAttachFixture) attach(volumeID, device string) (*ec2.VolumeAttachment, error) {
+	in := &ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String(volumeID)}
+	if device != "" {
+		in.Device = aws.String(device)
+	}
+	return f.svc.AttachVolumeToStopped(context.Background(), in, startVolumesAccount)
+}
+
+func (f *stoppedAttachFixture) volume(t *testing.T, id string) ebsmetadata.Volume {
+	t.Helper()
+	v, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, id)
+	require.NoError(t, err)
+	return v
+}
+
+func availableVolume(id string) ebsmetadata.Volume {
+	return ebsmetadata.Volume{VolumeID: id, State: "available", AvailabilityZone: stoppedAttachAZ}
+}
+
+// AWS lets a volume be attached to a stopped instance; the record then names it
+// so the next start mounts it, and the volume is held in-use meanwhile.
+func TestAttachVolumeToStopped_AddsTheVolumeAndClaimsIt(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	out, err := f.attach("vol-new", "")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device), "the first free data device")
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, requestedNames(f.stored))
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, mappedNames(f.stored))
+	assert.Equal(t, vm.StateStopped, f.stored.Status)
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "in-use", got.State)
+	assert.Equal(t, f.id, got.AttachedInstance)
+	assert.Equal(t, "/dev/sdf", got.DeviceName)
+	for _, r := range f.stored.EBSRequests.Requests {
+		if r.Name == "vol-new" {
+			assert.False(t, r.Boot, "a data device is not the root")
+		}
+	}
+}
+
+func TestAttachVolumeToStopped_Refusals(t *testing.T) {
+	tests := []struct {
+		name    string
+		volumes []ebsmetadata.Volume
+		volume  string
+		device  string
+		want    string
+	}{
+		{"volume in use elsewhere", []ebsmetadata.Volume{{VolumeID: "vol-busy", State: "in-use", AttachedInstance: "i-other", AvailabilityZone: stoppedAttachAZ}}, "vol-busy", "", awserrors.ErrorVolumeInUse},
+		{"wrong availability zone", []ebsmetadata.Volume{{VolumeID: "vol-far", State: "available", AvailabilityZone: "ap-southeast-2b"}}, "vol-far", "", awserrors.ErrorInvalidVolumeZoneMismatch},
+		{"no such volume", nil, "vol-missing", "", awserrors.ErrorInvalidVolumeNotFound},
+		{"device already taken", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sdg", awserrors.ErrorInvalidParameterValue},
+		{"root device already has a volume", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sda1", awserrors.ErrorInvalidParameterValue},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newStoppedAttachFixture(t, tt.volumes...)
+
+			_, err := f.attach(tt.volume, tt.device)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored), "a refused attach changes nothing")
+			for _, v := range tt.volumes {
+				got := f.volume(t, v.VolumeID)
+				assert.Equal(t, v.State, got.State, "a refused attach leaves the volume as it was")
+				assert.Equal(t, v.AttachedInstance, got.AttachedInstance)
+			}
+		})
+	}
+}
+
+func TestAttachVolumeToStopped_OtherAccountsInstanceIsNotFound(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	_, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String("vol-new")}, "000000000099")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound))
+	assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored))
+}
+
+// A record edit that cannot be applied must give the volume back, otherwise it
+// stays in-use against an instance that does not name it.
+func TestAttachVolumeToStopped_FailedRecordEditReleasesTheVolume(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+	f.svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{f.id: f.stored}, UpdateStoppedErr: assert.AnError}
+
+	_, err := f.attach("vol-new", "")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServerInternal))
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "available", got.State)
+	assert.Empty(t, got.AttachedInstance)
+}
+
+// Detaching the root volume of a stopped instance makes it unstartable; the way
+// back is attaching a volume at the root device name, which makes it the boot
+// volume again so the start finds a root.
+func TestAttachVolumeToStopped_RootDeviceNameRestoresTheRootAndTheStartWorks(t *testing.T) {
+	id := "i-root-roundtrip"
+	stopped := stoppedWithVolumes(startVolumesAccount, id)
+	stopped.InstanceType = "t3.micro"
+	stopped.Instance.RootDeviceName = aws.String("/dev/sda1")
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+	f.svc.config = &config.Config{AZ: stoppedAttachAZ}
+	meta := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-other", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+	)
+	f.svc.metadata = meta.metadata
+	stopped.AccountID = startVolumesAccount
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), startVolumesAccount)
+	require.NoError(t, err)
+	vol, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, "vol-root")
+	require.NoError(t, err)
+	vol.State, vol.AttachedInstance = "available", ""
+	require.NoError(t, f.svc.metadata.PutVolume(context.Background(), vol))
+
+	out, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(id), VolumeId: aws.String("vol-root"), Device: aws.String("/dev/sda1")}, startVolumesAccount)
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	var boot bool
+	for _, r := range stopped.EBSRequests.Requests {
+		if r.Name == "vol-root" {
+			boot = r.Boot
+		}
+	}
+	assert.True(t, boot, "the root device name makes it the boot volume")
+
+	_, err = f.svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, startVolumesAccount)
+	require.Error(t, err, "the recorder fails the launch after the mount")
+	assert.Contains(t, rec.mounted, "vol-root", "the start now mounts the restored root")
+}

spinifex/domains/ec2/instance/stopped_detach.go

@@ -0,0 +1,112 @@
+package instance
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 {
+		return viperblocklegacyv1.EBSRequest{}, false
+	}
+	return v.EBSRequests.Requests[i], true
+}
+
+// removeVolumeFromRecord drops the volume from the mount list and the API view
+// of the record, and reports whether it was a data volume that was there.
+func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 || v.EBSRequests.Requests[i].EFI {
+		return false
+	}
+	v.EBSRequests.Requests = slices.Delete(v.EBSRequests.Requests, i, i+1)
+	if v.Instance != nil {
+		v.Instance.BlockDeviceMappings = slices.DeleteFunc(v.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && aws.StringValue(m.Ebs.VolumeId) == volumeID
+		})
+	}
+	return true
+}
+
+// stoppedAttachDevice picks the device a volume takes on a stopped instance and
+// reports whether it is the root. An empty request takes the next free
+// /dev/sd[f-p]; a named device must be free, and the root device name is free
+// only while no boot volume is attached.
+func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
+	root := ebsmetadata.RootDeviceName
+	if v.Instance != nil && aws.StringValue(v.Instance.RootDeviceName) != "" {
+		root = aws.StringValue(v.Instance.RootDeviceName)
+	}
+
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	used := make(map[string]bool, len(v.EBSRequests.Requests))
+	hasBoot := false
+	for _, r := range v.EBSRequests.Requests {
+		if r.DeviceName != "" {
+			used[r.DeviceName] = true
+		}
+		hasBoot = hasBoot || r.Boot
+	}
+	if v.Instance != nil {
+		for _, m := range v.Instance.BlockDeviceMappings {
+			if m != nil && m.DeviceName != nil {
+				used[*m.DeviceName] = true
+			}
+		}
+	}
+
+	if requested == "" {
+		for c := 'f'; c <= 'p'; c++ {
+			if dev := fmt.Sprintf("/dev/sd%c", c); !used[dev] {
+				return dev, false, nil
+			}
+		}
+		return "", false, errors.New(awserrors.ErrorAttachmentLimitExceeded)
+	}
+	if used[requested] {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	boot := requested == root
+	if boot && hasBoot {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return requested, boot, nil
+}
+
+// addVolumeToRecord adds the volume to the mount list and the API view of the
+// record. The root device name makes it the boot volume.
+func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+		Name:       volumeID,
+		DeviceName: device,
+		Boot:       boot,
+	})
+	if v.Instance == nil {
+		return
+	}
+	mapping := &ec2.InstanceBlockDeviceMapping{}
+	mapping.SetDeviceName(device)
+	mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
+	mapping.Ebs.SetVolumeId(volumeID)
+	mapping.Ebs.SetAttachTime(time.Now())
+	mapping.Ebs.SetDeleteOnTermination(false)
+	mapping.Ebs.SetStatus("attached")
+	v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, mapping)
+}

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -0,0 +1,139 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stoppedWithVolumes(owner, id string) *vm.VM {
+	v := &vm.VM{
+		ID: id, AccountID: owner, Status: vm.StateStopped,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(id),
+			BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+				{DeviceName: aws.String("/dev/sda1"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-root")}},
+				{DeviceName: aws.String("/dev/sdf"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-data")}},
+				{DeviceName: aws.String("/dev/sdg"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-other")}},
+			},
+		},
+	}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
+		{Name: "vol-data", DeviceName: "/dev/sdf"},
+		{Name: "vol-other", DeviceName: "/dev/sdg"},
+	}
+	return v
+}
+
+func stoppedDetachInput(instanceID, volumeID string) *ec2.DetachVolumeInput {
+	return &ec2.DetachVolumeInput{InstanceId: aws.String(instanceID), VolumeId: aws.String(volumeID)}
+}
+
+func volumeNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+// AWS lets a stopped instance's data volume be detached; the record must stop
+// claiming it so a later start does not re-attach it.
+func TestDetachVolumeFromStopped_RemovesOnlyThatVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-detach"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device))
+	assert.Equal(t, id, aws.StringValue(out.InstanceId))
+	assert.Equal(t, []string{"vol-root", "vol-other"}, volumeNames(stored))
+	var mapped []string
+	for _, m := range stored.Instance.BlockDeviceMappings {
+		mapped = append(mapped, aws.StringValue(m.Ebs.VolumeId))
+	}
+	assert.Equal(t, []string{"vol-root", "vol-other"}, mapped)
+	assert.Equal(t, vm.StateStopped, stored.Status, "the instance stays stopped")
+}
+
+func TestDetachVolumeFromStopped_Refusals(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	tests := []struct {
+		name    string
+		caller  string
+		volume  string
+		stopped bool
+		want    string
+	}{
+		{"volume not on the instance", owner, "vol-elsewhere", true, awserrors.ErrorIncorrectState},
+		{"another account", "000000000099", "vol-data", true, awserrors.ErrorInvalidInstanceIDNotFound},
+		{"no such stopped instance", owner, "vol-data", false, awserrors.ErrorInvalidInstanceIDNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id := "i-stopped-refuse"
+			stored := stoppedWithVolumes(owner, id)
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{}}
+			if tt.stopped {
+				store.Stopped[id] = stored
+			}
+			svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+			_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, tt.volume), tt.caller)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data", "vol-other"}, volumeNames(stored),
+				"a refused detach changes nothing")
+		})
+	}
+}
+
+// AWS lets the root volume of a stopped instance be detached; the instance then
+// cannot start until a volume is attached at its root device name again.
+func TestDetachVolumeFromStopped_DetachesTheRootVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-root"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	assert.Equal(t, []string{"vol-data", "vol-other"}, volumeNames(stored))
+	assert.Equal(t, vm.StateStopped, stored.Status)
+}
+
+// The EFI variable store is Spinifex-internal and never shown to the customer,
+// so it cannot be detached even by its name.
+func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-efi"
+	stored := stoppedWithVolumes(owner, id)
+	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
+		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-efi"), owner)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorOperationNotPermitted), "got %v", err)
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-other", "vol-efi"}, volumeNames(stored))
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,185 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	objectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return objectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -130,3 +130,51 @@ func tallyOrphan(outcome orphanOutcome, removed, failed, unsettled *int) {
 	case orphanGone, orphanOwned:
 	}
 }
+
+// AccountTemplateDeletes is what one DeleteLiveTemplates run did, by template
+// ID. It is the owner's account of its attempts, not proof of absence.
+type AccountTemplateDeletes struct {
+	DeletedTemplateIDs []string
+	FailedTemplateIDs  []string
+	// Referenced are live templates left because referenced named them, with its reason.
+	Referenced map[string]string
+}
+
+// DeleteLiveTemplates deletes the account's live templates through the same
+// owner delete as DeleteLaunchTemplate, except those referenced names. A header
+// it cannot read or decode is left for the owner scan to report. A listing
+// failure is returned and nothing is deleted.
+func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list launch-template keys: %w", err)
+	}
+	prefix := accountID + "."
+	out := &AccountTemplateDeletes{Referenced: map[string]string{}}
+	for _, key := range keys {
+		if !isHeaderKey(key, prefix) {
+			continue
+		}
+		entry, err := s.kv.Get(ctx, key)
+		if err != nil {
+			continue
+		}
+		var header LaunchTemplateHeader
+		if json.Unmarshal(entry.Value(), &header) != nil || prefix+header.LaunchTemplateId != key {
+			continue
+		}
+		if reason := referenced(header.LaunchTemplateId, header.LaunchTemplateName); reason != "" {
+			out.Referenced[header.LaunchTemplateId] = reason
+			continue
+		}
+		if err := s.deleteTemplate(ctx, accountID, &header, entry); err != nil {
+			slog.WarnContext(ctx, "Launch-template account cleanup: live template delete failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
+			out.FailedTemplateIDs = append(out.FailedTemplateIDs, header.LaunchTemplateId)
+			continue
+		}
+		out.DeletedTemplateIDs = append(out.DeletedTemplateIDs, header.LaunchTemplateId)
+	}
+	slog.InfoContext(ctx, "Launch-template account live delete completed", "accountID", accountID,
+		"deleted", len(out.DeletedTemplateIDs), "failed", len(out.FailedTemplateIDs), "referenced", len(out.Referenced))
+	return out, nil
+}

spinifex/domains/ec2/snapshot/drain_test.go

@@ -2,12 +2,8 @@ package snapshot
 
 import (
 	"context"
-	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"net/http"
-	"net/http/httptest"
+	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,8 +11,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
@@ -24,321 +18,78 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const drainInstanceID = "i-drain-host"
+const attachedInstanceID = "i-attached-host"
 
-// setupDrainService returns a snapshot service configured like a daemon that
-// answers ec2.CreateSnapshot: a reachable Predastore host (so drains are not
-// skipped as metadata-only), a data dir with no drain socket in it, and a NATS
-// connection. The Predastore stub rejects everything, so a snapshot that gets
-// past the drain fails fast rather than hanging on a dead address.
-func setupDrainService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// setupAttachedSnapshot is a viperblockd node answering CreateSnapshot for a
+// volume attached to a running instance, with a memory provider behind it.
+func setupAttachedSnapshot(t *testing.T, volumeID string) (*SnapshotServiceImpl, *countingSnapshotProvider, *nats.Conn) {
 	t.Helper()
 
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
 	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: "test-bucket",
-		},
+		DataDir:    testutil.SocketTempDir(t),
+		EBS:        config.EBSConfig{Provider: config.EBSProviderViperblockd},
+		Predastore: config.PredastoreConfig{Host: "127.0.0.1:1", Bucket: "test-bucket"},
 	}
-	return NewSnapshotServiceImplWithStore(cfg, store, nc), store, nc
-}
+	svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nc)
+	provider := &countingSnapshotProvider{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
 
-// seedVolumeAttachment writes the control-plane state.json that decides whether
-// a snapshot must drain.
-func seedVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, "test-bucket", volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-// drainResponder answers the per-instance command subject the way the node
-// hosting the instance does, and hands each request it received to the test.
-func drainResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-// drainedAck is the reply of a node whose drain reached S3.
-func drainedAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-// requireReturnsWithin fails the test if fn has not returned successfully by
-// limit, so a drain that blocks on a socket it should not have dialed reports
-// as a failure rather than as a slow test.
-func requireReturnsWithin(t *testing.T, limit time.Duration, fn func() error) {
-	t.Helper()
-
-	done := make(chan error, 1)
-	go func() { done <- fn() }()
-
-	select {
-	case err := <-done:
-		require.NoError(t, err)
-	case <-time.After(limit):
-		t.Fatalf("drain did not return within %s", limit)
-	}
-}
-
-// awaitDrainCommand returns the command the hosting node received, failing the
-// test if none arrives.
-func awaitDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		assert.Equal(t, testAccountID, msg.Header.Get(natsmsg.AccountIDHeader))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// An attached volume is drained on the node hosting its instance, which is the
-// only node with the socket — the node answering CreateSnapshot usually is not.
-func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached"))
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID))
-
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-attached", command.DrainVolumeData.VolumeID)
-}
-
-// No responder on the command subject means no node is running the instance,
-// which is what a stopped instance looks like: stop tears down both the plugin
-// and the subscription but deliberately leaves a boot volume attached. Failing
-// here would make every stopped instance's root volume unsnapshottable.
-func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID))
-}
-
-// A host that still holds the instance but reports it not running has nothing
-// to flush, so the sealed checkpoint stands. This is the host-drain stop, which
-// unmounts the volumes but keeps the subscription.
-func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-not-running", "in-use", drainInstanceID)
-	ack, err := json.Marshal(ec2v1.DrainVolumeResponse{
-		VolumeID: "vol-not-running", Status: ec2v1.DrainVolumeStatusNotRunning,
-	})
-	require.NoError(t, err)
-	got := drainResponder(t, nc, drainInstanceID, ack)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID))
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// A hosting node that reports the drain failed (wedged plugin, socket gone
-// mid-restart) fails the snapshot rather than falling through.
-func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-drain-err", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "drain failed")
-}
-
-// An ack that is not the drained ack is not evidence the writes reached S3.
-func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID, []byte(`{}`))
-
-	err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "unexpected ack")
-}
-
-// Nothing writes to an unattached volume, so the checkpoint its Close() left
-// behind is current and no drain is attempted — not even locally. Dialing the
-// socket is not a probe: it makes the plugin run a full flush, so an available
-// volume must be decided from the record without touching it.
-func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-available", "available", "")
-
-	// A socket that never answers: a dial would park here until the read
-	// deadline, so returning promptly is what proves none was made.
-	testutil.StartBlockingDrainSocket(t, svc.config.DataDir, "vol-available", "OK\n")
-
-	// A nil connection proves no command was issued: routing one would error.
-	svc.natsConn = nil
-
-	requireReturnsWithin(t, 2*time.Second, func() error {
-		return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID)
-	})
-}
-
-// A volume recorded as in-use but with no instance (drifted state) has no node
-// to address, so it takes the stopped path rather than failing every snapshot.
-func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "")
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID))
-}
-
-// When the volume is served by this node the socket is local, so the drain
-// short-circuits without a hop. This is every single-node deployment.
-func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local", "OK\n")
-
-	// A nil connection proves the local ack was enough: no responder exists for
-	// drainInstanceID, so a routed command would fail.
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID))
-}
-
-// A socket that answers without acking means this node does serve the volume
-// and the flush failed. Routing that onward would come back to this same node
-// and re-run the drain that has already failed, so it fails here instead.
-func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local-err", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n")
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err"))
-
-	err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID)
-	require.Error(t, err)
-
-	assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node")
-}
-
-// state.json is the control-plane-owned attachment record; the copy inside
-// config.json is rewritten by the live NBD plugin from its stale in-memory
-// state, so a volume reading "available" there can still be attached.
-func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config"))
-
-	// "available" mirrors the stale State a live NBD plugin leaves in
-	// config.json; drainVolume must not trust it (see the test's own doc above).
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Volumes predating the state.json split have no state object, so the state
-// embedded in config.json is all there is to decide from.
-func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy"))
-
-	// The state/instance embedded in a legacy config.json, with no state.json to overlay it.
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Without Predastore the snapshot is metadata-only and never reads a live
-// checkpoint, so there is nothing for a drain to make current.
-func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-meta-only", "in-use", drainInstanceID)
-	svc.config.Predastore.Host = ""
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID))
-}
-
-// putProviderVolume seeds both the control-plane document CreateSnapshot's
-// provider branch reads for attachment state, and the provider's own record,
-// mirroring what CreateVolume's provider branch leaves behind.
-func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
-		State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a",
-		ProviderHandle: "memory://volume/" + volumeID,
-	}))
 	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
 		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024},
 		AvailabilityZone: "us-east-1a",
 	})
 	require.NoError(t, err)
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
+		State: "in-use", AttachedInstance: attachedInstanceID, AvailabilityZone: "us-east-1a",
+	}))
+	return svc, provider, nc
 }
 
-// The provider branch used to skip the drain entirely, silently snapshotting a
-// stale checkpoint of an attached, actively-written volume. It must drain the
-// same way the legacy branch does.
-func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain"))
+// The provider drains a buffered volume inside its own CreateSnapshot, so the
+// domain must not dial the drain socket itself: that would drain twice.
+func TestCreateSnapshot_AttachedVolumeOpensNoDrainSocket(t *testing.T) {
+	svc, provider, _ := setupAttachedSnapshot(t, "vol-attached-local")
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, "vol-attached-local", "OK\n")
 
 	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID)
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-local")}, testAccountID)
 	require.NoError(t, err)
 
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID)
+	assert.Zero(t, drains(), "the domain must leave the drain to the provider")
+	assert.Equal(t, int32(1), provider.creates.Load())
 }
 
-// A provider-branch snapshot whose drain fails must fail closed rather than
-// return a snapshot of stale data: the provider must never be asked to
-// snapshot, and no snapshot metadata may be left behind.
-func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-
-	out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID)
+// With no drain socket on this node, an attached volume is snapshotted by the
+// provider without asking the instance's host for a drain.
+func TestCreateSnapshot_AttachedVolumeSendsItsHostNoDrain(t *testing.T) {
+	svc, provider, nc := setupAttachedSnapshot(t, "vol-attached-remote")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(attachedInstanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
 	require.NoError(t, err)
-	assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind")
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	snap, err := svc.CreateSnapshot(context.Background(),
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-remote")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "vol-attached-remote", aws.StringValue(snap.VolumeId))
+	assert.Equal(t, int32(1), provider.creates.Load())
+	assert.Zero(t, commands.Load(), "the instance's host must not be asked to drain")
 }

spinifex/domains/ec2/snapshot/service_impl.go

@@ -5,10 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
-	"net"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -18,10 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -209,19 +204,12 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	// Flush the writes buffered by whichever node serves the volume, so the
-	// provider reads a current checkpoint. An attached volume that cannot be
-	// drained fails here rather than silently snapshotting stale data.
-	if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
 	})
 	if err != nil || created == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
 	}
 	now := time.Now()
 	snapshotCfg := ebsmetadata.Snapshot{
@@ -259,177 +247,6 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
-const (
-	// drainDialTimeout bounds the connect to the local drain socket. The socket
-	// is either being served on this node or it is not, so a slow connect means
-	// a wedged plugin rather than a busy one.
-	drainDialTimeout = time.Second
-
-	// drainAckTimeout bounds DrainToBackend on the hosting node: it flushes the
-	// WAL and the live checkpoint to S3, so it scales with the dirty set rather
-	// than with a fixed unit of work.
-	drainAckTimeout = 30 * time.Second
-
-	// drainRequestTimeout bounds the ec2.cmd round-trip. It exceeds
-	// drainAckTimeout so a slow drain surfaces as the hosting node's error
-	// instead of an opaque NATS timeout here.
-	drainRequestTimeout = 35 * time.Second
-)
-
-// ErrNoDrainSocket reports that this node has no drain socket for the volume,
-// which is how a node learns it does not serve it. It is deliberately distinct
-// from a socket that answered badly: only the former is worth routing onward,
-// and conflating the two makes a node re-run a drain that has already failed.
-var ErrNoDrainSocket = errors.New("no drain socket on this node")
-
-// DrainVolumeSocket asks the NBD plugin serving volumeID on this node to flush
-// its WAL chunks and live checkpoint to S3, then waits for the ack. The plugin
-// creates the socket under {dataDir}/viperblock/{volumeID}/, so this only
-// succeeds on the node hosting the volume; the daemon calls it when a drain
-// command is routed to it.
-func DrainVolumeSocket(dataDir, volumeID string) error {
-	sockPath := filepath.Join(dataDir, "viperblock", volumeID, "snapshot.sock")
-	conn, err := net.DialTimeout("unix", sockPath, drainDialTimeout)
-	if err != nil {
-		return fmt.Errorf("%w: dial %s: %w", ErrNoDrainSocket, sockPath, err)
-	}
-	defer conn.Close()
-
-	_ = conn.SetDeadline(time.Now().Add(drainAckTimeout))
-	buf := make([]byte, 8)
-	n, err := conn.Read(buf)
-	if err != nil {
-		return fmt.Errorf("read drain ack for %s: %w", volumeID, err)
-	}
-	if !strings.HasPrefix(string(buf[:n]), "OK") {
-		return fmt.Errorf("drain of %s did not ack OK: %q", volumeID, string(buf[:n]))
-	}
-	return nil
-}
-
-// drainVolume flushes the volume's in-flight writes to S3 before the snapshot
-// reads the live checkpoint from there.
-func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID)
-}
-
-// DrainVolume flushes the volume's in-flight writes to S3 before a live
-// checkpoint is read from there. Shared by ec2.CreateSnapshot and
-// ec2.CreateImage's running-instance path (domains/ec2/image) so the two
-// live-snapshot call sites cannot diverge on when a drain is required.
-//
-// The drain socket lives on the node serving the volume, but both callers are
-// queue-grouped across every node, so the node answering the request usually
-// is not that node. Whether a drain is required is therefore decided from the
-// volume's attachment record, never from whether the socket happens to be
-// local: an attached volume is being written to right now, and reading its
-// checkpoint without draining silently captures an older one.
-func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	// A metadata-only snapshot never reads the live checkpoint, so there is
-	// nothing for a drain to make current.
-	if cfg == nil || cfg.Predastore.Host == "" {
-		return nil
-	}
-
-	state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance)
-	if err != nil {
-		return err
-	}
-
-	// Nothing is writing to an unattached volume: the checkpoint its Close()
-	// left behind is the current one. Decide this before touching the socket —
-	// dialing it is not a probe, it makes the plugin run a full flush.
-	if state != "in-use" || instanceID == "" {
-		slog.InfoContext(ctx, "DrainVolume: volume not attached, snapshotting the checkpoint left by Close (stopped instance path)",
-			"volumeId", volumeID, "state", state, "attachedInstance", instanceID)
-		return nil
-	}
-
-	// Fast path: the volume is served by this node, so no hop is needed. This is
-	// every single-node deployment, and the node hosting the instance on a
-	// cluster.
-	localErr := DrainVolumeSocket(cfg.DataDir, volumeID)
-	if localErr == nil {
-		return nil
-	}
-
-	// A socket that answered but did not ack means this node does serve the
-	// volume and the flush itself failed. Routing that onward would land back on
-	// this same node and repeat the drain that has already failed.
-	if !errors.Is(localErr, ErrNoDrainSocket) {
-		return fmt.Errorf("drain volume %s on this node: %w", volumeID, localErr)
-	}
-
-	slog.InfoContext(ctx, "DrainVolume: volume attached, routing drain to the node hosting it",
-		"volumeId", volumeID, "instanceId", instanceID)
-	return drainOnHostNode(ctx, natsConn, volumeID, instanceID, accountID)
-}
-
-// volumeAttachment returns the volume's authoritative state and attached
-// instance. state.json is control-plane-owned and authoritative; the fallback
-// pair (passed in from the caller's own volume record) is rewritten by the
-// live NBD plugin from its stale in-memory state and is only used for volumes
-// predating the split.
-func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) {
-	rec, found, err := volumestate.Read(ctx, store, bucket, volumeID)
-	if err != nil {
-		return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err)
-	}
-	if found {
-		return rec.State, rec.AttachedInstance, nil
-	}
-	return fallbackState, fallbackInstance, nil
-}
-
-// drainOnHostNode issues the drain on the node hosting instanceID. Only that
-// node subscribes ec2.cmd.{instanceID}, so the command is self-routing and no
-// volume-to-node resolution is needed here.
-//
-// An attachment record is not proof of a live writer: stop deliberately leaves
-// a boot volume attached (runtime/daemon/vm_adapters.go's volumeMounterAdapter.Unmount)
-// while tearing down both the NBD plugin and this subscription. The two
-// not-running signals below are therefore the stopped-instance path, not a
-// failure — treating them as one would make every stopped instance's root
-// volume permanently unsnapshottable.
-func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanceID, accountID string) error {
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-
-	resp, err := natsmsg.NATSRequest[ec2v1.DrainVolumeResponse](ctx, natsConn,
-		ec2v1.InstanceCommandSubject(instanceID), command, drainRequestTimeout, accountID)
-	if err != nil {
-		// No subscriber at all: the instance runs nowhere in the cluster, so it
-		// was stopped (stop migrates it to shared KV before unsubscribing) and
-		// its volumes were sealed by Close. Warn rather than Info because the
-		// same shape appears if the hosting node has lost NATS while its VM
-		// keeps writing, which yields a stale snapshot as it did before routing
-		// existed. A live host that answers and fails is still a hard error.
-		if errors.Is(err, nats.ErrNoResponders) {
-			slog.WarnContext(ctx, "drainVolume: no node hosts the instance, treating the volume as stopped and snapshotting its sealed checkpoint",
-				"volumeId", volumeID, "instanceId", instanceID)
-			return nil
-		}
-		return fmt.Errorf("drain volume %s on the node hosting %s: %w", volumeID, instanceID, err)
-	}
-
-	switch resp.Status {
-	case ec2v1.DrainVolumeStatusDrained:
-		return nil
-	// The host still holds the instance but it is not running: nothing is
-	// writing, so there is nothing to flush. This is a host-drain stop, which
-	// keeps the VM (and this subscription) in place after unmounting.
-	case ec2v1.DrainVolumeStatusNotRunning:
-		slog.InfoContext(ctx, "drainVolume: instance is not running on its host, snapshotting the volume's sealed checkpoint",
-			"volumeId", volumeID, "instanceId", instanceID)
-		return nil
-	default:
-		return fmt.Errorf("drain volume %s on the node hosting %s: unexpected ack %q", volumeID, instanceID, resp.Status)
-	}
-}
-
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
 var describeSnapshotsValidFilters = map[string]bool{
 	"snapshot-id": true,
@@ -740,6 +557,21 @@ func unreadableRecordError(ctx context.Context, snapshotID, accountID, kind stri
 		"Cannot confirm that snapshot '%s' is not in use: %d %s record(s) could not be read", snapshotID, len(unreadable), kind)
 }
 
+// ProviderCreateError maps a provider CreateSnapshot failure of volumeID to
+// the code a client can act on: retry-later for a provider that cannot answer
+// now, IncorrectState for one that needs the volume detached, else internal.
+func ProviderCreateError(volumeID string, err error) error {
+	switch {
+	case errors.Is(err, ebsprovider.ErrUnavailable):
+		return errors.New(awserrors.ErrorServiceUnavailable)
+	case errors.Is(err, ebsprovider.ErrVolumeInUse):
+		return awserrors.Errorf(awserrors.ErrorIncorrectState,
+			"The volume '%s' is in use. Detach it before creating a snapshot.", volumeID)
+	default:
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+}
+
 // providerDeleteError maps a provider DeleteSnapshot failure to the code a
 // client can act on. A transport failure is retryable: the delete is idempotent.
 func providerDeleteError(snapshotID string, err error) error {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1519,3 +1519,55 @@ func TestDropEmptySnapshotRefsDoesNotDropAConcurrentAdd(t *testing.T) {
 		assert.True(t, has, "iteration %d: the concurrent add was dropped by the empty-key delete", i)
 	}
 }
+
+// createFailingProvider fails only CreateSnapshot, with err.
+type createFailingProvider struct {
+	ebsprovider.EBSProvider
+
+	err error
+}
+
+func (p *createFailingProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, p.err
+}
+
+// A provider that cannot snapshot a volume while it is attached refuses it
+// as in use, which the client can act on by detaching the volume first.
+func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+	createTestVolume(t, svc, store, "vol-create-inuse", 10)
+	svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider,
+		err: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}})
+
+	_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-inuse")}, testAccountID)
+
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, "vol-create-inuse")
+	assert.Contains(t, message, "Detach")
+}
+
+// A provider that cannot answer now, such as one refusing a volume another
+// node holds, is retry-later for the client; any other failure is internal.
+func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
+	cases := []struct {
+		name     string
+		err      error
+		wantCode string
+	}{
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
+		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestSnapshotService(t)
+			createTestVolume(t, svc, store, "vol-create-fail", 10)
+			svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider, err: tc.err})
+
+			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, tc.wantCode, err.Error())
+		})
+	}
+}

spinifex/domains/ec2/volume/service_impl_test.go

@@ -1725,6 +1725,28 @@ func TestDeleteVolume_VolumeAttachedButAvailable(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
 }
 
+// A stopped instance keeps its volumes in use, so deleting one is refused
+// until it is detached; AWS answers the same.
+func TestDeleteVolume_VolumeHeldByStoppedInstanceIsRefused(t *testing.T) {
+	kv := setupTestVolumeKV(t)
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+	svc.snapshotKV = kv
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-held", ebsmetadata.Volume{
+		VolumeID:         "vol-held",
+		CapacityGiB:      10,
+		State:            "in-use",
+		AttachedInstance: "i-stopped",
+	})
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-held"),
+	}, testVolAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
+}
+
 func TestDeleteVolume_EmptyStateUnattachedDeletable(t *testing.T) {
 	kv := setupTestVolumeKV(t)
 	store := objectstore.NewMemoryObjectStore()

spinifex/domains/ec2/volumestate/volumestate.go

@@ -1,78 +0,0 @@
-// Package volumestate reads and writes the control-plane-owned attachment
-// state for an EBS volume, persisted to a per-volume state.json object.
-//
-// It is kept out of config.json because config.json is rewritten by the live
-// nbdkit VB on every SaveState (clobbering any State the control plane wrote
-// there) and is a sealed object for encrypted volumes (a second writer reuses
-// the AES-GCM nonce). state.json is plaintext, viperblock never touches it, so
-// the control plane is its single writer. Readers outside the volume service
-// (the snapshot path) need the same record, which is why it lives here.
-package volumestate
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// Record is the control-plane-owned attachment state of a volume.
-type Record struct {
-	State            string    `json:"state"`
-	AttachedInstance string    `json:"attachedInstance"`
-	DeviceName       string    `json:"deviceName"`
-	AttachedAt       time.Time `json:"attachedAt"`
-}
-
-// Key is the S3 key for a volume's control-plane state object.
-func Key(volumeID string) string { return volumeID + "/state.json" }
-
-// Write persists the attachment state, replacing any previous record.
-func Write(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, rec Record) error {
-	data, err := json.Marshal(rec)
-	if err != nil {
-		return fmt.Errorf("marshal volume state: %w", err)
-	}
-	_, err = store.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-		Body:   bytes.NewReader(data),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to write volume state to S3: %w", err)
-	}
-	return nil
-}
-
-// Read returns the attachment state. found=false with a nil error means the
-// object is absent (a volume predating the state.json split), in which case the
-// caller falls back to the State embedded in config.json.
-func Read(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string) (Record, bool, error) {
-	getResult, err := store.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-	})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return Record{}, false, nil
-		}
-		return Record{}, false, fmt.Errorf("failed to get volume state: %w", err)
-	}
-	defer getResult.Body.Close()
-
-	body, err := io.ReadAll(getResult.Body)
-	if err != nil {
-		return Record{}, false, fmt.Errorf("failed to read volume state body: %w", err)
-	}
-	var rec Record
-	if err := json.Unmarshal(body, &rec); err != nil {
-		return Record{}, false, fmt.Errorf("failed to unmarshal volume state: %w", err)
-	}
-	return rec, true, nil
-}

spinifex/domains/ec2/vpc/external_ipam.go

@@ -122,7 +122,7 @@ func (m *ExternalIPAM) AllocateIP(ctx context.Context, region, az, purpose, allo
 	if m == nil {
 		return "", "", fmt.Errorf("external IPAM unavailable on this node: %w", errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
-	pool := m.findPool(region, az)
+	pool := external.FindPool(m.pools, region, az)
 	if pool == nil {
 		return "", "", fmt.Errorf("no external pool available for region=%q az=%q: %w", region, az, errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
@@ -221,30 +221,6 @@ func (m *ExternalIPAM) allocatorFor(poolName string) (external.Allocator, error)
 	return m.static, nil
 }
 
-// findPool returns the best pool for the given region/AZ using the same
-// fallback order as topology.go: AZ-scoped → region-scoped → unscoped.
-func (m *ExternalIPAM) findPool(region, az string) *external.ExternalPoolConfig {
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ != "" && p.AZ == az && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ == "" && p.Region != "" && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.Region == "" && p.AZ == "" {
-			return p
-		}
-	}
-	return nil
-}
-
 // filterStatic returns only the static pools (Source unset or "static").
 func filterStatic(pools []external.ExternalPoolConfig) []external.ExternalPoolConfig {
 	out := make([]external.ExternalPoolConfig, 0, len(pools))

spinifex/domains/ecr/auth/rotator_test.go

@@ -99,7 +99,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) {
 	r.rotateOnce(t.Context(), time.Now().UTC())
 
 	// A new active key took over.
-	assert.NotEqual(t, key.Kid, issuer.ActiveKid(), "issuer should sign with the rotated key")
+	assert.NotEqual(t, key.Kid, activeKid(issuer), "issuer should sign with the rotated key")
 
 	// Both old and new tokens verify (previous key retained).
 	_, err = verifier.Verify(oldTok)
@@ -142,7 +142,7 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, metas, 1, "the older key should be pruned")
 	assert.Equal(t, second.Kid, active.Kid, "newest key remains active")
-	assert.Equal(t, second.Kid, issuer.ActiveKid())
+	assert.Equal(t, second.Kid, activeKid(issuer))
 }
 
 // TestRotator_RunRotatesThenStops drives the scheduler loop on a fast interval
@@ -169,7 +169,7 @@ func TestRotator_RunRotatesThenStops(t *testing.T) {
 	stopped := make(chan struct{})
 	go func() { r.Run(ctx); close(stopped) }()
 
-	require.Eventually(t, func() bool { return issuer.ActiveKid() != key.Kid },
+	require.Eventually(t, func() bool { return activeKid(issuer) != key.Kid },
 		2*time.Second, 5*time.Millisecond, "rotator should mint a new active key")
 
 	cancel()
@@ -218,3 +218,10 @@ func TestIssuerVerifier_ConcurrentSwap(t *testing.T) {
 	}
 	<-done
 }
+
+// activeKid reads the kid the issuer currently signs with.
+func activeKid(i *Issuer) string {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	return i.key.Kid
+}

spinifex/domains/ecr/auth/token.go

@@ -91,16 +91,6 @@ func (i *Issuer) SetActiveKey(key *SigningKey) {
 	i.mu.Unlock()
 }
 
-// ActiveKid returns the kid of the current signing key, or "" if none.
-func (i *Issuer) ActiveKid() string {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-	if i.key == nil {
-		return ""
-	}
-	return i.key.Kid
-}
-
 // Mint returns a signed ES256 JWT for p and its expiry. The kid header lets any
 // verifier resolve the public key from the shared signing-key set.
 func (i *Issuer) Mint(p Principal) (token string, expiresAt time.Time, err error) {

spinifex/domains/ecr/awsapi/registration.go

@@ -2,14 +2,12 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ecrauth "github.com/mulgadc/spinifex/spinifex/domains/ecr/auth"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -34,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -55,25 +53,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -92,7 +76,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.Error("ECR dispatch: failed to read body", "err", err)
 		return err
@@ -178,19 +162,6 @@ func roleIDOf(actor dispatch.Actor) string {
 	return roleID
 }
 
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 func without(values, excluded []string) []string {
 	return slices.DeleteFunc(slices.Clone(values), func(value string) bool {
 		return slices.Contains(excluded, value)

spinifex/domains/ecr/awsapi/registration_test.go

@@ -172,18 +172,6 @@ var composedCases = []struct{ action, body string }{
 	{"DeleteRepository", `{"repositoryName":"app"}`},
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		TargetPrefix + ".CreateRepository": "CreateRepository",
-		"a.b.ListRepositories":             "ListRepositories",
-		"GetAuthorizationToken":            "GetAuthorizationToken",
-		TargetPrefix + ".":                 "",
-		"":                                 "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // Every composed action must be in the Actions namespace, or dispatch rejects
 // it as InvalidAction before its resources are ever scoped.
 func TestComposedActionsAreInTheNamespace(t *testing.T) {

spinifex/domains/ecr/storage.go

@@ -28,42 +28,22 @@ func AccountBucket(accountID string) string {
 // shapes every ECR storage path must use. Blobs are pooled per account for
 // dedup; manifests and tags are indexed per repository.
 const (
-	// repoMetaKeyFmt is the repo config object (scan-on-push, tag mutability).
-	repoMetaKeyFmt = "repos/%s/_meta.json"
-	// tagKeyFmt maps a tag to a digest pointer (body is "sha256:...").
-	tagKeyFmt = "repos/%s/manifests/by-tag/%s"
 	// manifestKeyFmt holds manifest JSON keyed by its own digest.
 	manifestKeyFmt = "repos/%s/manifests/by-digest/%s"
-	// indexKeyFmt is the rolling tag->digest index used by ListImages.
-	indexKeyFmt = "repos/%s/manifests/index.json"
 	// blobKeyFmt is the account-wide blob pool, two-char hex shard.
 	blobKeyFmt = "blobs/%s/%s"
-	// uploadKeyFmt holds in-progress chunked upload bytes.
-	uploadKeyFmt = "uploads/%s"
 	// uploadChunkKeyFmt holds a single revision's assembled bytes under a unique
 	// per-write token, so a CAS-winning PATCH always points at exactly the bytes
 	// it hashed (a losing concurrent write lands on a different, orphaned token).
 	uploadChunkKeyFmt = "uploads/%s/%s"
 )
 
-// RepoMetaKey returns the key for a repository's config object.
-func RepoMetaKey(repo string) string { return fmt.Sprintf(repoMetaKeyFmt, repo) }
-
-// TagKey returns the key for a tag's digest pointer.
-func TagKey(repo, tag string) string { return fmt.Sprintf(tagKeyFmt, repo, tag) }
-
 // ManifestKey returns the key for a manifest stored by its digest. The digest
 // is sanitized via DigestToken so the object key carries no ':' separator.
 func ManifestKey(repo, digest string) string {
 	return fmt.Sprintf(manifestKeyFmt, repo, DigestToken(digest))
 }
 
-// IndexKey returns the key for a repository's tag->digest index.
-func IndexKey(repo string) string { return fmt.Sprintf(indexKeyFmt, repo) }
-
-// UploadKey returns the key for an in-progress upload's bytes.
-func UploadKey(uploadID string) string { return fmt.Sprintf(uploadKeyFmt, uploadID) }
-
 // UploadChunkKey returns the key for one revision's assembled upload bytes,
 // addressed by a unique per-write token.
 func UploadChunkKey(uploadID, token string) string {

spinifex/domains/ecr/storage_test.go

@@ -17,11 +17,7 @@ func TestKeyHelpers(t *testing.T) {
 		got  string
 		want string
 	}{
-		{"repo meta", RepoMetaKey("team/app"), "repos/team/app/_meta.json"},
-		{"tag", TagKey("team/app", "v1"), "repos/team/app/manifests/by-tag/v1"},
 		{"manifest", ManifestKey("team/app", "sha256:abcd"), "repos/team/app/manifests/by-digest/sha256-abcd"},
-		{"index", IndexKey("team/app"), "repos/team/app/manifests/index.json"},
-		{"upload", UploadKey("u-1"), "uploads/u-1"},
 	}
 	for _, c := range cases {
 		if c.got != c.want {

spinifex/domains/ecs/awsapi/registration.go

@@ -2,14 +2,11 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -35,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -50,35 +47,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 // dispatch resolves, scopes and authorizes the request, then runs the action.
 // Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -99,7 +72,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ECS dispatch: failed to read body", "err", err)
 		return err

spinifex/domains/ecs/awsapi/registration_target_test.go

@@ -1,13 +0,0 @@
-package awsapi
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestActionFromTarget(t *testing.T) {
-	assert.Equal(t, "RunTask", actionFromTarget("AmazonEC2ContainerServiceV20141113.RunTask"))
-	assert.Equal(t, "CreateCluster", actionFromTarget("CreateCluster"))
-	assert.Empty(t, actionFromTarget(""))
-}

spinifex/domains/ecs/capacity_providers.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -23,7 +24,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -33,7 +34,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 
 	rec.CapacityProviders = awsStringSlice(input.CapacityProviders)
 	rec.DefaultCapacityProviderStrategy = strategyFromAWS(input.DefaultCapacityProviderStrategy)
-	if err := putJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.PutClusterCapacityProvidersOutput{Cluster: rec.toAWS()}, nil
@@ -53,7 +54,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 	}
 
 	var existing CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &existing)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &existing)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +70,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 		Tags:                     tagsToMap(input.Tags),
 		CreatedAt:                time.Now().UTC(),
 	}
-	if err := putJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.CreateCapacityProviderOutput{CapacityProvider: rec.toAWS()}, nil
@@ -86,13 +87,13 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 
 	names := awsStringSlice(input.CapacityProviders)
 	if len(names) == 0 {
-		keys, kerr := keysWithPrefix(ctx, kv, CapacityProvidersPrefix())
+		keys, kerr := taskdefinition.KeysWithPrefix(ctx, kv, CapacityProvidersPrefix())
 		if kerr != nil {
 			return nil, kerr
 		}
 		for _, k := range keys {
 			var rec CapacityProviderRecord
-			ok, gerr := getJSON(ctx, kv, k, &rec)
+			ok, gerr := taskdefinition.GetJSON(ctx, kv, k, &rec)
 			if gerr != nil {
 				return nil, gerr
 			}
@@ -106,7 +107,7 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 	for _, ref := range names {
 		name := CapacityProviderShortName(ref)
 		var rec CapacityProviderRecord
-		ok, gerr := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+		ok, gerr := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -132,7 +133,7 @@ func (s *Service) DeleteCapacityProvider(ctx context.Context, input *ecs.DeleteC
 		return nil, err
 	}
 	var rec CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ecs/cluster_teardown.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -34,7 +35,7 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -71,7 +72,7 @@ func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.De
 		return nil, err
 	}
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +125,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 	for _, ref := range awsStringSlice(input.ContainerInstances) {
 		id := ContainerInstanceShortID(ref)
 		var rec InstanceRecord
-		found, gerr := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -134,7 +135,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 		}
 		rec.Status = status
 		rec.Reaped = false
-		if perr := putJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
 			return nil, perr
 		}
 		if status == InstanceStatusDraining {

spinifex/domains/ecs/cluster_teardown_test.go

@@ -28,7 +28,7 @@ func TestService_DeleteCluster_CascadesAndSweeps(t *testing.T) {
 	list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, list.ClusterArns)
-	keys, err := keysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
 	require.NoError(t, err)
 	assert.Empty(t, keys)
 }

spinifex/domains/ecs/deployments_test.go

@@ -19,7 +19,7 @@ import (
 func reloadService(t *testing.T, kv jetstream.KeyValue, cluster, name string) *ServiceRecord {
 	t.Helper()
 	var rec ServiceRecord
-	found, err := getJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return &rec
@@ -329,7 +329,7 @@ func TestDeployment_LegacyServiceSynthesizesPrimary(t *testing.T) {
 		DesiredCount: 1, Status: ServiceStatusActive,
 		SchedulingStrategy: SchedulingStrategyReplica, DeploymentID: "legacy-dep",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
 
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, rec))
 	reloaded := reloadService(t, kv, "web", "legacy")
@@ -388,7 +388,7 @@ func TestDeployment_RepeatedStartFailuresBackOffWithoutTheCircuitBreaker(t *test
 	// Once it passes, launching resumes: the backoff is a brake, not a stop.
 	rec = reloadService(t, kv, "web", "web")
 	rec.primaryDeployment().NextLaunchAt = time.Now().UTC().Add(-time.Second)
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, reloadService(t, kv, "web", "web")))
 
 	assert.Positive(t, liveServiceTasks(t, svc, kv, "web", "web"), "backoff never released the deployment")

spinifex/domains/ecs/eip_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,14 +38,14 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", AssignPublicIP: "ENABLED",
 	}))
 	task := &TaskRecord{
 		TaskID: "t-1", Cluster: "web", Group: serviceTaskGroup("web"),
 		ENIID: "eni-1", ENIPrivateIP: "172.31.0.8",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
 
 	svc.assignTaskPublicIP(context.Background(), kv, testAccountID, task)
 
@@ -53,7 +54,7 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	assert.Equal(t, "eipalloc-1", task.ENIEIPAllocationID)
 
 	var persisted TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, "192.0.2.50", persisted.ENIPublicIP)
@@ -64,7 +65,7 @@ func TestAssignTaskPublicIP_DisabledNoOp(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", // AssignPublicIP unset
 	}))
 	task := &TaskRecord{

spinifex/domains/ecs/eni_test.go

@@ -189,14 +189,14 @@ func singleTaskRecord(t *testing.T, svc *Service, cluster string) (TaskRecord, b
 	t.Helper()
 	kv, err := svc.bucket(context.Background(), testAccountID)
 	require.NoError(t, err)
-	keys, err := keysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
 	require.NoError(t, err)
 	require.LessOrEqual(t, len(keys), 1, "test expects at most one task record")
 	if len(keys) == 0 {
 		return TaskRecord{}, false
 	}
 	var rec TaskRecord
-	found, err := getJSON(context.Background(), kv, keys[0], &rec)
+	found, err := taskdefinition.GetJSON(context.Background(), kv, keys[0], &rec)
 	require.NoError(t, err)
 	return rec, found
 }
@@ -317,10 +317,10 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	_, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())

spinifex/domains/ecs/export_test.go

@@ -15,8 +15,6 @@ const TestAccountID = testAccountID
 var (
 	NewTestService   = newTestService
 	RegisterInstance = registerInstance
-	PutJSON          = putJSON
-	GetJSON          = getJSON
 	ServiceTaskGroup = serviceTaskGroup
 )
 

spinifex/domains/ecs/gpu_report.go

@@ -1,6 +1,10 @@
 package ecs
 
-import "context"
+import (
+	"context"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+)
 
 // ReportTaskGPUInput is the agent's per-task GPU device-assignment report
 // (agent → gateway). Internal agent↔gateway shape, not an AWS SDK action:
@@ -36,7 +40,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		return nil, err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &task)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +59,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		}
 	}
 	if changed {
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
 			return nil, err
 		}
 	}

spinifex/domains/ecs/gpu_report_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -17,7 +18,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", Cluster: "web",
 		Containers: []ContainerState{
 			{Name: "web", Status: "RUNNING"},
@@ -33,7 +34,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	assert.Equal(t, "OK", out.Acknowledgment)
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	for _, c := range rec.Containers {
@@ -63,7 +64,7 @@ func TestDescribeTasks_GPUIDsPopulatedFromReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		Containers: []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa", "GPU-bbb"}}},
 	}))
@@ -85,7 +86,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "PENDING"}},
@@ -99,7 +100,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -114,7 +115,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 		Containers: []ecsv1.ContainerStatus{{Name: "trainer", Status: ecsv1.TaskStatusStopped, ExitCode: &exit}},
 	}))
 
-	found, err = getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -137,7 +138,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa"}}},
@@ -150,7 +151,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -163,7 +164,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "web", Status: "PENDING"}},
@@ -183,7 +184,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)

spinifex/domains/ecs/instance_recovery_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -27,7 +28,7 @@ func instanceStatus(t *testing.T, svc *Service, cluster, id string) InstanceReco
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return rec
@@ -46,7 +47,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	require.NoError(t, err)
 	rec := instanceStatus(t, svc, "web", "i-1")
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
@@ -86,7 +87,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 

spinifex/domains/ecs/instances.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"slices"
 	"time"
@@ -17,14 +18,14 @@ import (
 
 // listInstanceRecords returns all container-instance records in a cluster.
 func (s *Service) listInstanceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]InstanceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, InstancesPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, InstancesPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]InstanceRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -138,7 +139,7 @@ func (s *Service) DescribeContainerInstances(ctx context.Context, input *ecs.Des
 	for _, ref := range awsStringSlice(input.ContainerInstances) {
 		id := ContainerInstanceShortID(ref)
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -194,7 +195,7 @@ func (s *Service) ListContainerInstances(ctx context.Context, input *ecs.ListCon
 // it back. Used by both the AWS register path and the bus register handler.
 func (s *Service) upsertInstance(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string, mutate func(*InstanceRecord)) (*InstanceRecord, error) {
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -212,7 +213,7 @@ func (s *Service) upsertInstance(ctx context.Context, kv jetstream.KeyValue, acc
 	if mutate != nil {
 		mutate(&rec)
 	}
-	if err := putJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec); err != nil {
 		return nil, err
 	}
 	return &rec, nil
@@ -345,7 +346,7 @@ func (s *Service) recordHeartbeat(ctx context.Context, msg *ecsv1.Heartbeat) err
 		return err
 	}
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
 	if err != nil || !found {
 		return err
 	}
@@ -353,7 +354,7 @@ func (s *Service) recordHeartbeat(ctx context.Context, msg *ecsv1.Heartbeat) err
 	if msg.Status != "" {
 		rec.Status = msg.Status
 	}
-	return putJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
+	return taskdefinition.PutJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
 }
 
 // recordTaskState applies an agent task-state report: it updates the task record
@@ -364,7 +365,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		return err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task)
 	if err != nil || !found {
 		return err
 	}
@@ -406,7 +407,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		// filtering on desired status would otherwise see it as still wanted.
 		task.DesiredStatus = TaskStatusStopped
 	}
-	if err := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil {
 		return err
 	}
 
@@ -430,7 +431,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		s.reclaimAssignInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID)
 		s.reclaimStopInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID)
 		s.reclaimTaskENI(ctx, msg.AccountID, &task)
-		if perr := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil {
 			slog.ErrorContext(ctx, "ECS task STOPPED: persist after EIP release failed", "task", msg.TaskID, "err", perr)
 		}
 		return s.releaseReservation(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU, eniReservationFor(task.NetworkMode))
@@ -485,7 +486,7 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv jetstream.KeyV
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &svc)
 	if err != nil || !found {
 		return
 	}
@@ -493,7 +494,7 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv jetstream.KeyV
 		if svc.Deployments[i].ID == depID {
 			svc.Deployments[i].FailedTasks++
 			svc.Deployments[i].UpdatedAt = time.Now().UTC()
-			if perr := putJSON(ctx, kv, ServiceKey(cluster, name), &svc); perr != nil {
+			if perr := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &svc); perr != nil {
 				slog.ErrorContext(ctx, "ECS deployment failure accounting: persist failed", "service", name, "err", perr)
 			}
 			return

spinifex/domains/ecs/poll.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/nats-io/nats.go/jetstream"
@@ -55,14 +56,14 @@ func (s *Service) PollAssignments(ctx context.Context, input *PollAssignmentsInp
 		_ = kv.Delete(ctx, StopKey(cluster, instanceID, TaskShortID(taskID)))
 	}
 
-	keys, err := keysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, instanceID))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, instanceID))
 	if err != nil {
 		return nil, err
 	}
 	out := &PollAssignmentsOutput{}
 	for _, key := range keys {
 		var as ecsv1.Assign
-		found, err := getJSON(ctx, kv, key, &as)
+		found, err := taskdefinition.GetJSON(ctx, kv, key, &as)
 		if err != nil {
 			return nil, err
 		}
@@ -71,13 +72,13 @@ func (s *Service) PollAssignments(ctx context.Context, input *PollAssignmentsInp
 		}
 	}
 
-	stopKeys, err := keysWithPrefix(ctx, kv, StopsPrefix(cluster, instanceID))
+	stopKeys, err := taskdefinition.KeysWithPrefix(ctx, kv, StopsPrefix(cluster, instanceID))
 	if err != nil {
 		return nil, err
 	}
 	for _, key := range stopKeys {
 		var sd ecsv1.StopDirective
-		found, err := getJSON(ctx, kv, key, &sd)
+		found, err := taskdefinition.GetJSON(ctx, kv, key, &sd)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/poll_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
@@ -16,9 +17,9 @@ import (
 func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) {
 	svc, _, kv := serviceTestRig(t)
 
-	require.NoError(t, putJSON(t.Context(), kv, AssignmentKey("web", "i-1", "t-1"),
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, AssignmentKey("web", "i-1", "t-1"),
 		&ecsv1.Assign{TaskID: "t-1", ClusterName: "web", InstanceID: "i-1"}))
-	require.NoError(t, putJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"),
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"),
 		&ecsv1.StopDirective{TaskID: "t-2", Reason: "drain"}))
 
 	out, err := svc.PollAssignments(context.Background(), &PollAssignmentsInput{
@@ -41,7 +42,7 @@ func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) {
 	assert.Empty(t, out.Assignments)
 	assert.Empty(t, out.Stops)
 
-	found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &ecsv1.StopDirective{})
+	found, err := taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &ecsv1.StopDirective{})
 	require.NoError(t, err)
 	assert.False(t, found)
 }

spinifex/domains/ecs/revisit_test.go

@@ -7,6 +7,7 @@ package ecs
 import (
 	"context"
 	"encoding/json"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -60,11 +61,11 @@ func runOneTask(t *testing.T, svc *Service, kv jetstream.KeyValue) string {
 		Cluster: aws.String("web"), TaskDefinition: aws.String("app"), Count: aws.Int64(1),
 	}, testAccountID)
 	require.NoError(t, err)
-	keys, err := keysWithPrefix(t.Context(), kv, TasksPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, TasksPrefix("web"))
 	require.NoError(t, err)
 	require.Len(t, keys, 1)
 	var task TaskRecord
-	found, err := getJSON(t.Context(), kv, keys[0], &task)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, keys[0], &task)
 	require.NoError(t, err)
 	require.True(t, found)
 	return task.TaskID
@@ -128,11 +129,11 @@ func TestScheduler_TheReaperAsksForTheSoonestHeartbeatWindow(t *testing.T) {
 	now := time.Now().UTC()
 
 	var inst InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	inst.LastSeen = now.Add(-30 * time.Second)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 
 	due, err := sc.reap(t.Context())
 	require.NoError(t, err)
@@ -142,12 +143,12 @@ func TestScheduler_TheReaperAsksForTheSoonestHeartbeatWindow(t *testing.T) {
 	// Past the window the reaper acts, and with nothing live left it asks for
 	// nothing: the resync is the only thing that need bring it back.
 	inst.LastSeen = now.Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 	due, err = sc.reap(t.Context())
 	require.NoError(t, err)
 	assert.Zero(t, due)
 
-	found, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, InstanceStatusDraining, inst.Status, "an instance past its window is still reaped")
@@ -195,17 +196,17 @@ func TestScheduler_TheTimerPassReturnsTheSoonestOfItsJobs(t *testing.T) {
 	// A live instance due in 10s, and a stopped task due much later: the pass must
 	// come back for the sooner of the two.
 	var inst InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	inst.LastSeen = now.Add(10*time.Second - heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 
 	stopped := TaskRecord{
 		TaskID: "t-1", Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", "t-1"),
 		LastStatus: TaskStatusStopped, DesiredStatus: TaskStatusStopped, StoppedAt: now,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &stopped))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &stopped))
 
 	reapDue, err := sc.reap(t.Context())
 	require.NoError(t, err)
@@ -231,11 +232,11 @@ func TestScheduler_AnInFlightServiceAsksToBeRevisited(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, reconcileInterval, due)
 
-	keys, err := keysWithPrefix(t.Context(), kv, TasksPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, TasksPrefix("web"))
 	require.NoError(t, err)
 	require.Len(t, keys, 1)
 	var task TaskRecord
-	found, err := getJSON(t.Context(), kv, keys[0], &task)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, keys[0], &task)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.NoError(t, svc.recordTaskState(t.Context(), &ecsv1.TaskState{

spinifex/domains/ecs/scheduler.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"sync"
@@ -362,7 +363,7 @@ func (sc *Scheduler) reap(ctx context.Context) (time.Duration, error) {
 // reapBucket reaps one account's disconnected instances and reports when the
 // soonest instance it left alone falls due.
 func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, accountID string, now time.Time) time.Duration {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return 0
 	}
@@ -372,7 +373,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 			continue
 		}
 		var inst InstanceRecord
-		found, err := getJSON(ctx, kv, k, &inst)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &inst)
 		if err != nil || !found {
 			continue
 		}
@@ -387,7 +388,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 			"lastSeen", inst.LastSeen.Format(time.RFC3339))
 		inst.Status = InstanceStatusDraining
 		inst.Reaped = true
-		if perr := putJSON(ctx, kv, k, &inst); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, k, &inst); perr != nil {
 			continue
 		}
 		sc.stopInstanceTasks(ctx, kv, accountID, inst.Cluster, inst.InstanceID)
@@ -398,13 +399,13 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 // stopInstanceTasks transitions a reaped instance's non-stopped tasks to STOPPED
 // and reclaims each awsvpc task's ENI (leak guard for a dead agent).
 func (sc *Scheduler) stopInstanceTasks(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string) {
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return
 	}
 	for _, k := range keys {
 		var task TaskRecord
-		found, err := getJSON(ctx, kv, k, &task)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &task)
 		if err != nil || !found {
 			continue
 		}

spinifex/domains/ecs/service.go

@@ -5,9 +5,7 @@ package ecs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
-	"fmt"
 	"log/slog"
 	"maps"
 	"slices"
@@ -168,39 +166,12 @@ func (s *Service) bucket(ctx context.Context, accountID string) (jetstream.KeyVa
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
 
-// getJSON reads key into out. Returns (false, nil) when the key is absent.
-func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
-	entry, err := kv.Get(ctx, key)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return false, nil
-	}
-	if err != nil {
-		return false, err
-	}
-	if err := json.Unmarshal(entry.Value(), out); err != nil {
-		return false, fmt.Errorf("unmarshal %s: %w", key, err)
-	}
-	return true, nil
-}
-
-// putJSON marshals v and writes it at key.
-func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
-	data, err := json.Marshal(v)
-	if err != nil {
-		return err
-	}
-	if _, err := kv.Put(ctx, key, data); err != nil {
-		return fmt.Errorf("put %s: %w", key, err)
-	}
-	return nil
-}
-
 // deleteKeysWithPrefix deletes every KV key under prefix. Used by DeleteCluster
 // to sweep a cluster's whole key subtree (meta/instances/tasks/services/
 // assignments) after its tasks are stopped. Best-effort per key; the first
 // delete error is returned after attempting the rest.
 func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) error {
-	keys, err := keysWithPrefix(ctx, kv, prefix)
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, prefix)
 	if err != nil {
 		return err
 	}
@@ -213,25 +184,6 @@ func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix str
 	return firstErr
 }
 
-// keysWithPrefix returns all KV keys under prefix.
-func keysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
-	keys, err := kv.Keys(ctx)
-	if errors.Is(err, jetstream.ErrNoKeysFound) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	out := make([]string, 0, len(keys))
-	for _, k := range keys {
-		if strings.HasPrefix(k, prefix) {
-			out = append(out, k)
-		}
-	}
-	slices.Sort(out)
-	return out, nil
-}
-
 // --- Cluster ---
 
 // CreateCluster persists a cluster meta record. Idempotent: re-creating an
@@ -247,7 +199,7 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 	}
 
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -269,7 +221,7 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 				rec.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
 			}
 		}
-		if err := putJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil {
 			return nil, err
 		}
 	}
@@ -292,7 +244,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 	for _, name := range names {
 		name = ClusterShortName(name)
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(name), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -358,7 +310,7 @@ func (s *Service) ListClusters(ctx context.Context, _ *ecs.ListClustersInput, ac
 	if err != nil {
 		return nil, err
 	}
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return nil, err
 	}
@@ -368,7 +320,7 @@ func (s *Service) ListClusters(ctx context.Context, _ *ecs.ListClustersInput, ac
 			continue
 		}
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/service_test.go

@@ -378,7 +378,7 @@ func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
 		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
 
 		var rec TaskRecord
-		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, TaskKey("web", taskID), &rec)
 		require.NoError(t, gerr)
 		require.True(t, found)
 		assert.Equal(t, as.CredID, rec.CredID)
@@ -558,10 +558,10 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	_, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
@@ -721,7 +721,7 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 2, rec.TotalGPU)
@@ -732,7 +732,7 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) {
 		AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", TaskID: taskID,
 		LastStatus: ecsv1.TaskStatusStopped, Reason: "exited",
 	}))
-	found, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 0, rec.ReservedGPU)

spinifex/domains/ecs/services.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -145,7 +146,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -154,7 +155,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 	}
 
 	var existing ServiceRecord
-	if ok, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &existing); gerr != nil {
+	if ok, gerr := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &existing); gerr != nil {
 		return nil, gerr
 	} else if ok && existing.Status == ServiceStatusActive {
 		return &ecs.CreateServiceOutput{Service: s.serviceToAWS(accountID, &existing)}, nil
@@ -195,7 +196,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 	applyDeploymentConfig(&rec, input.DeploymentConfiguration)
 	rec.LastGoodTaskDefARN = taskDef.ARN
 	rec.Deployments = []Deployment{newPrimaryDeployment(rec.DeploymentID, taskDef, rec.DesiredCount)}
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 
@@ -214,7 +215,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp
 		return nil, err
 	}
 	var rec ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +254,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp
 		}
 	}
 	rec.UpdatedAt = time.Now().UTC()
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 	if err := s.reconcileService(ctx, kv, accountID, &rec); err != nil {
@@ -272,7 +273,7 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp
 		return nil, err
 	}
 	var rec ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -292,7 +293,7 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp
 	rec.RunningCount = 0
 	rec.PendingCount = 0
 	rec.UpdatedAt = time.Now().UTC()
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.DeleteServiceOutput{Service: s.serviceToAWS(accountID, &rec)}, nil
@@ -309,7 +310,7 @@ func (s *Service) DescribeServices(ctx context.Context, input *ecs.DescribeServi
 	for _, ref := range awsStringSlice(input.Services) {
 		name := ServiceShortName(ref)
 		var rec ServiceRecord
-		found, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -430,7 +431,7 @@ func persistServiceIfChanged(ctx context.Context, kv jetstream.KeyValue, svc *Se
 	if bytes.Equal(before, after) {
 		return nil
 	}
-	return putJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc)
+	return taskdefinition.PutJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc)
 }
 
 // reconcileAllServices reconciles every ACTIVE service in every ECS account
@@ -603,14 +604,14 @@ func (s *Service) listServiceTasks(ctx context.Context, kv jetstream.KeyValue, c
 
 // listTaskRecords returns every task record in a cluster.
 func (s *Service) listTaskRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]TaskRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]TaskRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -623,14 +624,14 @@ func (s *Service) listTaskRecords(ctx context.Context, kv jetstream.KeyValue, cl
 
 // listServiceRecords returns every service record in a cluster.
 func (s *Service) listServiceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]ServiceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, ServicesPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, ServicesPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]ServiceRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -643,7 +644,7 @@ func (s *Service) listServiceRecords(ctx context.Context, kv jetstream.KeyValue,
 
 // allServiceRecords returns every service record across all clusters in a bucket.
 func (s *Service) allServiceRecords(ctx context.Context, kv jetstream.KeyValue) ([]ServiceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return nil, err
 	}
@@ -653,7 +654,7 @@ func (s *Service) allServiceRecords(ctx context.Context, kv jetstream.KeyValue)
 			continue
 		}
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/services_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -469,7 +470,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.9",
 		ReservedCPU: 64, ReservedMemoryMiB: 64,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-9"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-9"), task))
 
 	// Cooperative stop: the task stays RUNNING with desiredStatus=STOPPED and a
 	// stop directive lands in the instance inbox; nothing is deregistered until the
@@ -483,7 +484,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 	assert.Empty(t, reg.deregistered)
 
 	var sd ecsv1.StopDirective
-	found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, "bye", sd.Reason)
@@ -494,7 +495,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 		TaskID: "t-9", LastStatus: ecsv1.TaskStatusStopped,
 	}))
 	assert.Equal(t, []string{key(tgARN, "10.0.1.9", 80)}, reg.deregistered)
-	found, err = getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
 	require.NoError(t, err)
 	assert.False(t, found)
 
@@ -518,7 +519,7 @@ func TestService_StopTask_UnplacedForceStops(t *testing.T) {
 		TaskID: "t-np", Cluster: "web",
 		LastStatus: TaskStatusPending, DesiredStatus: TaskStatusRunning,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-np"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-np"), task))
 
 	out, err := svc.StopTask(context.Background(), &ecs.StopTaskInput{
 		Cluster: aws.String("web"), Task: aws.String("t-np"), Reason: aws.String("cancel"),
@@ -550,7 +551,7 @@ func TestService_TargetGroup_RegisterOnRunningDeregisterOnStopped(t *testing.T)
 		ContainerInstanceID: "i-1", LastStatus: TaskStatusPending,
 		NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.5",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
 
 	// RUNNING → register.
 	require.NoError(t, svc.recordTaskState(context.Background(), &ecsv1.TaskState{

spinifex/domains/ecs/store.go

@@ -107,12 +107,6 @@ func ServiceKey(cluster, serviceName string) string {
 	return ServicesPrefix(cluster) + serviceName
 }
 
-// LeaderLeaseKey returns the per-cluster scheduler leader-lease key in the shared
-// leader bucket (Q3).
-func LeaderLeaseKey(accountID, clusterName string) string {
-	return fmt.Sprintf("%s/%s", accountID, clusterName)
-}
-
 // CapacityProvidersPrefix returns the KV key prefix under which all
 // capacity-provider records live. Account-scoped (not cluster-scoped),
 // matching the AWS ARN shape; a cluster references providers by name.

spinifex/domains/ecs/store_test.go

@@ -124,8 +124,6 @@ func TestKeyPaths(t *testing.T) {
 
 	assert.Equal(t, "clusters/web/services/", ServicesPrefix("web"))
 	assert.Equal(t, "clusters/web/services/api", ServiceKey("web", "api"))
-
-	assert.Equal(t, "123456789012/web", LeaderLeaseKey("123456789012", "web"))
 }
 
 // Prefix helpers must be a true prefix of their per-record key so a KV

spinifex/domains/ecs/tagging_current_behaviour_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"strings"
 	"sync"
 	"testing"
@@ -57,11 +58,11 @@ func TestCurrentBehaviour_Tagging_ECSTagWriteRevertsInterleavedRecordWrite(t *te
 	key := ServiceKey("web", "api")
 	wrapped := &interleavingKV{KeyValue: kv, key: key, between: func() {
 		var rec ServiceRecord
-		found, gerr := getJSON(ctx, kv, key, &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, key, &rec)
 		require.NoError(t, gerr)
 		require.True(t, found)
 		rec.DesiredCount = 5
-		require.NoError(t, putJSON(ctx, kv, key, &rec))
+		require.NoError(t, taskdefinition.PutJSON(ctx, kv, key, &rec))
 	}}
 
 	err = svc.mutateResourceTags(ctx, wrapped, ecsResourceService, "web", "api", func(existing map[string]string) map[string]string {
@@ -74,7 +75,7 @@ func TestCurrentBehaviour_Tagging_ECSTagWriteRevertsInterleavedRecordWrite(t *te
 	require.NoError(t, err)
 
 	var final ServiceRecord
-	found, err := getJSON(ctx, kv, key, &final)
+	found, err := taskdefinition.GetJSON(ctx, kv, key, &final)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 0, final.DesiredCount, "the interleaved DesiredCount=5 write is lost")

spinifex/domains/ecs/tags.go

@@ -78,7 +78,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 	switch kind {
 	case ecsResourceCluster:
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -92,7 +92,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 			return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		var rec taskdefinition.Record
-		found, err := getJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -102,7 +102,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceService:
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -112,7 +112,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceTask:
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -122,7 +122,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceContainerInstance:
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -142,7 +142,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 	switch kind {
 	case ecsResourceCluster:
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(id), &rec)
 		if err != nil {
 			return err
 		}
@@ -150,14 +150,14 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSClusterNotFound)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(id), &rec)
 	case ecsResourceTaskDefinition:
 		family, rev := taskdefinition.ParseRef(id)
 		if family == "" || rev == 0 {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		var rec taskdefinition.Record
-		found, err := getJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 		if err != nil {
 			return err
 		}
@@ -165,10 +165,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		return taskdefinition.PutJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 	case ecsResourceService:
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -176,10 +176,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSServiceNotFound)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 	case ecsResourceTask:
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -187,10 +187,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, id), &rec)
 	case ecsResourceContainerInstance:
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -198,7 +198,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	default:
 		return errors.New(awserrors.ErrorECSInvalidParameter)
 	}

spinifex/domains/ecs/task_accounting_test.go

@@ -1,6 +1,7 @@
 package ecs_test
 
 import (
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -23,7 +24,7 @@ func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ec
 	for _, r := range recs {
 		r.Cluster = cluster
 		r.ARN = ecsdomain.TaskARN(svc.Region(), ecsdomain.TestAccountID, cluster, r.TaskID)
-		require.NoError(t, ecsdomain.PutJSON(t.Context(), kv, ecsdomain.TaskKey(cluster, r.TaskID), r))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ecsdomain.TaskKey(cluster, r.TaskID), r))
 	}
 }
 
@@ -124,7 +125,7 @@ func TestRecordTaskState_SelfExitMovesDesiredStatusToStopped(t *testing.T) {
 	kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
 	require.NoError(t, err)
 	var rec ecsdomain.TaskRecord
-	found, err := ecsdomain.GetJSON(t.Context(), kv, ecsdomain.TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, ecsdomain.TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, ecsdomain.TaskStatusStopped, rec.DesiredStatus)

spinifex/domains/ecs/task_stop.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -33,7 +34,7 @@ func (s *Service) StopTask(ctx context.Context, input *ecs.StopTaskInput, accoun
 		return nil, err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &task)
 	if err != nil {
 		return nil, err
 	}
@@ -57,7 +58,7 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -101,7 +102,7 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco
 				continue
 			}
 		}
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
 			return nil, err
 		}
 		if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil {
@@ -168,12 +169,12 @@ func (s *Service) requestStopTask(ctx context.Context, kv jetstream.KeyValue, ac
 		return
 	}
 	task.DesiredStatus = TaskStatusStopped
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS requestStopTask: persist failed", "task", task.TaskID, "err", perr)
 		return
 	}
 	sd := ecsv1.StopDirective{TaskID: task.TaskID, Reason: reason}
-	if perr := putJSON(ctx, kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil {
 		slog.ErrorContext(ctx, "ECS requestStopTask: post stop directive failed", "task", task.TaskID, "err", perr)
 	}
 }
@@ -196,7 +197,7 @@ func (s *Service) forceStopTask(ctx context.Context, kv jetstream.KeyValue, acco
 	// cleared fields land in this single write.
 	s.releaseTaskPublicIP(ctx, accountID, task)
 	s.reclaimTaskENI(ctx, accountID, task)
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS forceStopTask: persist failed", "task", task.TaskID, "err", perr)
 		return
 	}
@@ -238,7 +239,7 @@ func (s *Service) applyServiceTargets(ctx context.Context, kv jetstream.KeyValue
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
 	if err != nil || !found || len(svc.LoadBalancers) == 0 {
 		return
 	}
@@ -270,7 +271,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv jetstream.KeyValue,
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
 	if err != nil || !found || !strings.EqualFold(svc.AssignPublicIP, "ENABLED") {
 		return
 	}
@@ -281,7 +282,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv jetstream.KeyValue,
 	}
 	task.ENIPublicIP = publicIP
 	task.ENIEIPAllocationID = allocID
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS auto-EIP: persist public IP failed", "task", task.TaskID, "err", perr)
 	}
 }

spinifex/domains/ecs/taskdef_validation_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -240,7 +241,7 @@ func TestRunTask_AssignCarriesGPU(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", taskID), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", taskID), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 1, rec.GPU)

spinifex/domains/ecs/taskdefinition/owner.go

@@ -63,10 +63,10 @@ func (o *Owner) Register(ctx context.Context, kv jetstream.KeyValue, accountID s
 		RequiresCompatibilities: spec.RequiresCompatibilities,
 		RuntimePlatform:         spec.RuntimePlatform,
 	}
-	if err := putJSON(ctx, kv, RevKey(spec.Family, rev), rec); err != nil {
+	if err := PutJSON(ctx, kv, RevKey(spec.Family, rev), rec); err != nil {
 		return nil, err
 	}
-	if err := putJSON(ctx, kv, LatestRevKey(spec.Family), rev); err != nil {
+	if err := PutJSON(ctx, kv, LatestRevKey(spec.Family), rev); err != nil {
 		return nil, err
 	}
 	return rec, nil
@@ -75,7 +75,7 @@ func (o *Owner) Register(ctx context.Context, kv jetstream.KeyValue, accountID s
 // nextRevision reads the family's latest-rev and returns latest+1 (1 if absent).
 func nextRevision(ctx context.Context, kv jetstream.KeyValue, family string) (int, error) {
 	var latest int
-	found, err := getJSON(ctx, kv, LatestRevKey(family), &latest)
+	found, err := GetJSON(ctx, kv, LatestRevKey(family), &latest)
 	if err != nil {
 		return 0, err
 	}
@@ -88,7 +88,7 @@ func nextRevision(ctx context.Context, kv jetstream.KeyValue, family string) (in
 // Deregister marks one revision INACTIVE; the revision stays describable.
 func (o *Owner) Deregister(ctx context.Context, kv jetstream.KeyValue, family string, rev int) (*Record, error) {
 	var rec Record
-	found, err := getJSON(ctx, kv, RevKey(family, rev), &rec)
+	found, err := GetJSON(ctx, kv, RevKey(family, rev), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +100,7 @@ func (o *Owner) Deregister(ctx context.Context, kv jetstream.KeyValue, family st
 		rec.DeregisteredAt = &now
 	}
 	rec.Status = StatusInactive
-	if err := putJSON(ctx, kv, RevKey(family, rev), &rec); err != nil {
+	if err := PutJSON(ctx, kv, RevKey(family, rev), &rec); err != nil {
 		return nil, err
 	}
 	return &rec, nil
@@ -115,7 +115,7 @@ func (o *Owner) Resolve(ctx context.Context, kv jetstream.KeyValue, ref string)
 	}
 	if rev == 0 {
 		var latest int
-		found, err := getJSON(ctx, kv, LatestRevKey(family), &latest)
+		found, err := GetJSON(ctx, kv, LatestRevKey(family), &latest)
 		if err != nil {
 			return nil, err
 		}
@@ -125,7 +125,7 @@ func (o *Owner) Resolve(ctx context.Context, kv jetstream.KeyValue, ref string)
 		rev = latest
 	}
 	var rec Record
-	found, err := getJSON(ctx, kv, RevKey(family, rev), &rec)
+	found, err := GetJSON(ctx, kv, RevKey(family, rev), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -142,7 +142,7 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 	if family != "" {
 		prefix = RevsPrefix(family)
 	}
-	keys, err := keysWithPrefix(ctx, kv, prefix)
+	keys, err := KeysWithPrefix(ctx, kv, prefix)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +152,7 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 			continue
 		}
 		var rec Record
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -163,8 +163,8 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 	return arns, nil
 }
 
-// getJSON reads key into out. Returns (false, nil) when the key is absent.
-func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
+// GetJSON reads key into out. Returns (false, nil) when the key is absent.
+func GetJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
 	entry, err := kv.Get(ctx, key)
 	if errors.Is(err, jetstream.ErrKeyNotFound) {
 		return false, nil
@@ -178,8 +178,8 @@ func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (b
 	return true, nil
 }
 
-// putJSON marshals v and writes it at key.
-func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
+// PutJSON marshals v and writes it at key.
+func PutJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
 	data, err := json.Marshal(v)
 	if err != nil {
 		return err
@@ -190,8 +190,8 @@ func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) erro
 	return nil
 }
 
-// keysWithPrefix returns all KV keys under prefix, sorted.
-func keysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
+// KeysWithPrefix returns all KV keys under prefix, sorted.
+func KeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
 	keys, err := kv.Keys(ctx)
 	if errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, nil

spinifex/domains/ecs/tasks.go

@@ -34,7 +34,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI
 	}
 
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -82,7 +82,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI
 			}
 		}
 
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
 			return nil, err
 		}
 		if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil {
@@ -166,7 +166,7 @@ func (s *Service) provisionTaskENI(ctx context.Context, kv jetstream.KeyValue, a
 			rec.DesiredStatus = TaskStatusStopped
 			rec.StoppedReason = fmt.Sprintf("ENI attach failed: %s", err)
 			rec.StoppedAt = time.Now().UTC()
-			if perr := putJSON(ctx, kv, TaskKey(cluster, rec.TaskID), rec); perr != nil {
+			if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, rec.TaskID), rec); perr != nil {
 				slog.ErrorContext(ctx, "ECS RunTask: persist ENI rollback failed", "task", rec.TaskID, "err", perr)
 			}
 		}
@@ -271,7 +271,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 	for _, c := range td.Containers {
 		msg.Containers = append(msg.Containers, containerToAssign(c))
 	}
-	return putJSON(ctx, kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg)
+	return taskdefinition.PutJSON(ctx, kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg)
 }
 
 // DescribeTasks returns task records for the named tasks in a cluster.
@@ -285,7 +285,7 @@ func (s *Service) DescribeTasks(ctx context.Context, input *ecs.DescribeTasksInp
 	for _, ref := range awsStringSlice(input.Tasks) {
 		taskID := ContainerInstanceShortID(ref)
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -308,14 +308,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
 	if !found {
 		return nil, errors.New(awserrors.ErrorECSClusterNotFound)
 	}
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
@@ -323,7 +323,7 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 	arns := make([]string, 0, len(keys))
 	for _, k := range keys {
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/tasks_gc.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -52,7 +53,7 @@ func (sc *Scheduler) sweepStoppedTasks(ctx context.Context) (time.Duration, erro
 // the ENI release for any that still owe one. A missing StoppedAt or an owed
 // (unreleased) ENI both block deletion; the latter until the release succeeds.
 func (s *Service) sweepStoppedBucket(ctx context.Context, kv jetstream.KeyValue, accountID string, now time.Time, retention time.Duration) (int, time.Duration, error) {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return 0, 0, err
 	}
@@ -64,7 +65,7 @@ func (s *Service) sweepStoppedBucket(ctx context.Context, kv jetstream.KeyValue,
 			continue
 		}
 		var task TaskRecord
-		found, gerr := getJSON(ctx, kv, k, &task)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, k, &task)
 		if gerr != nil || !found {
 			continue
 		}
@@ -116,7 +117,7 @@ func (s *Service) retryTaskENIRelease(ctx context.Context, kv jetstream.KeyValue
 		task.ENIReleaseAttempts++
 		task.ENIReleaseNextTry = now.Add(eniReleaseBackoff(task.ENIReleaseAttempts))
 	}
-	if perr := putJSON(ctx, kv, key, task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, key, task); perr != nil {
 		slog.Warn("ECS sweep: persist ENI retry state failed", "key", key, "err", perr)
 	}
 }

spinifex/domains/ecs/tasks_gc_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -19,7 +20,7 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) {
 			TaskID: id, Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", id),
 			LastStatus: status, DesiredStatus: status, StoppedAt: stoppedAt,
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	put("stale", TaskStatusStopped, now.Add(-2*time.Hour))   // older than retention -> prune
@@ -36,7 +37,7 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) {
 
 	exists := func(id string) bool {
 		var rec TaskRecord
-		found, gerr := getJSON(t.Context(), kv, TaskKey("web", id), &rec)
+		found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", id), &rec)
 		require.NoError(t, gerr)
 		return found
 	}
@@ -58,7 +59,7 @@ func TestSweepStoppedBucket_RetriesOwedENI_Succeeds(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-1", ENIAttachmentID: "att-1",
 		StoppedAt: now.Add(-2 * time.Hour), // well past retention
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -67,7 +68,7 @@ func TestSweepStoppedBucket_RetriesOwedENI_Succeeds(t *testing.T) {
 	assert.Equal(t, 1, eni.releaseCalls)
 
 	var got TaskRecord
-	found, gerr := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
+	found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
 	require.NoError(t, gerr)
 	require.True(t, found)
 	assert.Equal(t, "eni-1", got.ENIID, "the identity survives release as the forensic record")
@@ -88,7 +89,7 @@ func TestSweepStoppedBucket_RetryFails_BacksOff(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-1",
 		StoppedAt: now.Add(-2 * time.Hour),
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -96,7 +97,7 @@ func TestSweepStoppedBucket_RetryFails_BacksOff(t *testing.T) {
 	assert.Equal(t, eniReleaseBackoffBase, due)
 
 	var got TaskRecord
-	found, gerr := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
+	found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
 	require.NoError(t, gerr)
 	require.True(t, found)
 	assert.Equal(t, "eni-1", got.ENIID, "a failed retry is still owed")
@@ -118,7 +119,7 @@ func TestSweepStoppedBucket_RetryNotDueYet_Skipped(t *testing.T) {
 		StoppedAt:         now.Add(-2 * time.Hour),
 		ENIReleaseNextTry: now.Add(10 * time.Minute),
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -142,7 +143,7 @@ func TestSweepStoppedBucket_HonoursPerPassCap(t *testing.T) {
 			NetworkMode: NetworkModeAwsvpc, ENIID: "eni-" + id,
 			StoppedAt: now.Add(-2 * time.Hour),
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	_, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
@@ -173,7 +174,7 @@ func TestSweepStoppedBucket_CappedOverflow_AsksForSweepInterval(t *testing.T) {
 			NetworkMode: NetworkModeAwsvpc, ENIID: "eni-" + id,
 			StoppedAt: now.Add(-2 * time.Hour),
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	_, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
@@ -196,7 +197,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-released", ENIReleased: true,
 		StoppedAt: now.Add(-2 * time.Hour), // well past retention
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "released"), &released))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "released"), &released))
 
 	owed := TaskRecord{
 		TaskID: "owed", Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", "owed"),
@@ -204,7 +205,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-owed", ENIReleased: false,
 		StoppedAt: now.Add(-2 * time.Hour), // also well past retention, but still owed
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "owed"), &owed))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "owed"), &owed))
 
 	pruned, _, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -212,7 +213,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 
 	exists := func(id string) bool {
 		var rec TaskRecord
-		found, gerr := getJSON(t.Context(), kv, TaskKey("web", id), &rec)
+		found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", id), &rec)
 		require.NoError(t, gerr)
 		return found
 	}

spinifex/domains/eks/awsapi/registration.go

@@ -3,11 +3,9 @@ package awsapi
 import (
 	"encoding/json"
 	"errors"
-	"io"
 	"log/slog"
 	"net/http"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -64,17 +62,6 @@ func CallerFromActor(actor dispatch.Actor) Caller {
 	}
 }
 
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 // dispatch runs the action and renames the generic parameter error to the code
 // EKS declares. The internal routes keep theirs: the CP-VM agents match on it.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
@@ -139,7 +126,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "EKS dispatch: failed to read body", "err", err)
 		return err

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -205,6 +205,8 @@ func TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete(t *testing.T) {
 		ClusterName: aws.String("alpha"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
 	f.svc.WaitTeardowns()
 
 	_, err = GetClusterMeta(t.Context(), f.kv, "alpha")

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -105,6 +105,9 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
 	// production 10m timeout; tests drive meta.NodeCount to simulate the reconciler.
+	svc.noDeleteKick = true
+	svc.workersEndWait = 20 * time.Millisecond
+	svc.workersEndPoll = time.Millisecond
 	svc.nodegroupReadyTimeout = 50 * time.Millisecond
 	svc.nodegroupReadyPoll = 2 * time.Millisecond
 	// Tight worker-launch-retry knobs so a launch-failure retry test resolves

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -0,0 +1,284 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A DELETING nodegroup becomes DELETE_FAILED once it has been deleting this
+// long and this many passes have failed. Vars for tests.
+var (
+	nodegroupDeleteFailAfter    = 30 * time.Minute
+	nodegroupDeleteFailAttempts = 5
+)
+
+// A pass waits this long, polling, for ended workers to be proven gone.
+const (
+	defaultWorkersEndWait = 90 * time.Second
+	defaultWorkersEndPoll = 3 * time.Second
+)
+
+// errWorkersEnding means workers were told to end but are not yet proven gone.
+var errWorkersEnding = errors.New("workers still ending")
+
+// EKSNodegroupDeletingReaper finishes recorded nodegroup deletes: it ends the
+// workers, removes the nodegroup's own instance profile and only then the
+// record, so a nodegroup is never gone while something it made still is.
+type EKSNodegroupDeletingReaper struct {
+	svc *EKSServiceImpl
+}
+
+var _ vm.Reaper = (*EKSNodegroupDeletingReaper)(nil)
+
+// NewNodegroupDeletingReaper builds the DELETING nodegroup converger.
+func (s *EKSServiceImpl) NewNodegroupDeletingReaper() *EKSNodegroupDeletingReaper {
+	return &EKSNodegroupDeletingReaper{svc: s}
+}
+
+func (r *EKSNodegroupDeletingReaper) Class() string         { return "eks-nodegroup-deleting" }
+func (r *EKSNodegroupDeletingReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep hands every due DELETING nodegroup to a background converger and
+// returns how many it started. The lease decides which node runs each one.
+func (r *EKSNodegroupDeletingReaper) Sweep(ctx context.Context) (int, error) {
+	if !r.svc.depsReadyForOrchestration() {
+		return 0, nil
+	}
+	js, err := jetstream.New(r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("jetstream: %w", err)
+	}
+	names, err := accountBucketNames(ctx, r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("enumerate account buckets: %w", err)
+	}
+
+	started := 0
+	for _, name := range names {
+		select {
+		case <-ctx.Done():
+			return started, ctx.Err()
+		default:
+		}
+		accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix)
+		acctKV, err := js.KeyValue(ctx, name)
+		if err != nil {
+			continue
+		}
+		clusters, err := listClusterNames(ctx, acctKV)
+		if err != nil {
+			continue
+		}
+		for _, cluster := range clusters {
+			recs, err := ListNodegroupRecords(ctx, acctKV, cluster)
+			if err != nil {
+				continue
+			}
+			for _, rec := range recs {
+				if r.due(rec) && r.svc.startNodegroupDelete(accountID, acctKV, cluster, rec.Name) {
+					started++
+				}
+			}
+		}
+	}
+	return started, nil
+}
+
+// due reports whether rec is DELETING and past its retry backoff.
+func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return false
+	}
+	if rec.DeleteReapAttempts == 0 || rec.LastDeleteReapAttempt.IsZero() {
+		return true
+	}
+	return time.Since(rec.LastDeleteReapAttempt) >= deleteReapBackoff(deletingReapMinAge, rec.DeleteReapAttempts)
+}
+
+// startNodegroupDelete runs one converger pass in the background and reports
+// whether it started; a pass already running here or on another node refuses.
+func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
+	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	if !ok {
+		return false
+	}
+	s.teardownWG.Go(func() {
+		defer release()
+		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
+		defer cancel()
+		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
+			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
+				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+		}
+	})
+	return true
+}
+
+// WaitNodegroupDeletes blocks until every converger this process started has
+// returned. Intended for tests.
+func (s *EKSServiceImpl) WaitNodegroupDeletes() { s.teardownWG.Wait() }
+
+// mutateNodegroup applies fn to the stored record under compare-and-swap. fn
+// returns false to leave the record unchanged.
+func (s *EKSServiceImpl) mutateNodegroup(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, fn func(*NodegroupRecord) bool) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			return nil, err
+		}
+		if !fn(rec) {
+			return rec, nil
+		}
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, fmt.Errorf("nodegroup %s/%s: concurrent writers kept winning", cluster, ng)
+}
+
+// convergeNodegroupDeletion finishes one recorded delete. It acts only on a
+// record that is DELETING when re-read, and keeps the record on any failure.
+func (s *EKSServiceImpl) convergeNodegroupDeletion(ctx context.Context, accountID string, kv jetstream.KeyValue, cluster, ng string) error {
+	rec, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.DeleteReapAttempts++
+		r.LastDeleteReapAttempt = time.Now().UTC()
+		return true
+	})
+	if err != nil {
+		if errors.Is(err, ErrNodegroupNotFound) {
+			return nil
+		}
+		return err
+	}
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return nil
+	}
+
+	if err := s.releaseNodegroupResources(ctx, accountID, kv, rec); err != nil {
+		if errors.Is(err, errWorkersEnding) {
+			s.noteWorkersEnding(ctx, kv, cluster, ng, err)
+			return err
+		}
+		s.noteNodegroupDeleteFailure(ctx, kv, rec, err)
+		return err
+	}
+	if err := DeleteNodegroupRecord(ctx, kv, cluster, ng); err != nil {
+		return err
+	}
+	// A cluster delete waits on its node groups; the last one going lets it run.
+	if meta, err := GetClusterMeta(ctx, kv, cluster); err == nil && meta.Status == ClusterStatusDeleting {
+		if rest, err := ListNodegroupRecords(ctx, kv, cluster); err == nil && len(rest) == 0 {
+			s.startTeardown(accountID, cluster)
+		}
+	}
+	return nil
+}
+
+// releaseNodegroupResources ends the workers, then removes the profile. The
+// profile and the recorded worker ids go only once every worker is proven
+// terminated or absent, since an accepted terminate is only a request.
+func (s *EKSServiceImpl) releaseNodegroupResources(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	if len(rec.InstanceIDs) > 0 {
+		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
+			return fmt.Errorf("terminate workers: %w", err)
+		}
+		if err := s.awaitWorkersGone(ctx, rec.InstanceIDs, accountID); err != nil {
+			return err
+		}
+		if _, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+			r.InstanceIDs = nil
+			return true
+		}); err != nil {
+			return fmt.Errorf("record workers ended: %w", err)
+		}
+	}
+	return s.deleteNodegroupProfile(ctx, accountID, kv, rec)
+}
+
+// awaitWorkersGone polls until no worker is live, up to the end wait,
+// and returns errWorkersEnding if some still are.
+func (s *EKSServiceImpl) awaitWorkersGone(ctx context.Context, ids []string, accountID string) error {
+	wait, poll := s.workersEndWait, s.workersEndPoll
+	if wait <= 0 {
+		wait = defaultWorkersEndWait
+	}
+	if poll <= 0 {
+		poll = defaultWorkersEndPoll
+	}
+	deadline := time.Now().Add(wait)
+	for {
+		live, err := s.deps.Worker.LiveWorkerInstances(ctx, ids, accountID)
+		if err != nil {
+			return fmt.Errorf("confirm workers ended: %w", err)
+		}
+		if len(live) == 0 {
+			return nil
+		}
+		if !time.Now().Before(deadline) {
+			return fmt.Errorf("%w: %s", errWorkersEnding, strings.Join(live, ","))
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(poll):
+		}
+	}
+}
+
+// noteWorkersEnding handles workers that are not yet gone. Early on that is no
+// failed attempt: the next sweep checks again without backoff. Once the group
+// has been deleting past nodegroupDeleteFailAfter it settles DELETE_FAILED
+// naming the instances, keeping the profile and record for a customer retry.
+func (s *EKSServiceImpl) noteWorkersEnding(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		if time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+			r.StatusReason = "delete: " + cause.Error()
+			r.ModifiedAt = time.Now().UTC()
+			return true
+		}
+		r.DeleteReapAttempts--
+		r.LastDeleteReapAttempt = time.Time{}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record workers still ending", "cluster", cluster, "nodegroup", ng, "err", err)
+	}
+}
+
+// noteNodegroupDeleteFailure records why a pass failed and, once the group has
+// been deleting long enough and failed often enough, settles it DELETE_FAILED.
+func (s *EKSServiceImpl) noteNodegroupDeleteFailure(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.StatusReason = "delete: " + cause.Error()
+		r.ModifiedAt = time.Now().UTC()
+		if r.DeleteReapAttempts >= nodegroupDeleteFailAttempts && time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+		}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record nodegroup delete failure", "cluster", rec.ClusterName, "nodegroup", rec.Name, "err", err)
+	}
+}

spinifex/domains/eks/cluster/nodegroup.go

@@ -379,6 +379,12 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 		ModifiedAt:     now,
 	}
 
+	// Recorded before the claim, so a name is durable before any worker launches.
+	if rec.NodeRole != "" {
+		rec.InstanceProfileID = uuid.NewV4().String()
+		rec.InstanceProfileName = nodegroupProfilePrefix + rec.InstanceProfileID
+	}
+
 	// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
 	owned, err := ClaimNodegroupRecord(ctx, acctKV, rec)
 	if err != nil {
@@ -716,7 +722,7 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
 	if s.iamEnsurer() != nil && rec.NodeRole != "" {
-		profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+		profileARN, perr := s.nodeProfileForLaunch(accountID, rec)
 		if perr != nil {
 			return "", fmt.Errorf("ensure node instance profile: %w", perr)
 		}
@@ -855,14 +861,10 @@ func gatewayHostIP(gatewayURL string) string {
 	return host
 }
 
-// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
-// nodegroup's node role exists and carries that role, returning its ARN. Workers
-// launch with this profile so IMDS serves the node role to the ECR credential
-// provider, mirroring the implicit instance profile real EKS creates for a node
-// role. Idempotent: concurrent worker launches converge on the same profile.
-func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
-	// Resolution compares the supplied ARN against the stored one, so an invented
-	// path cannot bind a profile to the role its trailing name happens to match.
+// resolveNodeRoleName returns the name of the stored role nodeRoleARN names.
+// Resolution compares the supplied ARN against the stored one, so an invented
+// path cannot bind a profile to the role its trailing name happens to match.
+func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (string, error) {
 	_, roleName, err := auth.ResolveRoleARN(nodeRoleARN, func(account, name string) (string, error) {
 		// The profile is created under the caller's account, so a role from
 		// another account would silently bind a name that account does not own.
@@ -881,6 +883,19 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	if err != nil {
 		return "", fmt.Errorf("node role ARN %q: %w", nodeRoleARN, err)
 	}
+	return roleName, nil
+}
+
+// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
+// nodegroup's node role exists and carries that role, returning its ARN. Workers
+// launch with this profile so IMDS serves the node role to the ECR credential
+// provider, mirroring the implicit instance profile real EKS creates for a node
+// role. Idempotent: concurrent worker launches converge on the same profile.
+func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, nodeRoleARN)
+	if err != nil {
+		return "", err
+	}
 	profileName := roleName
 
 	out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -1018,6 +1033,9 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstrea
 			}
 			return nil, err
 		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
 		applyScalingUpdate(rec, scaling)
 		if labels != nil {
 			rec.Labels = applyLabelUpdate(rec.Labels, labels)
@@ -1227,33 +1245,49 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.K
 	if err := requireCluster(ctx, acctKV, cluster); err != nil {
 		return nil, err
 	}
-	rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng)
+	rec, err := s.markNodegroupDeleting(ctx, acctKV, cluster, ng)
 	if err != nil {
-		if errors.Is(err, ErrNodegroupNotFound) {
-			return nil, nodegroupNotFound(ng)
-		}
 		return nil, err
 	}
-
-	if len(rec.InstanceIDs) > 0 {
-		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
-			return nil, fmt.Errorf("terminate workers: %w", err)
-		}
+	if !s.noDeleteKick {
+		s.startNodegroupDelete(accountID, acctKV, cluster, ng)
 	}
-	if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil {
-		return nil, err
-	}
-	// A cluster delete waits on its node groups; the last one going lets it run.
-	if meta, err := GetClusterMeta(ctx, acctKV, cluster); err == nil && meta.Status == ClusterStatusDeleting {
-		if rest, err := ListNodegroupRecords(ctx, acctKV, cluster); err == nil && len(rest) == 0 {
-			s.startTeardown(accountID, cluster)
-		}
-	}
-
-	rec.Status = eks.NodegroupStatusDeleting
 	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
 }
 
+// markNodegroupDeleting records DELETING and returns. The workers, the profile
+// and the record itself are the nodegroup reaper's to remove, so the request
+// never waits on work that scales with the group (a launching group is refused).
+func (s *EKSServiceImpl) markNodegroupDeleting(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			if errors.Is(err, ErrNodegroupNotFound) {
+				return nil, nodegroupNotFound(ng)
+			}
+			return nil, err
+		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusCreating {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
+		now := time.Now().UTC()
+		rec.Status = eks.NodegroupStatusDeleting
+		rec.StatusReason = ""
+		rec.DeletingSince = now
+		rec.DeleteReapAttempts = 0
+		rec.LastDeleteReapAttempt = time.Time{}
+		rec.ModifiedAt = now
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
 // decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
 func (s *EKSServiceImpl) decryptNodeToken(ctx context.Context, kv jetstream.KeyValue, cluster string) (string, error) {
 	if kv == nil {
@@ -1496,8 +1530,12 @@ func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
 	// wrong; a nil slice here would marshal the key away entirely.
 	issues := make([]*eks.Issue, 0, 1)
 	if rec.StatusReason != "" {
+		code := eks.NodegroupIssueCodeNodeCreationFailure
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			code = eks.NodegroupIssueCodeNodeTerminationFailure
+		}
 		issues = append(issues, &eks.Issue{
-			Code:        aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
+			Code:        aws.String(code),
 			Message:     aws.String(rec.StatusReason),
 			ResourceIds: aws.StringSlice(rec.InstanceIDs),
 		})