test(e2e): attach and detach a volume on a stopped instance, as AWS allows (#1508)

* test: pin guest poweroff handling and stopped-instance volume state

* test: gofmt the poweroff and stopped-volume tests

* test: pin start after detach, one-poll poweroff, reattach of released volumes and the delete refusal

* test: separate the embedded mounter from the recorder fields

* fix: end a guest poweroff as a stop and keep stopped instances' data volumes attached

A clean QEMU exit after the heartbeat saw the guest power off now stops the
instance instead of waiting for crash detection. Unmount no longer releases
data volumes. DetachVolume on a stopped instance edits the record, then
releases the volume. Starting an instance stopped on an older build drops data
volumes that are attached elsewhere or deleted and fails on a missing boot
volume.

* fix: keep the stopped-instance detach inside the instance service file and format start reconcile

* docs: describe guest poweroff, stopped-instance volumes and detach on a stopped instance

* fix: allow detaching the root volume of a stopped instance and refuse a start with no root volume as AWS does

* feat: attach volumes to a stopped instance and fix root-detach start fixtures

* docs: describe attach and root detach on a stopped instance

* test: Unmount seals only and leaves every attached volume in-use

Changed with the person's explicit approval, relayed by dev-design on 11-03.
EBS root and data volumes persist across stop on AWS, so stop and crash
unmount no longer release any attached volume.

* docs: note that the EFI variable store cannot be detached from a stopped instance

* test: unmount contract tests expect no volume state write on stop

* test: order imports in stopped-volume daemon tests

* refactor: drop the unused volume state updater from the volume mounter

contracts/ec2/v1/README.md

@@ -23,7 +23,7 @@ subject shape are wire compatibility. Callers and receivers must use this
 package rather than rebuild those details. `commands_test.go` preserves the
 version-1 JSON shape and subject form.
 
-This package owns shared command payloads and the drain-volume response. It
+This package owns shared command payloads. It
 does not define every action's reply body, authorize the request, store EC2
 state or implement NATS transport. Those remain responsibilities of the EC2
 domain and its runtime composition.

contracts/ec2/v1/commands.go

@@ -29,7 +29,6 @@ type EC2InstanceCommand struct {
 	Attributes                EC2CommandAttributes       `json:"attributes"`
 	AttachVolumeData          *AttachVolumeData          `json:"attach_volume_data,omitempty"`
 	DetachVolumeData          *DetachVolumeData          `json:"detach_volume_data,omitempty"`
-	DrainVolumeData           *DrainVolumeData           `json:"drain_volume_data,omitempty"`
 	AttachENIData             *AttachENIData             `json:"attach_eni_data,omitempty"`
 	DetachENIData             *DetachENIData             `json:"detach_eni_data,omitempty"`
 	IamProfileAssociationData *IamProfileAssociationData `json:"iam_profile_association_data,omitempty"`
@@ -45,7 +44,6 @@ type EC2CommandAttributes struct {
 	StartInstance               bool `json:"start_instance"`
 	AttachVolume                bool `json:"attach_volume"`
 	DetachVolume                bool `json:"detach_volume"`
-	DrainVolume                 bool `json:"drain_volume,omitempty"`
 	RebootInstance              bool `json:"reboot_instance"`
 	AttachENI                   bool `json:"attach_eni"`
 	DetachENI                   bool `json:"detach_eni"`
@@ -69,33 +67,6 @@ type DetachVolumeData struct {
 	Force    bool   `json:"force,omitempty"`
 }
 
-// DrainVolumeData carries parameters for a drain-volume command, which flushes
-// the volume's in-flight writes to S3 before a snapshot reads them. The command
-// is addressed to the instance the volume is attached to because the drain
-// socket only exists on the node hosting it.
-type DrainVolumeData struct {
-	VolumeID string `json:"volume_id"`
-}
-
-const (
-	// DrainVolumeStatusDrained is the ack a node returns once the volume's
-	// in-flight writes have reached S3.
-	DrainVolumeStatusDrained = "drained"
-
-	// DrainVolumeStatusNotRunning is the ack a node returns when it still holds
-	// the instance but the VM is not running, so no process is writing to the
-	// volume and its sealed checkpoint is already current. An attachment record
-	// outlives the writer — stop deliberately leaves boot volumes attached — so
-	// the caller cannot tell this from the record alone.
-	DrainVolumeStatusNotRunning = "not-running"
-)
-
-// DrainVolumeResponse is the reply to a drain-volume command.
-type DrainVolumeResponse struct {
-	VolumeID string `json:"volume_id"`
-	Status   string `json:"status"`
-}
-
 // AttachENIData carries parameters for an attach-network-interface command.
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`

contracts/ec2/v1/commands_test.go

@@ -16,7 +16,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			StartInstance:               true,
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			RebootInstance:              true,
 			AttachENI:                   true,
 			DetachENI:                   true,
@@ -28,7 +27,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
-		DrainVolumeData:           &DrainVolumeData{VolumeID: "vol-3"},
 		AttachENIData:             &AttachENIData{NetworkInterfaceID: "eni-1", DeviceIndex: 2},
 		DetachENIData:             &DetachENIData{AttachmentID: "eni-attach-1", Force: true},
 		IamProfileAssociationData: &IamProfileAssociationData{InstanceProfileArn: "arn:aws:iam::123456789012:instance-profile/example"},
@@ -47,7 +45,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"start_instance": true,
 			"attach_volume": true,
 			"detach_volume": true,
-			"drain_volume": true,
 			"reboot_instance": true,
 			"attach_eni": true,
 			"detach_eni": true,
@@ -59,7 +56,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
-		"drain_volume_data": {"volume_id": "vol-3"},
 		"attach_eni_data": {"network_interface_id": "eni-1", "device_index": 2},
 		"detach_eni_data": {"attachment_id": "eni-attach-1", "force": true},
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
@@ -69,16 +65,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 	}`, string(data))
 }
 
-func TestDrainVolumeResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(DrainVolumeResponse{
-		VolumeID: "vol-1",
-		Status:   DrainVolumeStatusDrained,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"volume_id":"vol-1","status":"drained"}`, string(data))
-	require.Equal(t, "not-running", DrainVolumeStatusNotRunning)
-}
-
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/viperblockd/legacy/v1/README.md

@@ -51,19 +51,6 @@ request/reply replies.
 removed its local state and export. These routes and JSON shapes are deployed
 legacy compatibility behaviour, not a generic EBS-provider interface.
 
-## Fence route
-
-`VolumeFencedSubject(node)` is `ebs.<node>.fenced`, or `ebs.fenced` with no
-node for a single-node daemon. The Viperblock adapter publishes
-`VolumeFencedEvent` on it after tearing down an export whose volume lease
-moved to another node; the local daemon subscribes to stop the guest that
-was using it. Node-addressed, like mount and unmount: the guest is local, and
-no other node's daemon has anything to stop.
-
-The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
-The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
-Repeats are harmless, since fencing a guest that is already fenced changes nothing.
-
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVolumeAbandonSubject_BothForms pins the abandon route the same way the
+// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
-// fenced route is pinned: node-addressed, with an empty-node fallback for a
+// with an empty-node fallback for a single-node daemon.
-// single-node daemon.
 func TestVolumeAbandonSubject_BothForms(t *testing.T) {
 	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
 	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))

contracts/viperblockd/legacy/v1/fence.go

@@ -1,29 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeFencedSubject is where a fenced volume is announced so the daemon on
-// this node can stop the guest that was using it. Node-addressed: the guest is
-// local, and every other node's daemon has nothing to do about it.
-func VolumeFencedSubject(node string) string {
-	if node == "" {
-		return "ebs.fenced"
-	}
-	return "ebs." + node + ".fenced"
-}
-
-// VolumeFencedEvent announces that a node tore down a volume's export because
-// the lease moved. Winner is who holds it now, for the operator reading the
-// guest's stop reason.
-type VolumeFencedEvent struct {
-	Volume string `json:"volume"`
-	Node   string `json:"node"`
-	Winner string `json:"winner"`
-	Reason string `json:"reason"`
-}
-
-// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
-// ends the sender's retries; Instance names the guest stopped, if one was, and
-// Error says why the daemon could not act on the event.
-type VolumeFencedAck struct {
-	Instance string `json:"instance,omitempty"`
-	Error    string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -1,54 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeFencedSubject_IsNodeAddressed pins the routing. A fenced guest is on
-// the node that lost the volume, so a subject without the node in it would reach
-// daemons with nothing to stop and, on a queue group, miss the one that has.
-func TestVolumeFencedSubject_IsNodeAddressed(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.fenced", viperblocklegacyv1.VolumeFencedSubject("node-a"))
-	assert.Equal(t, "ebs.fenced", viperblocklegacyv1.VolumeFencedSubject(""),
-		"a single-node daemon has no node name, and still has to hear its own fences")
-}
-
-// TestVolumeFencedEvent_JSON pins the exact field spelling of the fenced
-// event, published by viperblockd and consumed by the daemon.
-func TestVolumeFencedEvent_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","node":"node-a","winner":"node-b","reason":"volume lease moved to node-b while this node had the volume open"}`
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-1",
-		Node:   "node-a",
-		Winner: "node-b",
-		Reason: "volume lease moved to node-b while this node had the volume open",
-	}
-
-	marshaled, err := json.Marshal(event)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeFencedEvent
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, event, decoded)
-}
-
-// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
-// viperblockd's retries of a fence notice.
-func TestVolumeFencedAck_JSON(t *testing.T) {
-	golden := `{"instance":"i-1","error":"bad event"}`
-	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
-
-	marshaled, err := json.Marshal(ack)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
-	require.NoError(t, err)
-	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
-}

docs/DESIGN.md

@@ -112,14 +112,14 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
+3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
 6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
 
 ```go
-// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
+// Volume mounting through the EBS provider contract (runtime/daemon/vm_adapters.go)
-reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
+published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: req.Name, NodeID: a.node})
 
 // QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
 cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64

docs/compute/launching-instances/README.md

@@ -23,6 +23,7 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
+- [Stopping and Data Volumes](#stopping-and-data-volumes)
 - [Modify Instance Attributes](#modify-instance-attributes)
 - [Instance Metadata Options](#instance-metadata-options)
 - [Spot Instances](#spot-instances)
@@ -73,6 +74,31 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
 aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 ```
 
+## Stopping and Data Volumes
+
+A guest that powers itself off (`poweroff`, `shutdown -h`) goes `stopping` then `stopped`, the same as `stop-instances`.
+It is not restarted automatically.
+`InstanceInitiatedShutdownBehavior=terminate` is not honoured yet: a guest shutdown always stops the instance.
+
+Data volumes stay `in-use` and attached to a stopped instance, and `start-instances` mounts them again.
+While the instance is stopped, `attach-volume` of one of its data volumes to another instance returns `VolumeInUse`, and `delete-volume` is refused.
+
+`detach-volume` works on a stopped instance, with or without `--instance-id`.
+The volume becomes `available` and the next start does not attach it.
+The root volume of a stopped instance can be detached too, as on AWS.
+The internal EFI variable store is not a customer volume: it never appears in `describe-volumes` or `BlockDeviceMappings`, and detaching it is refused with `OperationNotPermitted`.
+The instance then cannot start: `start-instances` returns `InvalidParameterValue` with "Instance does not have a volume attached at root (/dev/sda1)".
+
+`attach-volume` works on a stopped instance too.
+The volume must be `available` and in the instance's availability zone, otherwise the call returns `VolumeInUse` or `InvalidVolume.ZoneMismatch`.
+The attachment is recorded on the stopped instance and the volume becomes `in-use` straight away.
+Attaching at the root device name (for example `/dev/sda1`) makes the volume the root again, and the next `start-instances` boots from it.
+
+An instance stopped by an earlier build had its data volumes released to `available` while its record still listed them.
+On start, a listed data volume that was deleted or is now attached to another instance is dropped with a warning in the daemon log, and the start continues.
+A listed volume that is still `available` and unattached is attached again.
+A missing root volume fails the start with the same `InvalidParameterValue` answer.
+
 ## Modify Instance Attributes
 
 Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
@@ -135,6 +161,30 @@ aws ec2 get-console-output --instance-id $INSTANCE_ID \
   --query 'Output' --output text | base64 -d
 ```
 
+Like AWS, the call returns the most recent 64 KiB of the console.
+While an instance is shutting down after a terminate request, the call reads the renamed file described below.
+Once the instance has left the node's running set, which includes a stopped or terminated instance, the node no longer answers and the call returns `InvalidInstanceID.NotFound`.
+AWS still lists a terminated instance for a while after termination, so a caller may expect this call to work then; it does not here, a known gap.
+Whether the API should answer for a stopped instance is not settled here; only the file on the host is kept.
+
+### Where console logs live and how big they get
+
+The serial console of each instance is written on its host to `/run/spinifex/console-<instance-id>.log`.
+`/run` is memory-backed, so these files are cleared when the host reboots, including any left by older releases.
+The daemon keeps them bounded.
+A file over 4 MiB is cut back in place to its newest 1 MiB, and the guest keeps writing while it happens.
+The console is opened in append mode, so a reboot or auto-restart of the guest adds to the same file instead of starting over.
+A live `console-<instance-id>.log` is deleted only when it has not changed for an hour and no QEMU process on the host has it open, so a stopped instance keeps its console for an hour and a quiet running guest keeps it indefinitely.
+If the daemon cannot read the open files of every QEMU process on the host, it deletes nothing on that pass and logs a warning.
+This also clears the file of an instance that was stopped and then terminated, or that was recovered on another node, within about an hour of its last write.
+When an instance is terminated, including a failed first launch and the system instances behind load balancers, queues and EKS control planes, its file is renamed to `console-<instance-id>.log.terminated`.
+The daemon removes a `.terminated` file an hour after its last write.
+
+Known limits:
+
+- The log shipper follows only `console-*.log`, so it stops following a file when it is renamed on terminate, and the last few seconds of output that it had not yet read can be lost.
+- The instance password retrieval reads only the live file name, so it does not find a console renamed after terminate.
+
 ## Instance Types
 
 List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.

internal/testkit/drainsocket.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,10 +29,11 @@ func SocketTempDir(t *testing.T) string {
 // StartDrainSocket stands in for the NBD plugin's drain socket, at the path the
 // plugin serves it on the node hosting volumeID ({dataDir}/viperblock/{volumeID}
 // /snapshot.sock). Every connection is answered with ack ("OK\n" for a drain
-// that reached S3, "ERR\n" for one that did not) and closed.
+// that reached S3, "ERR\n" for one that did not) and closed. connections
-func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) {
+// reports how many drains have been asked for so far.
+func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) (connections func() int) {
 	t.Helper()
-	serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
+	return serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
 }
 
 // StartBlockingDrainSocket stands in for a plugin whose flush is still running:
@@ -43,7 +45,7 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 
 	acceptedCh := make(chan struct{})
 	releaseCh := make(chan struct{})
-	serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
+	_ = serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
 
 	var once sync.Once
 	t.Cleanup(func() { once.Do(func() { close(releaseCh) }) })
@@ -52,8 +54,9 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 }
 
 // serveDrainSocket answers every connection with ack, optionally closing
-// accepted on the first one and waiting for release before it replies.
+// accepted on the first one and waiting for release before it replies. The
-func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) {
+// returned func counts the connections accepted so far.
+func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) func() int {
 	t.Helper()
 
 	dir := filepath.Join(dataDir, "viperblock", volumeID)
@@ -63,6 +66,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = ln.Close() })
 
+	var count atomic.Int64
 	go func() {
 		var once sync.Once
 		for {
@@ -70,6 +74,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			if err != nil {
 				return // listener closed by the cleanup above
 			}
+			count.Add(1)
 			if accepted != nil {
 				once.Do(func() { close(accepted) })
 			}
@@ -80,4 +85,5 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			_ = conn.Close()
 		}
 	}()
+	return func() int { return int(count.Load()) }
 }

scripts/images/eks-node/eks-node-role.sh

@@ -159,6 +159,7 @@ case "${ROLE}" in
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     server-join)
@@ -178,12 +179,14 @@ case "${ROLE}" in
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     agent)
         log "configuring agent role"
         svc_enable k3s-agent
         svc_start k3s-agent
+        svc_optional mulga-eks-log-forward
         ;;
     *)
         die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"

scripts/images/eks-node/eks-node-role_test.sh

@@ -146,6 +146,15 @@ for INIT in openrc systemd; do
     )"
     check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
     check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
@@ -196,6 +205,15 @@ for INIT in openrc systemd; do
     # published the cluster-identical artifacts, and a join re-publish would
     # only race the bootstrap bus. This is deliberate, not an oversight.
     check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
+    check "server-join(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
     check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
@@ -207,8 +225,39 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+    check "agent(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): log-forward is enabled after k3s-agent" "yes" "$(
+        awk -v k="$(enable_cmd "${INIT}" k3s-agent)" -v f="$(enable_cmd "${INIT}" mulga-eks-log-forward)" '
+            index($0, k) == 1 { kn = NR }
+            index($0, f) == 1 { fn = NR }
+            END { print (kn > 0 && fn > kn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
+
     check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
+    # A broken log forwarder must never stop a node joining.
+    for FAILING in enable start; do
+        run_case "${INIT}" agent_logfwd_${FAILING}_fail
+        printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
+        if [ "${FAILING}" = enable ]; then
+            STUB_FAIL="$(enable_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
+        else
+            STUB_FAIL="$(start_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
+        fi
+        invoke
+        unset STUB_FAIL
+        check "agent(${INIT}): log-forward ${FAILING} failure still exits 0" 0 "${rc}"
+        check "agent(${INIT}): log-forward ${FAILING} failure still enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still starts k3s-agent" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
+    done
+
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred
     printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"

scripts/images/eks-node/manifest.conf

@@ -45,6 +45,8 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
 scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward \
+scripts/images/eks-node/mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-log-forward.initd

@@ -0,0 +1,30 @@
+#!/sbin/openrc-run
+# mulga-eks-log-forward — copies k3s warnings and errors to the serial console
+# so they reach the Elasticsearch sink. Best effort: nothing depends on it and
+# a failure here never affects the node.
+
+description="Forward k3s warnings and errors to the serial console"
+command="/bin/sh"
+command_background="yes"
+supervisor="supervise-daemon"
+respawn_delay=5
+pidfile="/run/mulga-eks-log-forward.pid"
+output_log="/dev/null"
+error_log="/dev/null"
+
+depend() {
+    after k3s k3s-agent
+}
+
+start_pre() {
+    role=$(cat /etc/spinifex-eks/role 2>/dev/null || true)
+    if [ "${role}" = "agent" ]; then
+        log=/var/log/k3s-agent.log
+        FORWARD_PREFIX="[k3s-agent]"
+    else
+        log=/var/log/k3s.log
+        FORWARD_PREFIX="[k3s]"
+    fi
+    export FORWARD_PREFIX
+    command_args="-c 'tail -n +1 -F ${log} 2>/dev/null | /usr/local/sbin/mulga-eks-log-forward'"
+}

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# mulga-eks-log-forward — copy the warnings and errors of a k3s log to the
+# serial console, where the host picks them up for the Elasticsearch sink.
+#
+# Reads the tailed log on stdin. Lines that matter are redacted first, then
+# cut to FORWARD_MAX_BYTES, then rate limited: at most FORWARD_RATE_PER_MIN a
+# minute reach the console and the rest are reported as one "N lines dropped"
+# line. Every console write has a timeout, so a stuck console loses lines
+# instead of stalling the tail or k3s.
+#
+# Env: FORWARD_CONSOLE (default /dev/console), FORWARD_PREFIX, FORWARD_MAX_BYTES
+# (512), FORWARD_RATE_PER_MIN (60), FORWARD_WRITE_TIMEOUT seconds (2) and
+# FORWARD_CLOCK_CMD (default: seconds of uptime, read without forking).
+
+CONSOLE=${FORWARD_CONSOLE:-/dev/console}
+PREFIX=${FORWARD_PREFIX:-[k3s]}
+MAX_BYTES=${FORWARD_MAX_BYTES:-512}
+RATE=${FORWARD_RATE_PER_MIN:-60}
+WRITE_TIMEOUT=${FORWARD_WRITE_TIMEOUT:-2}
+CLOCK_CMD=${FORWARD_CLOCK_CMD:-}
+
+now() {
+    if [ -n "${CLOCK_CMD}" ]; then
+        sh -c "${CLOCK_CMD}"
+    else
+        read -r up _ < /proc/uptime && echo "${up%%.*}"
+    fi
+}
+
+# emit TEXT: one console line, abandoned after WRITE_TIMEOUT. Returns non-zero
+# when the console did not take it.
+emit() {
+    timeout "${WRITE_TIMEOUT}" sh -c 'printf "%s\n" "$1" >> "$2"' sh "$1" "${CONSOLE}" 2>/dev/null
+}
+
+# Filter, redact and truncate in one awk pass so lines that do not matter cost
+# no fork. Redaction runs before the cut so a secret straddling the limit
+# cannot leave a readable prefix behind.
+select_lines() {
+    LC_ALL=C awk -v max="${MAX_BYTES}" '
+    {
+        line = $0
+        gsub(/\r/, "", line)
+        keep = 0
+        if (line ~ /level=(warning|error)/) keep = 1
+        else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
+        else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
+        if (!keep) next
+
+        gsub(/[Bb]earer +[A-Za-z0-9._~+\/=-]+/, "Bearer [REDACTED]", line)
+        gsub(/K10[0-9a-f]+(::[A-Za-z0-9-]+:[^ "\047]+)?/, "[REDACTED]", line)
+        gsub(/eyJ[A-Za-z0-9_.-]+/, "[REDACTED]", line)
+        gsub(/(AKIA|ASIA)[A-Z0-9]+/, "[REDACTED]", line)
+        gsub(/X-Amz-Signature=[^& "\047]*/, "X-Amz-Signature=[REDACTED]", line)
+        gsub(/X-Amz-Credential=[^& "\047]*/, "X-Amz-Credential=[REDACTED]", line)
+        gsub(/X-Amz-Security-Token=[^& "\047]*/, "X-Amz-Security-Token=[REDACTED]", line)
+        gsub(/[^[:print:]\t]/, " ", line)
+        print substr(line, 1, max)
+    }'
+}
+
+win_start=0
+sent=0
+dropped=0
+
+flush_dropped() {
+    [ "${dropped}" -gt 0 ] || return 0
+    emit "${PREFIX} ${dropped} lines dropped"
+    dropped=0
+}
+
+select_lines | {
+    while IFS= read -r line; do
+        t=$(now) || t=0
+        if [ "${win_start}" -eq 0 ] || [ $((t - win_start)) -ge 60 ]; then
+            flush_dropped
+            win_start=${t}
+            sent=0
+            [ "${win_start}" -eq 0 ] && win_start=1
+        fi
+        if [ "${sent}" -ge "${RATE}" ]; then
+            dropped=$((dropped + 1))
+            continue
+        fi
+        sent=$((sent + 1))
+        emit "${PREFIX} ${line}" || dropped=$((dropped + 1))
+    done
+    flush_dropped
+}
+exit 0

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -0,0 +1,190 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-log-forward.sh: the line filter,
+# redaction, truncation, rate cap and blocked-console behaviour. The script
+# reads the tailed log on stdin and writes to FORWARD_CONSOLE, which the test
+# points at a temp file (or a reader-less FIFO), so no root or tty is needed.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-log-forward_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-log-forward.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+check() { # check <desc> <expected> <actual>
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        echo "FAIL: $1"
+        echo "  expected: [$2]"
+        echo "  actual:   [$3]"
+    fi
+}
+
+# forward <name>: feeds stdin to the script with a per-case console file and a
+# fixed clock, leaving the forwarded text in ${OUT}.
+forward() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}"
+    OUT="${CASE}/console"
+    : > "${OUT}"
+    FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" \
+        FORWARD_CLOCK_CMD="echo 1000" sh "${SCRIPT}" 2>/dev/null || true
+}
+forwarded() { grep -c . "${OUT}" || true; }
+
+# --- filter: lines that must pass ---
+forward filter_pass <<'EOF'
+time="2026-11-03T03:00:00Z" level=warning msg="slow response from apiserver"
+time="2026-11-03T03:00:01Z" level=error msg="failed to renew"
+W1103 03:00:02.123456    1234 reflector.go:561] watch of *v1.Pod ended
+E1103 03:00:03.123456    1234 kubelet.go:2400] node not found
+I1103 03:00:04.123456    1234 kubelet_node_status.go:73] Node status update failed
+I1103 03:00:05.123456    1234 controller.go:193] Failed to update lease
+I1103 03:00:06.123456    1234 kubelet.go:2200] PLEG is not healthy
+time="2026-11-03T03:00:07Z" level=info msg="Event: NodeNotReady kubelet"
+EOF
+check "filter: all eight interesting lines pass" "8" "$(forwarded)"
+check "filter: lines carry the prefix" "8" "$(grep -c '^\[k3s-agent\] ' "${OUT}" || true)"
+
+# --- filter: lines that must be dropped ---
+forward filter_drop <<'EOF'
+time="2026-11-03T03:00:00Z" level=info msg="Starting k3s agent"
+time="2026-11-03T03:00:01Z" level=debug msg="tick"
+I1103 03:00:02.123456    1234 server.go:100] Version: v1.32.5
+I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
+EOF
+check "filter: info and debug lines are dropped" "0" "$(forwarded)"
+
+# --- redaction: one case per secret shape ---
+redact() { # redact <name> <line> <secret-that-must-not-appear>
+    forward "redact_$1" <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="$2"
+EOF
+    check "redact(${1}): line still forwarded" "1" "$(forwarded)"
+    if grep -qF "$3" "${OUT}"; then
+        check "redact(${1}): secret removed" "absent" "present"
+    else
+        check "redact(${1}): secret removed" "absent" "absent"
+    fi
+    check "redact(${1}): marker present" "1" "$(grep -c 'REDACTED' "${OUT}" || true)"
+}
+redact bearer "request failed Authorization: Bearer abc123SECRETvalue456" "abc123SECRETvalue456"
+redact k3s_token "join failed token K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" "s3cr3tPassw0rd"
+redact jwt "bad token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.c2lnbmF0dXJlLXNlY3JldA" "eyJhbGciOiJIUzI1NiJ9"
+redact akia "credentials AKIAIOSFODNN7EXAMPLE rejected" "AKIAIOSFODNN7EXAMPLE"
+redact asia "credentials ASIAIOSFODNN7EXAMPLE rejected" "ASIAIOSFODNN7EXAMPLE"
+redact amz_signature "GET /obj?X-Amz-Signature=0123456789abcdef0123456789abcdef&X-Amz-Expires=60 failed" "0123456789abcdef0123456789abcdef"
+redact amz_credential "GET /obj?X-Amz-Credential=ZZZsecretscope%2F20261103%2Fap%2Fs3%2Faws4_request failed" "ZZZsecretscope"
+
+# --- truncation: text over 512 bytes is cut ---
+LONG=$(awk 'BEGIN { for (i = 0; i < 2000; i++) printf "a"; }')
+forward truncate <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="${LONG}"
+EOF
+check "truncate: still forwarded" "1" "$(forwarded)"
+BODY=$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1)
+check "truncate: body is exactly 512 bytes" "512" "$(printf '%s' "${BODY}" | wc -c | tr -d ' ')"
+check "truncate: keeps the start of the line" "time=\"2026-11-03T03:00:00Z\"" "$(printf '%s' "${BODY}" | cut -c1-27)"
+
+# --- redaction runs before the cut: a secret over the limit leaves no prefix ---
+# The pad puts the 512-byte cut right after the bare marker (AKIA, eyJ, K10).
+# A cut fragment that long matches no redaction pattern, so redacting after the
+# cut would leave it in the output.
+HEAD='time="2026-11-03T03:00:00Z" level=error msg="'
+straddle() { # straddle <name> <secret> <bytes of the secret kept by the cut>
+    PAD=$(awk -v n=$((512 - ${#HEAD} - 1 - $3)) 'BEGIN { for (i = 0; i < n; i++) printf "x"; }')
+    forward "straddle_$1" <<EOF
+${HEAD}${PAD} $2 tail"
+EOF
+    check "straddle(${1}): line forwarded" "1" "$(forwarded)"
+    check "straddle(${1}): cut keeps the marker length" "$3" "$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1 | awk -v p="${PAD} " '{ print length($0) - index($0, p) - length(p) + 1 }')"
+    if grep -qF "$(printf '%s' "$2" | cut -c1-"$3")" "${OUT}"; then
+        check "straddle(${1}): no fragment survives the cut" "absent" "present"
+    else
+        check "straddle(${1}): no fragment survives the cut" "absent" "absent"
+    fi
+}
+straddle jwt "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2ln" 3
+straddle akia "AKIAIOSFODNN7EXAMPLE" 4
+straddle k3s_token "K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" 3
+
+# --- rate cap: 60 a minute, then one summary of what was dropped ---
+gen() { # gen <count>
+    awk -v n="$1" 'BEGIN { for (i = 0; i < n; i++) printf "time=\"t\" level=error msg=\"loop %d\"\n", i }'
+}
+gen 100 > "${WORK}/burst"
+forward rate_cap < "${WORK}/burst"
+check "rate cap: 60 lines forwarded plus one summary" "61" "$(forwarded)"
+check "rate cap: one dropped summary" "1" "$(grep -c '40 lines dropped' "${OUT}" || true)"
+check "rate cap: first line kept" "1" "$(grep -c 'loop 0"' "${OUT}" || true)"
+check "rate cap: line past the cap is not forwarded" "0" "$(grep -c 'loop 60"' "${OUT}" || true)"
+
+# --- rate cap: a new minute starts a new window and reports the old drops ---
+# The clock command advances 60 s every 70 calls, so lines 0-69 share a window
+# (10 dropped) and lines 70-74 land in the next one.
+CASE="${WORK}/rate_window"
+mkdir -p "${CASE}"
+OUT="${CASE}/console"
+: > "${OUT}"
+echo 0 > "${CASE}/calls"
+cat > "${CASE}/clock" <<EOF
+#!/bin/sh
+n=\$(cat "${CASE}/calls")
+echo \$((n + 1)) > "${CASE}/calls"
+echo \$((1000 + n / 70 * 60))
+EOF
+chmod +x "${CASE}/clock"
+gen 75 | FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="${CASE}/clock" sh "${SCRIPT}" 2>/dev/null || true
+check "window: 60 + 5 forwarded plus one summary" "66" "$(forwarded)"
+check "window: summary names the 10 dropped" "1" "$(grep -c '10 lines dropped' "${OUT}" || true)"
+check "window: new window forwards its lines" "1" "$(grep -c 'loop 74"' "${OUT}" || true)"
+
+# --- blocked console: lines are dropped, the forwarder neither stalls nor fails ---
+CASE="${WORK}/blocked"
+mkdir -p "${CASE}"
+mkfifo "${CASE}/console"
+gen 3 > "${CASE}/in"
+(
+    FORWARD_CONSOLE="${CASE}/console" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="echo 1000" \
+        FORWARD_WRITE_TIMEOUT=1 sh "${SCRIPT}" < "${CASE}/in"
+) &
+PID=$!
+i=0
+while kill -0 "${PID}" 2>/dev/null && [ "${i}" -lt 20 ]; do
+    sleep 1
+    i=$((i + 1))
+done
+if kill -0 "${PID}" 2>/dev/null; then
+    kill "${PID}" 2>/dev/null || true
+    check "blocked console: forwarder finished" "finished" "stalled"
+else
+    check "blocked console: forwarder finished" "finished" "finished"
+fi
+rc=0
+wait "${PID}" || rc=$?
+check "blocked console: exit 0" "0" "${rc}"
+
+# --- the service is wired in, and nothing depends on it ---
+INITD="${SCRIPT_DIR}/mulga-eks-log-forward.initd"
+check "initd ships" "yes" "$([ -f "${INITD}" ] && echo yes || echo no)"
+check "manifest installs the initd" "1" "$(grep -c 'mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+check "manifest installs the script" "1" "$(grep -c 'mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+DEPS=0
+for f in "${SCRIPT_DIR}"/*.initd; do
+    [ "${f}" = "${INITD}" ] && continue
+    if grep -E '^[[:space:]]*(need|use|want)[[:space:]].*mulga-eks-log-forward' "${f}" >/dev/null 2>&1; then
+        DEPS=$((DEPS + 1))
+    fi
+done
+check "nothing depends on the forwarder" "0" "${DEPS}"
+if [ -f "${INITD}" ]; then
+    check "initd does not need anything it could block boot on" "0" "$(grep -cE '^[[:space:]]*need[[:space:]]' "${INITD}" || true)"
+fi
+
+echo "mulga-eks-log-forward_test: ${PASS} passed, ${FAIL} failed"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -45,10 +45,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
     /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
-    /etc/init.d/eks-pod-identity-webhook
+    /etc/init.d/eks-pod-identity-webhook /etc/init.d/mulga-eks-log-forward
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
     /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-node-gc /usr/local/sbin/mulga-eks-log-forward
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

spinifex/domains/ec2/awsapi/volume/AttachVolume.go

@@ -49,6 +49,12 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 		device = *input.Device
 	}
 
+	// The shared record decides: a stopped instance has no QMP to plug into, so
+	// its record is edited and the volume claimed there instead.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return attachToStopped(ctx, natsConn, accountID, input)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -97,6 +103,24 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 	return output, nil
 }
 
+// attachToStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const attachToStoppedSubject = "ec2.AttachVolumeToStopped"
+
+// attachToStopped asks the instance service to add the volume to the stopped
+// instance's record; the volume is claimed in-use by the same call.
+func attachToStopped(ctx context.Context, natsConn *nats.Conn, accountID string, input *ec2.AttachVolumeInput) (ec2.VolumeAttachment, error) {
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, attachToStoppedSubject, input, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolume: stopped-instance attach failed", "volumeId", aws.StringValue(input.VolumeId), "instanceId", aws.StringValue(input.InstanceId), "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}
+
 // isStoppedInstance checks the shared KV (via ec2.DescribeStoppedInstances) to
 // determine whether instanceID exists as a stopped instance.
 func isStoppedInstance(ctx context.Context, instanceID string, natsConn *nats.Conn, accountID string) bool {

spinifex/domains/ec2/awsapi/volume/AttachVolume_stopped_test.go

@@ -0,0 +1,55 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The record decides the path: a stopped instance has no QMP to plug into, so
+// the attach goes to the instance service and ec2.cmd.<id> is never used.
+func TestAttachVolume_StoppedInstanceUsesTheRecordPath(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		attachToStoppedSubject: func(m *nats.Msg) {
+			log.add(attachToStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"),
+				Device: aws.String("/dev/sdf"), State: aws.String("attached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := AttachVolume(context.Background(), &ec2.AttachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+		Device:     aws.String("/dev/sdf"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "attached", aws.StringValue(out.State))
+	assert.Equal(t, []string{attachToStoppedSubject}, log.get(),
+		"only the record path runs for a stopped instance")
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume.go

@@ -74,6 +74,12 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 
 	force := input.Force != nil && *input.Force
 
+	// The shared record decides: a stopped instance has no QMP to unplug from,
+	// so its record is edited first and the volume released after.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return detachFromStopped(ctx, natsConn, accountID, volumeID, instanceID, device)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -131,6 +137,10 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 // running the instance and so cannot help when that host is the problem.
 const forceDetachSubject = "ec2.ForceDetachVolume"
 
+// detachFromStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const detachFromStoppedSubject = "ec2.DetachVolumeFromStopped"
+
 // forceDetach releases a volume whose instance has no responder.
 //
 // A volume outlives the instance it was attached to whenever teardown failed
@@ -153,3 +163,27 @@ func forceDetach(ctx context.Context, natsConn *nats.Conn, accountID, volumeID,
 	}
 	return *attachment, nil
 }
+
+// detachFromStopped removes the volume from the stopped instance's record and
+// then releases the volume, in that order, so a failed release leaves the
+// record clean and the volume still held rather than the reverse.
+func detachFromStopped(ctx context.Context, natsConn *nats.Conn, accountID, volumeID, instanceID, device string) (ec2.VolumeAttachment, error) {
+	in := &ec2.DetachVolumeInput{VolumeId: &volumeID, InstanceId: &instanceID}
+	if device != "" {
+		in.Device = &device
+	}
+	if _, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, detachFromStoppedSubject, in, 30*time.Second, accountID); err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: stopped-instance record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, forceDetachSubject,
+		&ec2.DetachVolumeInput{VolumeId: &volumeID, Force: aws.Bool(true)}, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: release after record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume_stopped_test.go

@@ -0,0 +1,115 @@
+package volume
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type subjectLog struct {
+	mu   sync.Mutex
+	seen []string
+}
+
+func (l *subjectLog) add(s string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.seen = append(l.seen, s)
+}
+
+func (l *subjectLog) get() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.seen...)
+}
+
+func respondJSON(t *testing.T, msg *nats.Msg, v any) {
+	t.Helper()
+	data, err := json.Marshal(v)
+	require.NoError(t, err)
+	require.NoError(t, msg.Respond(data))
+}
+
+// The record decides the path. A stopped instance has no QMP to unplug from, so
+// its record is edited and the volume released, and ec2.cmd.<id> is never used.
+func TestDetachVolume_StoppedInstanceEditsTheRecordThenReleasesTheVolume(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		detachFromStoppedSubject: func(m *nats.Msg) {
+			log.add(detachFromStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), Device: aws.String("/dev/sdf"),
+			})
+		},
+		forceDetachSubject: func(m *nats.Msg) {
+			log.add(forceDetachSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), State: aws.String("detached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "detached", aws.StringValue(out.State))
+	assert.Equal(t, []string{detachFromStoppedSubject, forceDetachSubject}, log.get(),
+		"record first, then the volume; the host path is never used for a stopped instance")
+}
+
+// NoResponders alone must not be read as "stopped": a dead or partitioned node
+// gives the same answer, and editing the record then would race a live QEMU.
+func TestDetachVolume_NoRespondersAloneDoesNotMeanStopped(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	for _, subject := range []string{detachFromStoppedSubject, forceDetachSubject} {
+		sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
+			log.add(subject)
+			_ = m.Respond(nil)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	sub, err := nc.Subscribe("ec2.DescribeStoppedInstances", func(m *nats.Msg) {
+		respondJSON(t, m, &ec2.DescribeInstancesOutput{})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	_, err = DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-silent-node"),
+	}, nc, "000000000001")
+
+	require.Error(t, err)
+	assert.Empty(t, log.get(), "no record edit and no control-plane release without a stopped record")
+}

spinifex/domains/ec2/image/drain_test.go

@@ -1,140 +0,0 @@
-package image
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const drainImageInstanceID = "i-image-drain-host"
-
-// setupDrainImageService mirrors domains/ec2/snapshot's setupDrainService: a
-// reachable (but rejecting) Predastore host so a drained volume falls through
-// into the real snapshot path and fails fast, a data dir with no local drain
-// socket, and a NATS connection to route the drain to the hosting node.
-func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
-	t.Helper()
-
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
-	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
-	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: testBucket,
-		},
-	}
-	svc := NewImageServiceImplWithConfig(cfg, store, nc)
-	return svc, store, nc
-}
-
-func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, testBucket, volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-func drainImageResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-func drainedImageAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-func awaitImageDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// CreateImageFromInstance's IsRunning path must drain the volume's hosting node
-// before it reads the live checkpoint — the same guarantee ec2.CreateSnapshot
-// has. Without this, a checkpoint predating a guest-triggered GPT rewrite
-// (e.g. growpart on first boot) can be captured silently.
-func TestSnapshotRunningVolume_DrainsAttachedVolume(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-drain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-drain", "in-use", drainImageInstanceID)
-	got := drainImageResponder(t, nc, drainImageInstanceID, drainedImageAck(t, "vol-image-drain"))
-
-	err := svc.snapshotRunningVolume("vol-image-drain", "snap-image-drain", testAccountID)
-	// The rejecting Predastore stub fails the snapshot itself; what matters
-	// here is that the drain reached the hosting node first.
-	require.Error(t, err)
-
-	command := awaitImageDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainImageInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-image-drain", command.DrainVolumeData.VolumeID)
-}
-
-// An attached volume whose host reports the drain failed must fail
-// CreateImage rather than register an AMI built from a stale checkpoint.
-func TestSnapshotRunningVolume_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-nodrain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-nodrain", "in-use", drainImageInstanceID)
-	drainImageResponder(t, nc, drainImageInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.snapshotRunningVolume("vol-image-nodrain", "snap-image-nodrain", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// An unattached (stopped-instance) volume takes the no-drain path: the
-// checkpoint Close() left behind is already current.
-func TestSnapshotRunningVolume_AvailableSkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-available", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-available", "available", "")
-	svc.natsConn = nil
-
-	// No responder exists and the socket is never dialled: proceeding straight
-	// to the (rejecting) snapshot path proves the drain was skipped, not stuck.
-	err := svc.snapshotRunningVolume("vol-image-available", "snap-image-available", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -2,16 +2,20 @@ package image
 
 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -96,15 +100,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a
+// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
-// stopped-volume snapshot under a provider is created via provider.CreateSnapshot
+// root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -118,22 +122,22 @@ func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID)
+	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -222,3 +226,98 @@ func TestCreateImageFromInstance_SetsEnaSupportFromInstance(t *testing.T) {
 	require.NotNil(t, ami.EnaSupport)
 	assert.True(t, *ami.EnaSupport)
 }
+
+// createUnavailableProvider refuses every CreateSnapshot as retry-later.
+type createUnavailableProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
+}
+
+// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
+// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
+func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
+	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+}
+
+// createInUseProvider refuses every CreateSnapshot as volume in use.
+type createInUseProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
+}
+
+// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
+// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
+	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, doc.VolumeID)
+	assert.Contains(t, message, "Detach")
+}
+
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// An image of a running instance is snapshotted by the provider, which drains
+// a buffered volume itself, so CreateImage neither dials the drain socket nor
+// asks the instance's host to drain.
+func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *testing.T) {
+	const instanceID = "i-running-drain"
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	_, nc := testutil.StartTestNATS(t)
+	svc.natsConn = nc
+	svc.config.DataDir = testutil.SocketTempDir(t)
+	counting := &countingSnapshotProvider{EBSProvider: provider}
+	svc.SetEBSProvider(counting)
+
+	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
+	doc.State, doc.AttachedInstance = "in-use", instanceID
+	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, doc.VolumeID, "OK\n")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(instanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(instanceID), Name: aws.String("running-ami")},
+		RootVolumeID: doc.VolumeID,
+		IsRunning:    true,
+	}, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, out.ImageId)
+
+	assert.Equal(t, int32(1), counting.creates.Load(), "the provider must take the snapshot")
+	assert.Zero(t, drains(), "CreateImage must not dial the drain socket")
+	assert.Zero(t, commands.Load(), "CreateImage must not ask the instance's host to drain")
+}

spinifex/domains/ec2/image/service_impl.go

@@ -38,7 +38,7 @@ type CreateImageParams struct {
 	Input         *ec2.CreateImageInput
 	RootVolumeID  string
 	SourceImageID string
-	IsRunning     bool // true = use live checkpoint (instance still running), false = use numbered checkpoint from Close
+	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -69,9 +69,7 @@ func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store {
 	return s.metadata
 }
 
-// NewImageServiceImpl creates a new daemon-side image service. natsConn is used
+// NewImageServiceImpl creates a new daemon-side image service.
-// to drain a running instance's volume (routed to the node hosting it) before
-// CreateImageFromInstance reads its live checkpoint.
 func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceImpl {
 	store := objectstore.NewS3ObjectStoreFromConfig(
 		cfg.Predastore.Host,
@@ -99,8 +97,7 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
 }
 
 // NewImageServiceImplWithConfig creates an image service with an explicit
-// config and NATS connection (for testing the running-volume drain path,
+// config, object store and NATS connection (for testing).
-// which needs config.DataDir/Predastore and a NATS connection to route to).
 func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn) *ImageServiceImpl {
 	return &ImageServiceImpl{
 		config:     cfg,
@@ -447,15 +444,9 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume (live via NATS or offline from S3)
+	// Step 1: Snapshot root volume
-	var snapshotErr error
+	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
-	if params.IsRunning {
+		return nil, err
-		snapshotErr = s.snapshotRunningVolume(params.RootVolumeID, snapshotID, accountID)
-	} else {
-		snapshotErr = s.snapshotStoppedVolume(params.RootVolumeID, snapshotID, accountID)
-	}
-	if snapshotErr != nil {
-		return nil, snapshotErr
 	}
 
 	// Step 2: Read source volume config for size
@@ -521,51 +512,21 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}, nil
 }
 
-// snapshotRunningVolume creates a snapshot of a running instance's volume by
+// snapshotVolume snapshots the root volume through the EBS provider, whether
-// draining it to the node hosting it (so any writes buffered there reach S3),
+// or not the instance is running. A provider that buffers writes drains the
-// then reading the live checkpoint written by nbdkit on every NBD Flush. This
+// volume inside its own CreateSnapshot, so no drain is issued here.
-// shares domains/ec2/snapshot's DrainVolume with ec2.CreateSnapshot so the two
+func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
-// live-snapshot paths cannot diverge on when a drain is required.
-func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID string) error {
 	if s.config == nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
 	if err != nil {
-		slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Flush the writes still buffered by whichever node serves the volume, so
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
-	// the snapshot below is current rather than an arbitrarily stale one
-	// predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An
-	// attached volume that cannot be drained fails here rather than silently
-	// producing an image from a stale checkpoint.
-	if err := ec2snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil {
-		slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume")
-}
-
-// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the
-// volume is stopped/detached, so nothing is buffered on a serving node.
-// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening
-// a second one, so it handles the mounted case itself.
-func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume")
 }
 
 // createSnapshot delegates snapshot creation for volMeta to the EBS provider.
@@ -580,7 +541,7 @@ func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetada
 	})
 	if err != nil || created == nil {
 		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
+		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
 	}
 	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
 	return nil

spinifex/domains/ec2/image/service_impl_test.go

@@ -2551,56 +2551,19 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
 
-func TestSnapshotRunningVolume_VolumeConfigMissing(t *testing.T) {
+func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := &ImageServiceImpl{
 		config:     &config.Config{},
 		store:      store,
 		bucketName: testBucket,
 	}
-	err := svc.snapshotRunningVolume("vol-missing", "snap-r1", testAccountID)
+	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotRunningVolume_BackendInitFails(t *testing.T) {
+func TestSnapshotVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running1", 10)
-
-	err := svc.snapshotRunningVolume("vol-running1", "snap-r2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotStoppedVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 	}))
@@ -2620,7 +2583,7 @@ func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped1", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped1", "snap-s2", testAccountID)
+	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -2646,60 +2609,7 @@ func imageS3MockServer(t *testing.T) *httptest.Server {
 	}))
 }
 
-func TestSnapshotRunningVolume_EncryptionKeyLoadError(t *testing.T) {
+func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-run-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-enckey", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-enckey", "snap-run-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotRunningVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-lsf", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-lsf", "snap-run-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -2722,12 +2632,12 @@ func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
+func TestSnapshotVolume_LoadStateFails(t *testing.T) {
 	srv := imageS3MockServer(t)
 	defer srv.Close()
 
@@ -2747,7 +2657,7 @@ func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }

spinifex/domains/ec2/instance/service_impl.go

@@ -2689,15 +2689,6 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 // start returns it to the stopped store, a recovery returns it to the node it
 // was taken from.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
-	// Before anything is stamped or claimed, so the record handed back is the
-	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
-			"instanceId", instance.ID, "err", err)
-		undo()
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
 	instance.ResetNodeLocalState()
 
 	// AWS restamps LaunchTime on every start. The pending watchdog times a
@@ -2709,6 +2700,12 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 	// failure of this launch.
 	instance.FirstLaunch = false
 
+	if err := s.reconcileStartVolumes(ctx, instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: volumes cannot be reconciled", "instanceId", instance.ID, "err", err)
+		undo()
+		return err
+	}
+
 	instanceType, ok := s.resourceMgr.InstanceTypes()[instance.InstanceType]
 	if !ok {
 		slog.ErrorContext(ctx, "launchClaimedInstance: instance type not available on this node",
@@ -3683,3 +3680,153 @@ func (s *InstanceServiceImpl) DescribeInstanceStatus(ctx context.Context, input
 	slog.InfoContext(ctx, "DescribeInstanceStatus completed", "count", len(statuses))
 	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: statuses}, nil
 }
+
+// DetachVolumeFromStopped removes a data volume from a stopped instance's
+// record, so a later start no longer attaches it. It does not touch the
+// volume's own state; the caller releases that once the record is clean. The
+// device the volume held comes back on the attachment.
+func (s *InstanceServiceImpl) DetachVolumeFromStopped(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: stopped store not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	req, ok := findVolumeRequest(stored, volumeID)
+	if !ok {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+	if req.EFI {
+		return nil, errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+
+	var removed bool
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		removed = removeVolumeFromRecord(v, volumeID)
+	})
+	if err != nil {
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "DetachVolumeFromStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if !removed {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(req.DeviceName),
+		State:      aws.String(ec2.VolumeAttachmentStateDetaching),
+	}, nil
+}
+
+// AttachVolumeToStopped adds an available volume to a stopped instance's
+// record, so the next start mounts it. The volume is claimed in-use first and
+// released again if the record edit fails, so a failure never leaves the record
+// naming a volume another instance could take.
+func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *ec2.AttachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil || s.metadata == nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: stores not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to read volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if meta.TenantID != accountID {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+	}
+	if meta.State != "available" {
+		return nil, errors.New(awserrors.ErrorVolumeInUse)
+	}
+	if meta.AvailabilityZone != "" && s.config != nil && s.config.AZ != "" && meta.AvailabilityZone != s.config.AZ {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeZoneMismatch)
+	}
+
+	device, boot, err := stoppedAttachDevice(stored, aws.StringValue(input.Device))
+	if err != nil {
+		return nil, err
+	}
+
+	meta.State = "in-use"
+	meta.AttachedInstance = instanceID
+	meta.DeviceName = device
+	meta.AttachedAt = time.Now()
+	if err := s.metadata.PutVolume(ctx, meta); err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to claim volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	var conflict error
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		// The device was chosen against a snapshot, so it is checked again
+		// against the record the edit is applied to.
+		if _, _, cerr := stoppedAttachDevice(v, device); cerr != nil {
+			conflict = cerr
+			return
+		}
+		addVolumeToRecord(v, volumeID, device, boot)
+	})
+	if err == nil && conflict != nil {
+		err = conflict
+	}
+	if err != nil {
+		meta.State = "available"
+		meta.AttachedInstance = ""
+		meta.DeviceName = ""
+		meta.AttachedAt = time.Time{}
+		if rerr := s.metadata.PutVolume(ctx, meta); rerr != nil {
+			slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to release volume after a failed record edit", "volumeId", volumeID, "err", rerr)
+		}
+		if conflict != nil {
+			return nil, conflict
+		}
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "AttachVolumeToStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(device),
+		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
+	}, nil
+}

spinifex/domains/ec2/instance/service_impl_test.go

@@ -2605,34 +2605,6 @@ func TestStartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
 	assert.Equal(t, vm.StateStopped, store.WroteStopped[id].Status)
 }
 
-// A UEFI instance with no variable store pointer is refused before anything is
-// stamped or claimed, so the record handed back is the one that was claimed.
-func TestStartStoppedInstance_UEFIInstanceWithoutAPointerIsRefused(t *testing.T) {
-	id := "i-uefi-nopointer"
-	itype := "t3.micro"
-	launched := time.Unix(1700000000, 0).UTC()
-	stopped := &vm.VM{
-		ID: id, Status: vm.StateStopped, AccountID: "acc", InstanceType: itype, BootMode: "uefi",
-		Instance: &ec2.Instance{LaunchTime: aws.Time(launched)},
-	}
-	before := stopped.Record()
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stopped}}
-	prov := &fakeResourceCapacityProvider{instanceTypes: map[string]*ec2.InstanceTypeInfo{
-		itype: {InstanceType: aws.String(itype)},
-	}}
-	svc := &InstanceServiceImpl{stoppedStore: store, resourceMgr: prov, vmMgr: vm.NewManager()}
-
-	_, err := svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, "acc")
-
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.Empty(t, prov.allocated, "nothing is allocated for a start that cannot boot")
-	require.NotNil(t, store.WroteStopped[id], "the claimed instance is handed back")
-	assert.Equal(t, before, store.WroteStopped[id].Record(), "the record is unchanged")
-	_, inManager := svc.vmMgr.Get(id)
-	assert.False(t, inManager)
-}
-
 func TestStartStoppedInstance_AllocateFails(t *testing.T) {
 	id := "i-alloc-fail"
 	itype := "t3.micro"

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -0,0 +1,54 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountRecorder notes which volumes a launch asked to mount and then fails the
+// launch, which is the earliest point that shows what the start would attach.
+type mountRecorder struct {
+	refusingMounter
+
+	mu      sync.Mutex
+	mounted []string
+}
+
+func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, req := range v.EBSRequests.Requests {
+		r.mounted = append(r.mounted, req.Name)
+	}
+	return errors.New("stop after recording")
+}
+
+func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+	return nil
+}
+
+// A volume detached while the instance was stopped must not come back when the
+// instance is started: the start path reads the same record the detach edited.
+func TestStartStoppedInstance_DoesNotReattachAVolumeDetachedWhileStopped(t *testing.T) {
+	id := "i-detach-then-start"
+	stopped := stoppedWithVolumes("acc", id)
+	stopped.InstanceType = "t3.micro"
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), "acc")
+	require.NoError(t, err)
+	require.Error(t, f.start(t, id), "the recorder fails the launch after the mount")
+
+	assert.Equal(t, []string{"vol-root", "vol-other"}, rec.mounted,
+		"the detached volume is not part of the start")
+}

spinifex/domains/ec2/instance/start_volumes.go

@@ -0,0 +1,86 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// reconcileStartVolumes drops the data volumes a claimed instance can no longer
+// have, and fails the start when its boot volume is gone.
+func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
+	if s.metadata == nil {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	requests := slices.Clone(instance.EBSRequests.Requests)
+	instance.EBSRequests.Mu.Unlock()
+
+	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+		return noRootVolumeError(instance)
+	}
+
+	var drop []string
+	for _, req := range requests {
+		meta, err := s.metadata.GetVolume(ctx, instance.AccountID, req.Name)
+		missing := err != nil && objectstore.IsNoSuchKeyError(err)
+		if err != nil && !missing {
+			return fmt.Errorf("read volume %s: %w", req.Name, err)
+		}
+		if req.Boot {
+			if missing {
+				slog.ErrorContext(ctx, "start: root volume no longer exists", "instanceId", instance.ID, "volumeId", req.Name)
+				return noRootVolumeError(instance)
+			}
+			continue
+		}
+		if req.EFI {
+			continue
+		}
+		if missing {
+			slog.WarnContext(ctx, "start: dropping deleted data volume", "instanceId", instance.ID, "volumeId", req.Name)
+			drop = append(drop, req.Name)
+			continue
+		}
+		if meta.AttachedInstance != "" && meta.AttachedInstance != instance.ID {
+			slog.WarnContext(ctx, "start: dropping data volume attached to another instance",
+				"instanceId", instance.ID, "volumeId", req.Name, "attachedTo", meta.AttachedInstance)
+			drop = append(drop, req.Name)
+		}
+	}
+	if len(drop) == 0 {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+		return slices.Contains(drop, r.Name)
+	})
+	instance.EBSRequests.Mu.Unlock()
+	if instance.Instance != nil {
+		instance.Instance.BlockDeviceMappings = slices.DeleteFunc(instance.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && slices.Contains(drop, aws.StringValue(m.Ebs.VolumeId))
+		})
+	}
+	return nil
+}
+
+// noRootVolumeError is what AWS answers when an instance is started with
+// nothing attached at its root device name.
+func noRootVolumeError(instance *vm.VM) error {
+	device := ebsmetadata.RootDeviceName
+	if instance.Instance != nil && aws.StringValue(instance.Instance.RootDeviceName) != "" {
+		device = aws.StringValue(instance.Instance.RootDeviceName)
+	}
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Instance does not have a volume attached at root (%s)", device)
+}

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -0,0 +1,125 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const startVolumesAccount = "000000000011"
+
+func startVolumesService(t *testing.T, volumes ...ebsmetadata.Volume) *InstanceServiceImpl {
+	t.Helper()
+	store := objectstore.NewMemoryObjectStore()
+	meta := ebsmetadata.NewStore(store, testRootBucket)
+	for _, v := range volumes {
+		v.TenantID = startVolumesAccount
+		require.NoError(t, meta.PutVolume(context.Background(), v))
+	}
+	return &InstanceServiceImpl{metadata: meta}
+}
+
+func startingInstance(id string, names ...string) *vm.VM {
+	v := &vm.VM{ID: id, AccountID: startVolumesAccount, Status: vm.StateStopped, Instance: &ec2.Instance{InstanceId: aws.String(id)}}
+	for i, n := range names {
+		boot := i == 0
+		device := "/dev/sd" + string(rune('f'+i))
+		if boot {
+			device = "/dev/sda1"
+		}
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
+			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
+		})
+	}
+	return v
+}
+
+func requestedNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+func mappedNames(v *vm.VM) []string {
+	var out []string
+	for _, m := range v.Instance.BlockDeviceMappings {
+		out = append(out, aws.StringValue(m.Ebs.VolumeId))
+	}
+	return out
+}
+
+// An instance stopped on an older build had its data volumes released, and they
+// may since have been attached elsewhere or deleted; starting must not take a
+// volume that is no longer its own.
+func TestReconcileStartVolumes_DropsVolumesThatAreNoLongerTheInstances(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-elsewhere", State: "in-use", AttachedInstance: "i-someone-else"},
+		ebsmetadata.Volume{VolumeID: "vol-free", State: "available"},
+		ebsmetadata.Volume{VolumeID: "vol-mine", State: "in-use", AttachedInstance: "i-old"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-elsewhere", "vol-gone", "vol-free", "vol-mine")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	want := []string{"vol-root", "vol-free", "vol-mine"}
+	assert.Equal(t, want, requestedNames(inst),
+		"attached-elsewhere and deleted volumes are dropped; free and own ones stay")
+	assert.Equal(t, want, mappedNames(inst), "the API view stops listing them too")
+}
+
+func TestReconcileStartVolumes_MissingBootVolumeFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "available"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+}
+
+// A root volume detached while the instance was stopped leaves no boot request
+// in the record; AWS refuses the start with the same answer.
+func TestReconcileStartVolumes_NoRootVolumeAttachedFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: "i-old"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	require.True(t, removeVolumeFromRecord(inst, "vol-root"))
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+	assert.Equal(t, []string{"vol-data"}, requestedNames(inst), "a refused start changes nothing")
+}
+
+// An instance stopped on an older build had its data volumes released to
+// available with no attachment; those are still the instance's own and a
+// start must bring them back.
+func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-released", State: "available"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-released")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
+	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
+}

spinifex/domains/ec2/instance/stopped_attach_test.go

@@ -0,0 +1,180 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const stoppedAttachAZ = "ap-southeast-2a"
+
+type stoppedAttachFixture struct {
+	svc    *InstanceServiceImpl
+	stored *vm.VM
+	id     string
+}
+
+func newStoppedAttachFixture(t *testing.T, volumes ...ebsmetadata.Volume) *stoppedAttachFixture {
+	t.Helper()
+	id := "i-stopped-attach"
+	svc := startVolumesService(t, volumes...)
+	stored := startingInstance(id, "vol-root", "vol-data")
+	svc.config = &config.Config{AZ: stoppedAttachAZ}
+	svc.vmMgr = mgrWith(map[string]*vm.VM{})
+	svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	return &stoppedAttachFixture{svc: svc, stored: stored, id: id}
+}
+
+func (f *stoppedAttachFixture) attach(volumeID, device string) (*ec2.VolumeAttachment, error) {
+	in := &ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String(volumeID)}
+	if device != "" {
+		in.Device = aws.String(device)
+	}
+	return f.svc.AttachVolumeToStopped(context.Background(), in, startVolumesAccount)
+}
+
+func (f *stoppedAttachFixture) volume(t *testing.T, id string) ebsmetadata.Volume {
+	t.Helper()
+	v, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, id)
+	require.NoError(t, err)
+	return v
+}
+
+func availableVolume(id string) ebsmetadata.Volume {
+	return ebsmetadata.Volume{VolumeID: id, State: "available", AvailabilityZone: stoppedAttachAZ}
+}
+
+// AWS lets a volume be attached to a stopped instance; the record then names it
+// so the next start mounts it, and the volume is held in-use meanwhile.
+func TestAttachVolumeToStopped_AddsTheVolumeAndClaimsIt(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	out, err := f.attach("vol-new", "")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device), "the first free data device")
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, requestedNames(f.stored))
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, mappedNames(f.stored))
+	assert.Equal(t, vm.StateStopped, f.stored.Status)
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "in-use", got.State)
+	assert.Equal(t, f.id, got.AttachedInstance)
+	assert.Equal(t, "/dev/sdf", got.DeviceName)
+	for _, r := range f.stored.EBSRequests.Requests {
+		if r.Name == "vol-new" {
+			assert.False(t, r.Boot, "a data device is not the root")
+		}
+	}
+}
+
+func TestAttachVolumeToStopped_Refusals(t *testing.T) {
+	tests := []struct {
+		name    string
+		volumes []ebsmetadata.Volume
+		volume  string
+		device  string
+		want    string
+	}{
+		{"volume in use elsewhere", []ebsmetadata.Volume{{VolumeID: "vol-busy", State: "in-use", AttachedInstance: "i-other", AvailabilityZone: stoppedAttachAZ}}, "vol-busy", "", awserrors.ErrorVolumeInUse},
+		{"wrong availability zone", []ebsmetadata.Volume{{VolumeID: "vol-far", State: "available", AvailabilityZone: "ap-southeast-2b"}}, "vol-far", "", awserrors.ErrorInvalidVolumeZoneMismatch},
+		{"no such volume", nil, "vol-missing", "", awserrors.ErrorInvalidVolumeNotFound},
+		{"device already taken", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sdg", awserrors.ErrorInvalidParameterValue},
+		{"root device already has a volume", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sda1", awserrors.ErrorInvalidParameterValue},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newStoppedAttachFixture(t, tt.volumes...)
+
+			_, err := f.attach(tt.volume, tt.device)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored), "a refused attach changes nothing")
+			for _, v := range tt.volumes {
+				got := f.volume(t, v.VolumeID)
+				assert.Equal(t, v.State, got.State, "a refused attach leaves the volume as it was")
+				assert.Equal(t, v.AttachedInstance, got.AttachedInstance)
+			}
+		})
+	}
+}
+
+func TestAttachVolumeToStopped_OtherAccountsInstanceIsNotFound(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	_, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String("vol-new")}, "000000000099")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound))
+	assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored))
+}
+
+// A record edit that cannot be applied must give the volume back, otherwise it
+// stays in-use against an instance that does not name it.
+func TestAttachVolumeToStopped_FailedRecordEditReleasesTheVolume(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+	f.svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{f.id: f.stored}, UpdateStoppedErr: assert.AnError}
+
+	_, err := f.attach("vol-new", "")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServerInternal))
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "available", got.State)
+	assert.Empty(t, got.AttachedInstance)
+}
+
+// Detaching the root volume of a stopped instance makes it unstartable; the way
+// back is attaching a volume at the root device name, which makes it the boot
+// volume again so the start finds a root.
+func TestAttachVolumeToStopped_RootDeviceNameRestoresTheRootAndTheStartWorks(t *testing.T) {
+	id := "i-root-roundtrip"
+	stopped := stoppedWithVolumes(startVolumesAccount, id)
+	stopped.InstanceType = "t3.micro"
+	stopped.Instance.RootDeviceName = aws.String("/dev/sda1")
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+	f.svc.config = &config.Config{AZ: stoppedAttachAZ}
+	meta := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-other", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+	)
+	f.svc.metadata = meta.metadata
+	stopped.AccountID = startVolumesAccount
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), startVolumesAccount)
+	require.NoError(t, err)
+	vol, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, "vol-root")
+	require.NoError(t, err)
+	vol.State, vol.AttachedInstance = "available", ""
+	require.NoError(t, f.svc.metadata.PutVolume(context.Background(), vol))
+
+	out, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(id), VolumeId: aws.String("vol-root"), Device: aws.String("/dev/sda1")}, startVolumesAccount)
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	var boot bool
+	for _, r := range stopped.EBSRequests.Requests {
+		if r.Name == "vol-root" {
+			boot = r.Boot
+		}
+	}
+	assert.True(t, boot, "the root device name makes it the boot volume")
+
+	_, err = f.svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, startVolumesAccount)
+	require.Error(t, err, "the recorder fails the launch after the mount")
+	assert.Contains(t, rec.mounted, "vol-root", "the start now mounts the restored root")
+}

spinifex/domains/ec2/instance/stopped_detach.go

@@ -0,0 +1,112 @@
+package instance
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 {
+		return viperblocklegacyv1.EBSRequest{}, false
+	}
+	return v.EBSRequests.Requests[i], true
+}
+
+// removeVolumeFromRecord drops the volume from the mount list and the API view
+// of the record, and reports whether it was a data volume that was there.
+func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 || v.EBSRequests.Requests[i].EFI {
+		return false
+	}
+	v.EBSRequests.Requests = slices.Delete(v.EBSRequests.Requests, i, i+1)
+	if v.Instance != nil {
+		v.Instance.BlockDeviceMappings = slices.DeleteFunc(v.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && aws.StringValue(m.Ebs.VolumeId) == volumeID
+		})
+	}
+	return true
+}
+
+// stoppedAttachDevice picks the device a volume takes on a stopped instance and
+// reports whether it is the root. An empty request takes the next free
+// /dev/sd[f-p]; a named device must be free, and the root device name is free
+// only while no boot volume is attached.
+func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
+	root := ebsmetadata.RootDeviceName
+	if v.Instance != nil && aws.StringValue(v.Instance.RootDeviceName) != "" {
+		root = aws.StringValue(v.Instance.RootDeviceName)
+	}
+
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	used := make(map[string]bool, len(v.EBSRequests.Requests))
+	hasBoot := false
+	for _, r := range v.EBSRequests.Requests {
+		if r.DeviceName != "" {
+			used[r.DeviceName] = true
+		}
+		hasBoot = hasBoot || r.Boot
+	}
+	if v.Instance != nil {
+		for _, m := range v.Instance.BlockDeviceMappings {
+			if m != nil && m.DeviceName != nil {
+				used[*m.DeviceName] = true
+			}
+		}
+	}
+
+	if requested == "" {
+		for c := 'f'; c <= 'p'; c++ {
+			if dev := fmt.Sprintf("/dev/sd%c", c); !used[dev] {
+				return dev, false, nil
+			}
+		}
+		return "", false, errors.New(awserrors.ErrorAttachmentLimitExceeded)
+	}
+	if used[requested] {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	boot := requested == root
+	if boot && hasBoot {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return requested, boot, nil
+}
+
+// addVolumeToRecord adds the volume to the mount list and the API view of the
+// record. The root device name makes it the boot volume.
+func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+		Name:       volumeID,
+		DeviceName: device,
+		Boot:       boot,
+	})
+	if v.Instance == nil {
+		return
+	}
+	mapping := &ec2.InstanceBlockDeviceMapping{}
+	mapping.SetDeviceName(device)
+	mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
+	mapping.Ebs.SetVolumeId(volumeID)
+	mapping.Ebs.SetAttachTime(time.Now())
+	mapping.Ebs.SetDeleteOnTermination(false)
+	mapping.Ebs.SetStatus("attached")
+	v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, mapping)
+}

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -0,0 +1,139 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stoppedWithVolumes(owner, id string) *vm.VM {
+	v := &vm.VM{
+		ID: id, AccountID: owner, Status: vm.StateStopped,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(id),
+			BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+				{DeviceName: aws.String("/dev/sda1"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-root")}},
+				{DeviceName: aws.String("/dev/sdf"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-data")}},
+				{DeviceName: aws.String("/dev/sdg"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-other")}},
+			},
+		},
+	}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
+		{Name: "vol-data", DeviceName: "/dev/sdf"},
+		{Name: "vol-other", DeviceName: "/dev/sdg"},
+	}
+	return v
+}
+
+func stoppedDetachInput(instanceID, volumeID string) *ec2.DetachVolumeInput {
+	return &ec2.DetachVolumeInput{InstanceId: aws.String(instanceID), VolumeId: aws.String(volumeID)}
+}
+
+func volumeNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+// AWS lets a stopped instance's data volume be detached; the record must stop
+// claiming it so a later start does not re-attach it.
+func TestDetachVolumeFromStopped_RemovesOnlyThatVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-detach"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device))
+	assert.Equal(t, id, aws.StringValue(out.InstanceId))
+	assert.Equal(t, []string{"vol-root", "vol-other"}, volumeNames(stored))
+	var mapped []string
+	for _, m := range stored.Instance.BlockDeviceMappings {
+		mapped = append(mapped, aws.StringValue(m.Ebs.VolumeId))
+	}
+	assert.Equal(t, []string{"vol-root", "vol-other"}, mapped)
+	assert.Equal(t, vm.StateStopped, stored.Status, "the instance stays stopped")
+}
+
+func TestDetachVolumeFromStopped_Refusals(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	tests := []struct {
+		name    string
+		caller  string
+		volume  string
+		stopped bool
+		want    string
+	}{
+		{"volume not on the instance", owner, "vol-elsewhere", true, awserrors.ErrorIncorrectState},
+		{"another account", "000000000099", "vol-data", true, awserrors.ErrorInvalidInstanceIDNotFound},
+		{"no such stopped instance", owner, "vol-data", false, awserrors.ErrorInvalidInstanceIDNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id := "i-stopped-refuse"
+			stored := stoppedWithVolumes(owner, id)
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{}}
+			if tt.stopped {
+				store.Stopped[id] = stored
+			}
+			svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+			_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, tt.volume), tt.caller)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data", "vol-other"}, volumeNames(stored),
+				"a refused detach changes nothing")
+		})
+	}
+}
+
+// AWS lets the root volume of a stopped instance be detached; the instance then
+// cannot start until a volume is attached at its root device name again.
+func TestDetachVolumeFromStopped_DetachesTheRootVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-root"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	assert.Equal(t, []string{"vol-data", "vol-other"}, volumeNames(stored))
+	assert.Equal(t, vm.StateStopped, stored.Status)
+}
+
+// The EFI variable store is Spinifex-internal and never shown to the customer,
+// so it cannot be detached even by its name.
+func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-efi"
+	stored := stoppedWithVolumes(owner, id)
+	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
+		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-efi"), owner)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorOperationNotPermitted), "got %v", err)
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-other", "vol-efi"}, volumeNames(stored))
+}

spinifex/domains/ec2/snapshot/drain_test.go

@@ -2,12 +2,8 @@ package snapshot
 
 import (
 	"context"
-	"encoding/json"
+	"sync/atomic"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"net/http"
-	"net/http/httptest"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,8 +11,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
@@ -24,321 +18,78 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const drainInstanceID = "i-drain-host"
+const attachedInstanceID = "i-attached-host"
 
-// setupDrainService returns a snapshot service configured like a daemon that
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
-// answers ec2.CreateSnapshot: a reachable Predastore host (so drains are not
+type countingSnapshotProvider struct {
-// skipped as metadata-only), a data dir with no drain socket in it, and a NATS
+	ebsprovider.EBSProvider
-// connection. The Predastore stub rejects everything, so a snapshot that gets
+
-// past the drain fails fast rather than hanging on a dead address.
+	creates atomic.Int32
-func setupDrainService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// setupAttachedSnapshot is a viperblockd node answering CreateSnapshot for a
+// volume attached to a running instance, with a memory provider behind it.
+func setupAttachedSnapshot(t *testing.T, volumeID string) (*SnapshotServiceImpl, *countingSnapshotProvider, *nats.Conn) {
 	t.Helper()
 
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
 	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
+		DataDir:    testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
+		EBS:        config.EBSConfig{Provider: config.EBSProviderViperblockd},
-			Host:   srv.URL,
+		Predastore: config.PredastoreConfig{Host: "127.0.0.1:1", Bucket: "test-bucket"},
-			Bucket: "test-bucket",
-		},
 	}
-	return NewSnapshotServiceImplWithStore(cfg, store, nc), store, nc
+	svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nc)
-}
+	provider := &countingSnapshotProvider{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
 
-// seedVolumeAttachment writes the control-plane state.json that decides whether
-// a snapshot must drain.
-func seedVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, "test-bucket", volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-// drainResponder answers the per-instance command subject the way the node
-// hosting the instance does, and hands each request it received to the test.
-func drainResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-// drainedAck is the reply of a node whose drain reached S3.
-func drainedAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-// requireReturnsWithin fails the test if fn has not returned successfully by
-// limit, so a drain that blocks on a socket it should not have dialed reports
-// as a failure rather than as a slow test.
-func requireReturnsWithin(t *testing.T, limit time.Duration, fn func() error) {
-	t.Helper()
-
-	done := make(chan error, 1)
-	go func() { done <- fn() }()
-
-	select {
-	case err := <-done:
-		require.NoError(t, err)
-	case <-time.After(limit):
-		t.Fatalf("drain did not return within %s", limit)
-	}
-}
-
-// awaitDrainCommand returns the command the hosting node received, failing the
-// test if none arrives.
-func awaitDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		assert.Equal(t, testAccountID, msg.Header.Get(natsmsg.AccountIDHeader))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// An attached volume is drained on the node hosting its instance, which is the
-// only node with the socket — the node answering CreateSnapshot usually is not.
-func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached"))
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID))
-
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-attached", command.DrainVolumeData.VolumeID)
-}
-
-// No responder on the command subject means no node is running the instance,
-// which is what a stopped instance looks like: stop tears down both the plugin
-// and the subscription but deliberately leaves a boot volume attached. Failing
-// here would make every stopped instance's root volume unsnapshottable.
-func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID))
-}
-
-// A host that still holds the instance but reports it not running has nothing
-// to flush, so the sealed checkpoint stands. This is the host-drain stop, which
-// unmounts the volumes but keeps the subscription.
-func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-not-running", "in-use", drainInstanceID)
-	ack, err := json.Marshal(ec2v1.DrainVolumeResponse{
-		VolumeID: "vol-not-running", Status: ec2v1.DrainVolumeStatusNotRunning,
-	})
-	require.NoError(t, err)
-	got := drainResponder(t, nc, drainInstanceID, ack)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID))
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// A hosting node that reports the drain failed (wedged plugin, socket gone
-// mid-restart) fails the snapshot rather than falling through.
-func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-drain-err", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "drain failed")
-}
-
-// An ack that is not the drained ack is not evidence the writes reached S3.
-func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID, []byte(`{}`))
-
-	err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "unexpected ack")
-}
-
-// Nothing writes to an unattached volume, so the checkpoint its Close() left
-// behind is current and no drain is attempted — not even locally. Dialing the
-// socket is not a probe: it makes the plugin run a full flush, so an available
-// volume must be decided from the record without touching it.
-func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-available", "available", "")
-
-	// A socket that never answers: a dial would park here until the read
-	// deadline, so returning promptly is what proves none was made.
-	testutil.StartBlockingDrainSocket(t, svc.config.DataDir, "vol-available", "OK\n")
-
-	// A nil connection proves no command was issued: routing one would error.
-	svc.natsConn = nil
-
-	requireReturnsWithin(t, 2*time.Second, func() error {
-		return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID)
-	})
-}
-
-// A volume recorded as in-use but with no instance (drifted state) has no node
-// to address, so it takes the stopped path rather than failing every snapshot.
-func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "")
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID))
-}
-
-// When the volume is served by this node the socket is local, so the drain
-// short-circuits without a hop. This is every single-node deployment.
-func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local", "OK\n")
-
-	// A nil connection proves the local ack was enough: no responder exists for
-	// drainInstanceID, so a routed command would fail.
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID))
-}
-
-// A socket that answers without acking means this node does serve the volume
-// and the flush failed. Routing that onward would come back to this same node
-// and re-run the drain that has already failed, so it fails here instead.
-func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local-err", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n")
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err"))
-
-	err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID)
-	require.Error(t, err)
-
-	assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node")
-}
-
-// state.json is the control-plane-owned attachment record; the copy inside
-// config.json is rewritten by the live NBD plugin from its stale in-memory
-// state, so a volume reading "available" there can still be attached.
-func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config"))
-
-	// "available" mirrors the stale State a live NBD plugin leaves in
-	// config.json; drainVolume must not trust it (see the test's own doc above).
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Volumes predating the state.json split have no state object, so the state
-// embedded in config.json is all there is to decide from.
-func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy"))
-
-	// The state/instance embedded in a legacy config.json, with no state.json to overlay it.
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Without Predastore the snapshot is metadata-only and never reads a live
-// checkpoint, so there is nothing for a drain to make current.
-func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-meta-only", "in-use", drainInstanceID)
-	svc.config.Predastore.Host = ""
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID))
-}
-
-// putProviderVolume seeds both the control-plane document CreateSnapshot's
-// provider branch reads for attachment state, and the provider's own record,
-// mirroring what CreateVolume's provider branch leaves behind.
-func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
-		State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a",
-		ProviderHandle: "memory://volume/" + volumeID,
-	}))
 	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
 		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024},
 		AvailabilityZone: "us-east-1a",
 	})
 	require.NoError(t, err)
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
+		State: "in-use", AttachedInstance: attachedInstanceID, AvailabilityZone: "us-east-1a",
+	}))
+	return svc, provider, nc
 }
 
-// The provider branch used to skip the drain entirely, silently snapshotting a
+// The provider drains a buffered volume inside its own CreateSnapshot, so the
-// stale checkpoint of an attached, actively-written volume. It must drain the
+// domain must not dial the drain socket itself: that would drain twice.
-// same way the legacy branch does.
+func TestCreateSnapshot_AttachedVolumeOpensNoDrainSocket(t *testing.T) {
-func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) {
+	svc, provider, _ := setupAttachedSnapshot(t, "vol-attached-local")
-	svc, _, nc := setupDrainService(t)
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, "vol-attached-local", "OK\n")
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain"))
 
 	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID)
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-local")}, testAccountID)
 	require.NoError(t, err)
 
-	command := awaitDrainCommand(t, got)
+	assert.Zero(t, drains(), "the domain must leave the drain to the provider")
-	assert.True(t, command.Attributes.DrainVolume)
+	assert.Equal(t, int32(1), provider.creates.Load())
-	assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID)
 }
 
-// A provider-branch snapshot whose drain fails must fail closed rather than
+// With no drain socket on this node, an attached volume is snapshotted by the
-// return a snapshot of stale data: the provider must never be asked to
+// provider without asking the instance's host for a drain.
-// snapshot, and no snapshot metadata may be left behind.
+func TestCreateSnapshot_AttachedVolumeSendsItsHostNoDrain(t *testing.T) {
-func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) {
+	svc, provider, nc := setupAttachedSnapshot(t, "vol-attached-remote")
-	svc, _, nc := setupDrainService(t)
+	var commands atomic.Int32
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(attachedInstanceID), func(msg *nats.Msg) {
-	svc.SetEBSProvider(provider)
+		commands.Add(1)
-	putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
-
+	})
-	got := drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-
-	out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID)
 	require.NoError(t, err)
-	assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind")
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	snap, err := svc.CreateSnapshot(context.Background(),
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-remote")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "vol-attached-remote", aws.StringValue(snap.VolumeId))
+	assert.Equal(t, int32(1), provider.creates.Load())
+	assert.Zero(t, commands.Load(), "the instance's host must not be asked to drain")
 }

spinifex/domains/ec2/snapshot/service_impl.go

@@ -5,10 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
-	"net"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -18,10 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -209,19 +204,12 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	// Flush the writes buffered by whichever node serves the volume, so the
-	// provider reads a current checkpoint. An attached volume that cannot be
-	// drained fails here rather than silently snapshotting stale data.
-	if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
 	})
 	if err != nil || created == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
 	}
 	now := time.Now()
 	snapshotCfg := ebsmetadata.Snapshot{
@@ -259,177 +247,6 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
-const (
-	// drainDialTimeout bounds the connect to the local drain socket. The socket
-	// is either being served on this node or it is not, so a slow connect means
-	// a wedged plugin rather than a busy one.
-	drainDialTimeout = time.Second
-
-	// drainAckTimeout bounds DrainToBackend on the hosting node: it flushes the
-	// WAL and the live checkpoint to S3, so it scales with the dirty set rather
-	// than with a fixed unit of work.
-	drainAckTimeout = 30 * time.Second
-
-	// drainRequestTimeout bounds the ec2.cmd round-trip. It exceeds
-	// drainAckTimeout so a slow drain surfaces as the hosting node's error
-	// instead of an opaque NATS timeout here.
-	drainRequestTimeout = 35 * time.Second
-)
-
-// ErrNoDrainSocket reports that this node has no drain socket for the volume,
-// which is how a node learns it does not serve it. It is deliberately distinct
-// from a socket that answered badly: only the former is worth routing onward,
-// and conflating the two makes a node re-run a drain that has already failed.
-var ErrNoDrainSocket = errors.New("no drain socket on this node")
-
-// DrainVolumeSocket asks the NBD plugin serving volumeID on this node to flush
-// its WAL chunks and live checkpoint to S3, then waits for the ack. The plugin
-// creates the socket under {dataDir}/viperblock/{volumeID}/, so this only
-// succeeds on the node hosting the volume; the daemon calls it when a drain
-// command is routed to it.
-func DrainVolumeSocket(dataDir, volumeID string) error {
-	sockPath := filepath.Join(dataDir, "viperblock", volumeID, "snapshot.sock")
-	conn, err := net.DialTimeout("unix", sockPath, drainDialTimeout)
-	if err != nil {
-		return fmt.Errorf("%w: dial %s: %w", ErrNoDrainSocket, sockPath, err)
-	}
-	defer conn.Close()
-
-	_ = conn.SetDeadline(time.Now().Add(drainAckTimeout))
-	buf := make([]byte, 8)
-	n, err := conn.Read(buf)
-	if err != nil {
-		return fmt.Errorf("read drain ack for %s: %w", volumeID, err)
-	}
-	if !strings.HasPrefix(string(buf[:n]), "OK") {
-		return fmt.Errorf("drain of %s did not ack OK: %q", volumeID, string(buf[:n]))
-	}
-	return nil
-}
-
-// drainVolume flushes the volume's in-flight writes to S3 before the snapshot
-// reads the live checkpoint from there.
-func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID)
-}
-
-// DrainVolume flushes the volume's in-flight writes to S3 before a live
-// checkpoint is read from there. Shared by ec2.CreateSnapshot and
-// ec2.CreateImage's running-instance path (domains/ec2/image) so the two
-// live-snapshot call sites cannot diverge on when a drain is required.
-//
-// The drain socket lives on the node serving the volume, but both callers are
-// queue-grouped across every node, so the node answering the request usually
-// is not that node. Whether a drain is required is therefore decided from the
-// volume's attachment record, never from whether the socket happens to be
-// local: an attached volume is being written to right now, and reading its
-// checkpoint without draining silently captures an older one.
-func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	// A metadata-only snapshot never reads the live checkpoint, so there is
-	// nothing for a drain to make current.
-	if cfg == nil || cfg.Predastore.Host == "" {
-		return nil
-	}
-
-	state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance)
-	if err != nil {
-		return err
-	}
-
-	// Nothing is writing to an unattached volume: the checkpoint its Close()
-	// left behind is the current one. Decide this before touching the socket —
-	// dialing it is not a probe, it makes the plugin run a full flush.
-	if state != "in-use" || instanceID == "" {
-		slog.InfoContext(ctx, "DrainVolume: volume not attached, snapshotting the checkpoint left by Close (stopped instance path)",
-			"volumeId", volumeID, "state", state, "attachedInstance", instanceID)
-		return nil
-	}
-
-	// Fast path: the volume is served by this node, so no hop is needed. This is
-	// every single-node deployment, and the node hosting the instance on a
-	// cluster.
-	localErr := DrainVolumeSocket(cfg.DataDir, volumeID)
-	if localErr == nil {
-		return nil
-	}
-
-	// A socket that answered but did not ack means this node does serve the
-	// volume and the flush itself failed. Routing that onward would land back on
-	// this same node and repeat the drain that has already failed.
-	if !errors.Is(localErr, ErrNoDrainSocket) {
-		return fmt.Errorf("drain volume %s on this node: %w", volumeID, localErr)
-	}
-
-	slog.InfoContext(ctx, "DrainVolume: volume attached, routing drain to the node hosting it",
-		"volumeId", volumeID, "instanceId", instanceID)
-	return drainOnHostNode(ctx, natsConn, volumeID, instanceID, accountID)
-}
-
-// volumeAttachment returns the volume's authoritative state and attached
-// instance. state.json is control-plane-owned and authoritative; the fallback
-// pair (passed in from the caller's own volume record) is rewritten by the
-// live NBD plugin from its stale in-memory state and is only used for volumes
-// predating the split.
-func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) {
-	rec, found, err := volumestate.Read(ctx, store, bucket, volumeID)
-	if err != nil {
-		return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err)
-	}
-	if found {
-		return rec.State, rec.AttachedInstance, nil
-	}
-	return fallbackState, fallbackInstance, nil
-}
-
-// drainOnHostNode issues the drain on the node hosting instanceID. Only that
-// node subscribes ec2.cmd.{instanceID}, so the command is self-routing and no
-// volume-to-node resolution is needed here.
-//
-// An attachment record is not proof of a live writer: stop deliberately leaves
-// a boot volume attached (runtime/daemon/vm_adapters.go's volumeMounterAdapter.Unmount)
-// while tearing down both the NBD plugin and this subscription. The two
-// not-running signals below are therefore the stopped-instance path, not a
-// failure — treating them as one would make every stopped instance's root
-// volume permanently unsnapshottable.
-func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanceID, accountID string) error {
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-
-	resp, err := natsmsg.NATSRequest[ec2v1.DrainVolumeResponse](ctx, natsConn,
-		ec2v1.InstanceCommandSubject(instanceID), command, drainRequestTimeout, accountID)
-	if err != nil {
-		// No subscriber at all: the instance runs nowhere in the cluster, so it
-		// was stopped (stop migrates it to shared KV before unsubscribing) and
-		// its volumes were sealed by Close. Warn rather than Info because the
-		// same shape appears if the hosting node has lost NATS while its VM
-		// keeps writing, which yields a stale snapshot as it did before routing
-		// existed. A live host that answers and fails is still a hard error.
-		if errors.Is(err, nats.ErrNoResponders) {
-			slog.WarnContext(ctx, "drainVolume: no node hosts the instance, treating the volume as stopped and snapshotting its sealed checkpoint",
-				"volumeId", volumeID, "instanceId", instanceID)
-			return nil
-		}
-		return fmt.Errorf("drain volume %s on the node hosting %s: %w", volumeID, instanceID, err)
-	}
-
-	switch resp.Status {
-	case ec2v1.DrainVolumeStatusDrained:
-		return nil
-	// The host still holds the instance but it is not running: nothing is
-	// writing, so there is nothing to flush. This is a host-drain stop, which
-	// keeps the VM (and this subscription) in place after unmounting.
-	case ec2v1.DrainVolumeStatusNotRunning:
-		slog.InfoContext(ctx, "drainVolume: instance is not running on its host, snapshotting the volume's sealed checkpoint",
-			"volumeId", volumeID, "instanceId", instanceID)
-		return nil
-	default:
-		return fmt.Errorf("drain volume %s on the node hosting %s: unexpected ack %q", volumeID, instanceID, resp.Status)
-	}
-}
-
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
 var describeSnapshotsValidFilters = map[string]bool{
 	"snapshot-id": true,
@@ -740,6 +557,21 @@ func unreadableRecordError(ctx context.Context, snapshotID, accountID, kind stri
 		"Cannot confirm that snapshot '%s' is not in use: %d %s record(s) could not be read", snapshotID, len(unreadable), kind)
 }
 
+// ProviderCreateError maps a provider CreateSnapshot failure of volumeID to
+// the code a client can act on: retry-later for a provider that cannot answer
+// now, IncorrectState for one that needs the volume detached, else internal.
+func ProviderCreateError(volumeID string, err error) error {
+	switch {
+	case errors.Is(err, ebsprovider.ErrUnavailable):
+		return errors.New(awserrors.ErrorServiceUnavailable)
+	case errors.Is(err, ebsprovider.ErrVolumeInUse):
+		return awserrors.Errorf(awserrors.ErrorIncorrectState,
+			"The volume '%s' is in use. Detach it before creating a snapshot.", volumeID)
+	default:
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+}
+
 // providerDeleteError maps a provider DeleteSnapshot failure to the code a
 // client can act on. A transport failure is retryable: the delete is idempotent.
 func providerDeleteError(snapshotID string, err error) error {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1519,3 +1519,55 @@ func TestDropEmptySnapshotRefsDoesNotDropAConcurrentAdd(t *testing.T) {
 		assert.True(t, has, "iteration %d: the concurrent add was dropped by the empty-key delete", i)
 	}
 }
+
+// createFailingProvider fails only CreateSnapshot, with err.
+type createFailingProvider struct {
+	ebsprovider.EBSProvider
+
+	err error
+}
+
+func (p *createFailingProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, p.err
+}
+
+// A provider that cannot snapshot a volume while it is attached refuses it
+// as in use, which the client can act on by detaching the volume first.
+func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+	createTestVolume(t, svc, store, "vol-create-inuse", 10)
+	svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider,
+		err: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}})
+
+	_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-inuse")}, testAccountID)
+
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, "vol-create-inuse")
+	assert.Contains(t, message, "Detach")
+}
+
+// A provider that cannot answer now, such as one refusing a volume another
+// node holds, is retry-later for the client; any other failure is internal.
+func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
+	cases := []struct {
+		name     string
+		err      error
+		wantCode string
+	}{
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
+		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestSnapshotService(t)
+			createTestVolume(t, svc, store, "vol-create-fail", 10)
+			svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider, err: tc.err})
+
+			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, tc.wantCode, err.Error())
+		})
+	}
+}

spinifex/domains/ec2/volume/service_impl_test.go

@@ -1725,6 +1725,28 @@ func TestDeleteVolume_VolumeAttachedButAvailable(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
 }
 
+// A stopped instance keeps its volumes in use, so deleting one is refused
+// until it is detached; AWS answers the same.
+func TestDeleteVolume_VolumeHeldByStoppedInstanceIsRefused(t *testing.T) {
+	kv := setupTestVolumeKV(t)
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+	svc.snapshotKV = kv
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-held", ebsmetadata.Volume{
+		VolumeID:         "vol-held",
+		CapacityGiB:      10,
+		State:            "in-use",
+		AttachedInstance: "i-stopped",
+	})
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-held"),
+	}, testVolAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
+}
+
 func TestDeleteVolume_EmptyStateUnattachedDeletable(t *testing.T) {
 	kv := setupTestVolumeKV(t)
 	store := objectstore.NewMemoryObjectStore()

spinifex/domains/ec2/volumestate/volumestate.go

@@ -1,78 +0,0 @@
-// Package volumestate reads and writes the control-plane-owned attachment
-// state for an EBS volume, persisted to a per-volume state.json object.
-//
-// It is kept out of config.json because config.json is rewritten by the live
-// nbdkit VB on every SaveState (clobbering any State the control plane wrote
-// there) and is a sealed object for encrypted volumes (a second writer reuses
-// the AES-GCM nonce). state.json is plaintext, viperblock never touches it, so
-// the control plane is its single writer. Readers outside the volume service
-// (the snapshot path) need the same record, which is why it lives here.
-package volumestate
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// Record is the control-plane-owned attachment state of a volume.
-type Record struct {
-	State            string    `json:"state"`
-	AttachedInstance string    `json:"attachedInstance"`
-	DeviceName       string    `json:"deviceName"`
-	AttachedAt       time.Time `json:"attachedAt"`
-}
-
-// Key is the S3 key for a volume's control-plane state object.
-func Key(volumeID string) string { return volumeID + "/state.json" }
-
-// Write persists the attachment state, replacing any previous record.
-func Write(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, rec Record) error {
-	data, err := json.Marshal(rec)
-	if err != nil {
-		return fmt.Errorf("marshal volume state: %w", err)
-	}
-	_, err = store.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-		Body:   bytes.NewReader(data),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to write volume state to S3: %w", err)
-	}
-	return nil
-}
-
-// Read returns the attachment state. found=false with a nil error means the
-// object is absent (a volume predating the state.json split), in which case the
-// caller falls back to the State embedded in config.json.
-func Read(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string) (Record, bool, error) {
-	getResult, err := store.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-	})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return Record{}, false, nil
-		}
-		return Record{}, false, fmt.Errorf("failed to get volume state: %w", err)
-	}
-	defer getResult.Body.Close()
-
-	body, err := io.ReadAll(getResult.Body)
-	if err != nil {
-		return Record{}, false, fmt.Errorf("failed to read volume state body: %w", err)
-	}
-	var rec Record
-	if err := json.Unmarshal(body, &rec); err != nil {
-		return Record{}, false, fmt.Errorf("failed to unmarshal volume state: %w", err)
-	}
-	return rec, true, nil
-}

spinifex/domains/iam/service_impl.go

@@ -597,8 +597,14 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	if len(user.AttachedPolicies) > 0 {
 		return nil, errDetachPoliciesFirst
 	}
-	if len(user.Groups) > 0 {
+	for _, groupName := range user.Groups {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		if _, err := s.getGroup(ctx, accountID, groupName); err == nil {
+			return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first.")
+		} else if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return nil, fmt.Errorf("get group: %w", err)
+		}
+		slog.Warn("DeleteUser: ignoring group name with no group record",
+			"accountID", accountID, "userName", userName, "groupName", groupName)
 	}
 	if len(user.InlinePolicies) > 0 {
 		return nil, errDeletePoliciesFirst

spinifex/domains/iam/service_impl_test.go

@@ -3180,3 +3180,37 @@ func TestDeleteUser_RevocationFailureKeepsDelete(t *testing.T) {
 	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("leaver")})
 	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIAMNoSuchEntity))
 }
+
+func TestDeleteUser_DanglingGroupName(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "danglinggroup")
+	createTestGroup(t, svc, "gone")
+	_, err := svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{
+		UserName: aws.String("danglinggroup"), GroupName: aws.String("gone"),
+	})
+	require.NoError(t, err)
+	require.NoError(t, svc.groupsBucket.Delete(t.Context(), testAccountID+".gone"))
+
+	_, err = svc.DeleteUser(testAccountID, &iam.DeleteUserInput{UserName: aws.String("danglinggroup")})
+	require.NoError(t, err)
+
+	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("danglinggroup")})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestDeleteUser_LiveGroupStillConflicts(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "ingroup")
+	createTestGroup(t, svc, "live")
+	_, err := svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{
+		UserName: aws.String("ingroup"), GroupName: aws.String("live"),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.DeleteUser(testAccountID, &iam.DeleteUserInput{UserName: aws.String("ingroup")})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorIAMDeleteConflict)
+}

spinifex/foundation/messaging/nats/nats_trace.go

@@ -48,10 +48,10 @@ func ExtractTraceContext(ctx context.Context, msg *nats.Msg) context.Context {
 // front door to whichever service ends up doing the work.
 const AttrAccountID = "aws.account_id"
 
-// accountAttrs returns the account attribute for accountID, or nothing. Work
+// AccountAttrs returns the account attribute for accountID, or nothing. Work
 // with no caller — a sweep, a reconciler, a node-to-node probe — is left
 // unattributed rather than being credited to whoever happens to be admin.
-func accountAttrs(accountID string) []attribute.KeyValue {
+func AccountAttrs(accountID string) []attribute.KeyValue {
 	if accountID == "" {
 		return nil
 	}
@@ -66,7 +66,7 @@ func StartConsumerSpan(msg *nats.Msg) (context.Context, trace.Span) {
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", msg.Subject),
-	}, accountAttrs(AccountIDFromMsg(msg))...)
+	}, AccountAttrs(AccountIDFromMsg(msg))...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+msg.Subject,
 		trace.WithSpanKind(trace.SpanKindConsumer),
@@ -78,7 +78,7 @@ func startProducerSpan(ctx context.Context, subject, accountID string) (context.
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", subject),
-	}, accountAttrs(accountID)...)
+	}, AccountAttrs(accountID)...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+subject,
 		trace.WithSpanKind(trace.SpanKindClient),

spinifex/operator/accountlifecycle/reapers_iam.go

@@ -162,6 +162,24 @@ func (r *iamUserReaper) Delete(_ context.Context, accountID string, resource Res
 		}
 	}
 
+	// AWS refuses DeleteUser while the user is in a group.
+	groups, err := r.svc.ListGroupsForUser(accountID, &iam.ListGroupsForUserInput{UserName: aws.String(resource.ID)})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if groups != nil {
+		for _, group := range groups.Groups {
+			if group == nil || group.GroupName == nil {
+				continue
+			}
+			if _, err := r.svc.RemoveUserFromGroup(accountID, &iam.RemoveUserFromGroupInput{
+				UserName: aws.String(resource.ID), GroupName: group.GroupName,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
 	_, err = r.svc.DeleteUser(accountID, &iam.DeleteUserInput{UserName: aws.String(resource.ID)})
 	return ignoreAlreadyGone(err)
 }
@@ -249,8 +267,46 @@ func (r *iamGroupReaper) List(_ context.Context, accountID string) ([]Resource,
 	return found, nil
 }
 
+// Delete strips the group's policies first: AWS refuses to delete a group that
+// still has any. Its members were removed with their users, earlier in the stage.
 func (r *iamGroupReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error {
-	_, err := r.svc.DeleteGroup(accountID, &iam.DeleteGroupInput{GroupName: aws.String(resource.ID)})
+	attached, err := r.svc.ListAttachedGroupPolicies(accountID, &iam.ListAttachedGroupPoliciesInput{
+		GroupName: aws.String(resource.ID),
+	})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if attached != nil {
+		for _, policy := range attached.AttachedPolicies {
+			if policy == nil || policy.PolicyArn == nil {
+				continue
+			}
+			if _, err := r.svc.DetachGroupPolicy(accountID, &iam.DetachGroupPolicyInput{
+				GroupName: aws.String(resource.ID), PolicyArn: policy.PolicyArn,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
+	inline, err := r.svc.ListGroupPolicies(accountID, &iam.ListGroupPoliciesInput{GroupName: aws.String(resource.ID)})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if inline != nil {
+		for _, name := range inline.PolicyNames {
+			if name == nil {
+				continue
+			}
+			if _, err := r.svc.DeleteGroupPolicy(accountID, &iam.DeleteGroupPolicyInput{
+				GroupName: aws.String(resource.ID), PolicyName: name,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
+	_, err = r.svc.DeleteGroup(accountID, &iam.DeleteGroupInput{GroupName: aws.String(resource.ID)})
 	return ignoreAlreadyGone(err)
 }
 

spinifex/operator/accountlifecycle/reapers_iam_groups_test.go

@@ -0,0 +1,95 @@
+package accountlifecycle
+
+//test:in-package — drives the unexported identity reapers against the real IAM
+// service, so the DeleteConflict refusals are the real ones and not a fake's.
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const groupTestAccount = "000000000042"
+
+const groupTestPolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListAllMyBuckets","Resource":"*"}]}`
+
+func newRealIAM(t *testing.T) *iamdomain.IAMServiceImpl {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	masterKey, err := iamdomain.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+	return svc
+}
+
+// drainIdentity runs the identity-stage reapers in registration order for a
+// few passes, as a stage does, and returns what each reaper still lists and
+// the delete errors of the last pass.
+func drainIdentity(t *testing.T, svc iamdomain.IAMService) (remaining []Resource, lastErrs []error) {
+	t.Helper()
+	for range 3 {
+		lastErrs = nil
+		for _, reaper := range IAMReapers(svc) {
+			if reaper.Stage() != StageIdentity {
+				continue
+			}
+			found, err := reaper.List(testCtx(t), groupTestAccount)
+			require.NoError(t, err)
+			for _, resource := range found {
+				if err := reaper.Delete(testCtx(t), groupTestAccount, resource, false); err != nil {
+					lastErrs = append(lastErrs, err)
+				}
+			}
+		}
+	}
+	for _, reaper := range IAMReapers(svc) {
+		if reaper.Stage() != StageIdentity {
+			continue
+		}
+		found, err := reaper.List(testCtx(t), groupTestAccount)
+		require.NoError(t, err)
+		remaining = append(remaining, found...)
+	}
+	return remaining, lastErrs
+}
+
+func seedGroupAccount(t *testing.T, svc *iamdomain.IAMServiceImpl) string {
+	t.Helper()
+	_, err := svc.CreateUser(groupTestAccount, &iam.CreateUserInput{UserName: aws.String("u")})
+	require.NoError(t, err)
+	_, err = svc.CreateGroup(groupTestAccount, &iam.CreateGroupInput{GroupName: aws.String("g")})
+	require.NoError(t, err)
+	_, err = svc.AddUserToGroup(groupTestAccount, &iam.AddUserToGroupInput{UserName: aws.String("u"), GroupName: aws.String("g")})
+	require.NoError(t, err)
+	policy, err := svc.CreatePolicy(groupTestAccount, &iam.CreatePolicyInput{
+		PolicyName: aws.String("p"), PolicyDocument: aws.String(groupTestPolicyDoc),
+	})
+	require.NoError(t, err)
+	_, err = svc.AttachGroupPolicy(groupTestAccount, &iam.AttachGroupPolicyInput{
+		GroupName: aws.String("g"), PolicyArn: policy.Policy.Arn,
+	})
+	require.NoError(t, err)
+	_, err = svc.PutGroupPolicy(groupTestAccount, &iam.PutGroupPolicyInput{
+		GroupName: aws.String("g"), PolicyName: aws.String("inline"), PolicyDocument: aws.String(groupTestPolicyDoc),
+	})
+	require.NoError(t, err)
+	return *policy.Policy.Arn
+}
+
+// A user in a group, a group carrying an attached and an inline policy, and the
+// customer policy attached to it all drain; none of them may strand the account.
+func TestIdentityStageDrainsGroupsMembersAndGroupPolicies(t *testing.T) {
+	svc := newRealIAM(t)
+	seedGroupAccount(t, svc)
+
+	remaining, lastErrs := drainIdentity(t, svc)
+
+	assert.Empty(t, remaining)
+	assert.Empty(t, lastErrs)
+}

spinifex/operator/accountlifecycle/reapers_iam_test.go

@@ -34,6 +34,9 @@ type fakeIAM struct {
 	profileRoles     map[string][]string
 	attachedUser     []string
 	inlineUser       []string
+	userGroups       []string
+	attachedGroup    []string
+	inlineGroup      []string
 	attachedRole     []string
 	inlineRole       []string
 
@@ -99,6 +102,48 @@ func (f *fakeIAM) DeleteAccessKey(_ string, _ *iam.DeleteAccessKeyInput) (*iam.D
 	return &iam.DeleteAccessKeyOutput{}, nil
 }
 
+func (f *fakeIAM) ListGroupsForUser(_ string, _ *iam.ListGroupsForUserInput) (*iam.ListGroupsForUserOutput, error) {
+	f.record("ListGroupsForUser")
+	out := &iam.ListGroupsForUserOutput{}
+	for _, name := range f.userGroups {
+		out.Groups = append(out.Groups, &iam.Group{GroupName: aws.String(name)})
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) RemoveUserFromGroup(_ string, _ *iam.RemoveUserFromGroupInput) (*iam.RemoveUserFromGroupOutput, error) {
+	f.record("RemoveUserFromGroup")
+	return &iam.RemoveUserFromGroupOutput{}, nil
+}
+
+func (f *fakeIAM) ListAttachedGroupPolicies(_ string, _ *iam.ListAttachedGroupPoliciesInput) (*iam.ListAttachedGroupPoliciesOutput, error) {
+	f.record("ListAttachedGroupPolicies")
+	out := &iam.ListAttachedGroupPoliciesOutput{}
+	for _, arn := range f.attachedGroup {
+		out.AttachedPolicies = append(out.AttachedPolicies, &iam.AttachedPolicy{PolicyArn: aws.String(arn)})
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) DetachGroupPolicy(_ string, _ *iam.DetachGroupPolicyInput) (*iam.DetachGroupPolicyOutput, error) {
+	f.record("DetachGroupPolicy")
+	return &iam.DetachGroupPolicyOutput{}, nil
+}
+
+func (f *fakeIAM) ListGroupPolicies(_ string, _ *iam.ListGroupPoliciesInput) (*iam.ListGroupPoliciesOutput, error) {
+	f.record("ListGroupPolicies")
+	out := &iam.ListGroupPoliciesOutput{}
+	for _, name := range f.inlineGroup {
+		out.PolicyNames = append(out.PolicyNames, aws.String(name))
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) DeleteGroupPolicy(_ string, _ *iam.DeleteGroupPolicyInput) (*iam.DeleteGroupPolicyOutput, error) {
+	f.record("DeleteGroupPolicy")
+	return &iam.DeleteGroupPolicyOutput{}, nil
+}
+
 func (f *fakeIAM) ListAttachedUserPolicies(_ string, _ *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) {
 	f.record("ListAttachedUserPolicies")
 	out := &iam.ListAttachedUserPoliciesOutput{}
@@ -337,10 +382,41 @@ func TestUserReaperStripsPoliciesBeforeDeleting(t *testing.T) {
 	assert.Equal(t, []string{
 		"ListAttachedUserPolicies", "DetachUserPolicy",
 		"ListUserPolicies", "DeleteUserPolicy",
+		"ListGroupsForUser", "DeleteUser",
+	}, svc.calls)
+}
+
+func TestUserReaperRemovesGroupMembershipsBeforeDeleting(t *testing.T) {
+	svc := newFakeIAM()
+	svc.userGroups = []string{"g1", "g2"}
+
+	err := (&iamUserReaper{svc: svc}).Delete(testCtx(t), "000000000042",
+		Resource{Kind: "iam-user", ID: "admin"}, false)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"ListAttachedUserPolicies", "ListUserPolicies",
+		"ListGroupsForUser", "RemoveUserFromGroup", "RemoveUserFromGroup",
 		"DeleteUser",
 	}, svc.calls)
 }
 
+func TestGroupReaperStripsPoliciesBeforeDeleting(t *testing.T) {
+	svc := newFakeIAM()
+	svc.attachedGroup = []string{"arn:aws:iam::000000000042:policy/p"}
+	svc.inlineGroup = []string{"inline-1"}
+
+	err := (&iamGroupReaper{svc: svc}).Delete(testCtx(t), "000000000042",
+		Resource{Kind: "iam-group", ID: "g"}, false)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"ListAttachedGroupPolicies", "DetachGroupPolicy",
+		"ListGroupPolicies", "DeleteGroupPolicy",
+		"DeleteGroup",
+	}, svc.calls)
+}
+
 func TestRoleReaperStripsPoliciesBeforeDeleting(t *testing.T) {
 	svc := newFakeIAM()
 	svc.attachedRole = []string{"arn:aws:iam::000000000042:policy/task"}

spinifex/providers/ebs/conformance/conformance.go

@@ -145,6 +145,16 @@ type SuiteConfig struct {
 	// the provider, standing in for a crash. Nil skips the dead-export case,
 	// for a provider whose export is not a separate process.
 	KillExport func(t *testing.T, published *ebsprovider.PublishedVolume)
+
+	// QueueOnOtherNode returns a client answered only on the queue subjects, by
+	// a node other than NodeID: what a request meets when the volume's owner
+	// does not answer. Nil skips the cases that need it.
+	QueueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
+
+	// RevokeExport makes the provider revoke a live export as it does when the
+	// volume is lost to another node, and returns the FenceNotice it sent; see
+	// CaptureFenceNotice. Nil skips the case, for a provider that never revokes.
+	RevokeExport func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
 }
 
 func (c SuiteConfig) node() string {
@@ -210,9 +220,9 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 	t.Run("ContractVersions", func(t *testing.T) {
 		t.Run("listed versions are ascending and include the legacy version", func(t *testing.T) {
-			require.NotEmpty(t, contractVersions, "a caller cannot choose a version for a provider that lists none")
+			require.NotEmpty(t, contractVersions, "a provider must list the contract versions it serves")
 			assert.Contains(t, contractVersions, ebsprovider.LegacySchemaVersion,
-				"dropping the legacy version strands every caller still in an N/N-1 rollout")
+				"callers send version-neutral requests at the legacy version, so a provider must serve it")
 			for i := 1; i < len(contractVersions); i++ {
 				assert.Lessf(t, contractVersions[i-1], contractVersions[i], "contract versions %v are not strictly ascending", contractVersions)
 			}
@@ -630,6 +640,39 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			_, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
+
+		// Only the owner can drain what it buffers, so a node answering in its
+		// place must refuse rather than snapshot a checkpoint the owner has not
+		// flushed. The owner's own snapshot then succeeds.
+		t.Run("refused on the queue subject while another node holds the volume", func(t *testing.T) {
+			if !capabilities.OwnerRouting || !capabilities.Exclusion.SingleWriter() {
+				t.Skip("no owner routing behind cluster exclusion, so no node answers in place of an owner")
+			}
+			if cfg.QueueOnOtherNode == nil {
+				t.Skip("no queue-only client on another node configured")
+			}
+			provider := newProvider(t)
+			queue := newTrackingProvider(t, cfg.QueueOnOtherNode(t))
+			ctx := context.Background()
+			volumeID := cfg.id("vol-snap-held")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			for _, version := range contractVersions {
+				snapshotID := cfg.id(fmt.Sprintf("snap-held-queue-v%d", version))
+				_, err = queue.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, SnapshotID: snapshotID, VolumeID: volumeID})
+				require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "v%d: a node not holding the volume must refuse as retry-later", version)
+				if capabilities.SnapshotEnumeration {
+					assert.NotContains(t, drainSnapshotIDs(t, provider, 0), snapshotID)
+				}
+			}
+
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-held-owner"), VolumeID: volumeID})
+			require.NoError(t, err, "the holder answers for its own volume")
+			assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
+		})
 	})
 
 	t.Run("DeleteSnapshot", func(t *testing.T) {
@@ -950,6 +993,24 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
 		})
 
+		t.Run("revoking a live export sends a fence notice to its node", func(t *testing.T) {
+			if cfg.RevokeExport == nil {
+				t.Skip("this provider never revokes a live export")
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-pub-revoked")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			notice := cfg.RevokeExport(t, published)
+			assert.Equal(t, volumeID, notice.VolumeID, "the daemon stops the guest on the volume the notice names")
+			assert.Equal(t, published.NodeID, notice.NodeID, "the notice must name the node whose export was revoked")
+			assert.True(t, ebsprovider.SupportsSchemaVersion(notice.SchemaVersion), "a daemon refuses a notice in a version it cannot read")
+		})
+
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
 			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})

spinifex/providers/ebs/conformance/conformance_test.go

@@ -75,6 +75,28 @@ func TestMemoryProviderNamePrefixConformance(t *testing.T) {
 	}
 }
 
+// TestMemoryProviderFenceNoticeConformance runs the suite with the revocation
+// case enabled, so the contract's fence notice is held to the same check here
+// as on viperblockd.
+func TestMemoryProviderFenceNoticeConformance(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	shared := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities)
+	shared.NotifyFencesOn(conn)
+
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		return shared
+	}, conformance.SuiteConfig{
+		RevokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			return conformance.CaptureFenceNotice(t, conn, published.NodeID, func() error {
+				_, err := shared.RevokePublication(t.Context(), published.VolumeID, "node-elsewhere")
+				return err
+			})
+		},
+	})
+}
+
 // TestNATSProviderConformance runs the suite against a NATSProvider backed
 // by natsserve.Serve, the production-shaped neutral server, rather than a
 // test-only twin: this exercises the transport this package's implementors actually run in production, not a hand-written stand-in for it.

spinifex/providers/ebs/conformance/fence_notice.go

@@ -0,0 +1,60 @@
+package conformance
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+// fenceNoticeWait bounds how long CaptureFenceNotice waits for the notice.
+const fenceNoticeWait = 10 * time.Second
+
+// CaptureFenceNotice answers nodeID's fence notices on conn as a daemon with no
+// guest on the volume does, starts revoke, and returns the first notice heard.
+// revoke runs on its own goroutine, since a sender may retry until answered.
+func CaptureFenceNotice(t *testing.T, conn *nats.Conn, nodeID string, revoke func() error) ebsprovider.FenceNotice {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject(nodeID)
+	require.NoError(t, err)
+
+	heard := make(chan ebsprovider.FenceNotice, 1)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		var notice ebsprovider.FenceNotice
+		ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned()}
+		if err := json.Unmarshal(msg.Data, &notice); err != nil {
+			ack.Error = &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: err.Error()}
+		} else {
+			ack.Versioned = ebsprovider.ReplyVersioned(notice.SchemaVersion)
+			select {
+			case heard <- notice:
+			default:
+			}
+		}
+		if payload, err := json.Marshal(ack); err == nil {
+			_ = msg.Respond(payload)
+		}
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+	require.NoError(t, conn.Flush())
+
+	revoked := make(chan error, 1)
+	go func() { revoked <- revoke() }()
+
+	timeout := time.After(fenceNoticeWait)
+	for {
+		select {
+		case notice := <-heard:
+			return notice
+		case err := <-revoked:
+			require.NoError(t, err, "revoking the export")
+			revoked = nil
+		case <-timeout:
+			t.Fatalf("no fence notice on %s within %s of revoking the export; the daemon would leave its guest running on a revoked disk", subject, fenceNoticeWait)
+		}
+	}
+}

spinifex/providers/ebs/fence_notice.go

@@ -0,0 +1,53 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// NotifyFenced sends notice once and returns the instance the daemon stopped.
+// An ErrInvalidArgument error is terminal: the notice can never be sent. A
+// *ProviderError (internal for an undecodable ack) was heard; others were not.
+func NotifyFenced(ctx context.Context, conn *nats.Conn, notice FenceNotice, timeout time.Duration) (string, error) {
+	if notice.VolumeID == "" {
+		return "", fmt.Errorf("%w: fence notice names no volume", ErrInvalidArgument)
+	}
+	subject, err := FenceNoticeSubject(notice.NodeID)
+	if err != nil {
+		return "", err
+	}
+	if conn == nil {
+		return "", fmt.Errorf("%w: no connection to send the fence notice on", ErrInvalidArgument)
+	}
+
+	reply, err := requestReply(ctx, conn, subject, timeout, notice)
+	if err != nil {
+		return "", err
+	}
+	var ack FenceNoticeAck
+	if err := json.Unmarshal(reply, &ack); err != nil {
+		// The daemon answered, so repeating the notice would only repeat this.
+		return "", &ProviderError{Code: ErrorCodeInternal, Message: fmt.Sprintf("decode %s ack: %v", subject, err)}
+	}
+	if ack.Error != nil {
+		return ack.InstanceID, ack.Error
+	}
+	return ack.InstanceID, nil
+}
+
+// FenceNoticeRetryable reports whether a NotifyFenced error leaves the notice
+// to be sent again: it was not heard, or the daemon refused it as unavailable.
+func FenceNoticeRetryable(err error) bool {
+	if err == nil || errors.Is(err, ErrInvalidArgument) {
+		return false
+	}
+	if _, refused := errors.AsType[*ProviderError](err); refused {
+		return errors.Is(err, ErrUnavailable)
+	}
+	return true
+}

spinifex/providers/ebs/fence_notice_test.go

@@ -0,0 +1,205 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// answerFenceNotices replies to node-a's fence notices with reply.
+func answerFenceNotices(t *testing.T, conn *nats.Conn, reply []byte) {
+	t.Helper()
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+}
+
+func fenceNoticeFor(volumeID string) FenceNotice {
+	return FenceNotice{Versioned: NewVersioned(), VolumeID: volumeID, NodeID: "node-a", Winner: "node-b"}
+}
+
+// TestNotifyFenced_SeparatesHeardFromUnheard pins what a sender's retry turns
+// on: an answer of any kind ends the notice, and only silence repeats it.
+func TestNotifyFenced_SeparatesHeardFromUnheard(t *testing.T) {
+	t.Run("acknowledged", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		instanceID, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.NoError(t, err)
+		assert.Equal(t, "i-stopped", instanceID)
+	})
+
+	t.Run("refused", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeInvalidArgument, Message: "no"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "a refusal was heard, and repeating it would only be refused again")
+		assert.ErrorIs(t, err, ErrInvalidArgument)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("refused as unavailable", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeUnavailable, Message: "later"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, ErrUnavailable)
+		assert.True(t, FenceNoticeRetryable(err), "a daemon that cannot act yet may act on the next notice")
+	})
+
+	t.Run("undecodable answer", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		answerFenceNotices(t, conn, []byte("{"))
+
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "the daemon answered, so the notice was heard")
+		assert.Equal(t, ErrorCodeInternal, refused.Code)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("nobody listening", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.Error(t, err)
+		var refused *ProviderError
+		assert.NotErrorAs(t, err, &refused, "an unheard notice must read as unheard, or the sender stops retrying")
+		assert.True(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("connection closed", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		conn.Close()
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, nats.ErrConnectionClosed)
+		assert.True(t, FenceNoticeRetryable(err), "a connection that comes back can still carry the notice")
+	})
+}
+
+// TestNotifyFenced_AttributesTheNoticeToItsAccount holds the notice to the
+// contract's request path: the daemon's span names the account it acts for.
+func TestNotifyFenced_AttributesTheNoticeToItsAccount(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	accounts := make(chan string, 1)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		accounts <- msg.Header.Get(AccountIDHeader)
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	_, err = NotifyFenced(WithAccountID(t.Context(), "123456789012"), conn, fenceNoticeFor("vol-1"), time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, "123456789012", <-accounts)
+}
+
+// TestNotifyFenced_RefusesAnUnaddressableNotice keeps a notice that can never
+// be sent from reading as unheard, which a retrying sender would repeat forever.
+func TestNotifyFenced_RefusesAnUnaddressableNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	cases := map[string]struct {
+		conn   *nats.Conn
+		notice FenceNotice
+	}{
+		"no volume": {conn, FenceNotice{Versioned: NewVersioned(), NodeID: "node-a"}},
+		"no node":   {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1"}},
+		"bad node":  {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.a"}},
+		"no conn":   {nil, fenceNoticeFor("vol-1")},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := NotifyFenced(context.Background(), tc.conn, tc.notice, time.Second)
+			require.ErrorIs(t, err, ErrInvalidArgument)
+			assert.False(t, FenceNoticeRetryable(err), "a notice that can never be sent must not be retried")
+		})
+	}
+}
+
+// TestMemoryProviderRevokePublication_RetriesAnUnheardNotice keeps the memory
+// provider to the fence's delivery rule: a notice nobody answered is sent
+// again until one does, not dropped after the first try.
+func TestMemoryProviderRevokePublication_RetriesAnUnheardNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := t.Context()
+	provider := NewMemoryProvider(Capabilities{})
+	provider.fenceNoticeTimeout = 100 * time.Millisecond
+	provider.fenceNoticeEvery = 10 * time.Millisecond
+	provider.NotifyFencesOn(conn)
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			return // the first notice is lost, as on a daemon still reconnecting
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	stopped, err := provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err, "a lost first notice must be sent again")
+	assert.Equal(t, "i-stopped", stopped)
+	assert.Equal(t, int32(2), heard.Load())
+}
+
+func TestMemoryProviderRevokePublication(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := context.Background()
+	provider := NewMemoryProvider(Capabilities{})
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.ErrorIs(t, err, ErrUnavailable, "with nowhere to announce it, a revocation would leave the guest unstopped")
+	stillPublished, err := provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.ErrorIs(t, err, ErrVolumeInUse, "a refused revocation must leave the publication in place")
+	assert.Nil(t, stillPublished)
+
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	answerFenceNotices(t, conn, ack)
+	provider.NotifyFencesOn(conn)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.NoError(t, err, "a revoked publication no longer holds the volume")
+
+	_, err = provider.RevokePublication(ctx, "vol-never", "node-b")
+	require.ErrorIs(t, err, ErrNotFound)
+}

spinifex/providers/ebs/memory.go

@@ -2,12 +2,15 @@ package ebsprovider
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"fmt"
 	"maps"
 	"slices"
 	"sync"
 	"time"
+
+	"github.com/nats-io/nats.go"
 )
 
 // MemoryProvider is a deterministic, concurrency-safe Provider used by
@@ -19,6 +22,10 @@ type MemoryProvider struct {
 	volumes      map[string]*memoryVolume
 	snapshots    map[string]*Snapshot
 	now          func() time.Time
+	fenceConn    *nats.Conn
+	// Zero takes the defaults; tests shorten them.
+	fenceNoticeTimeout time.Duration
+	fenceNoticeEvery   time.Duration
 }
 
 type memoryVolume struct {
@@ -406,13 +413,73 @@ func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeR
 	if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID {
 		return result, nil
 	}
-	volume.published = nil
+	volume.unpublish()
-	volume.readOnly = false
-	volume.volume.State = VolumeStateAvailable
 	result.TornDown = req.SchemaVersion >= NoSealSchemaVersion
 	return result, nil
 }
 
+// NotifyFencesOn sets the connection RevokePublication announces fences on.
+func (m *MemoryProvider) NotifyFencesOn(conn *nats.Conn) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fenceConn = conn
+}
+
+// unpublish drops the volume's publication. The caller holds the provider lock.
+func (v *memoryVolume) unpublish() {
+	v.published = nil
+	v.readOnly = false
+	v.volume.State = VolumeStateAvailable
+}
+
+// Defaults for RevokePublication's notice: each attempt's wait for an answer,
+// and the pause before an unanswered notice is sent again.
+const (
+	memoryFenceNoticeTimeout = 2 * time.Second
+	memoryFenceNoticeEvery   = time.Second
+)
+
+// RevokePublication tears volumeID's publication down as a provider that lost
+// the volume to winner does, then sends a FenceNotice to the node it was
+// published on until heard or ctx ends. It returns the instance stopped there.
+func (m *MemoryProvider) RevokePublication(ctx context.Context, volumeID, winner string) (string, error) {
+	m.mu.Lock()
+	conn := m.fenceConn
+	volume := m.volumes[volumeID]
+	timeout := cmp.Or(m.fenceNoticeTimeout, memoryFenceNoticeTimeout)
+	every := cmp.Or(m.fenceNoticeEvery, memoryFenceNoticeEvery)
+	if conn == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: no connection to announce the fence on", ErrUnavailable)
+	}
+	if volume == nil || volume.published == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: volume %s is not published", ErrNotFound, volumeID)
+	}
+	nodeID := volume.published.NodeID
+	volume.unpublish()
+	m.mu.Unlock()
+
+	notice := FenceNotice{
+		Versioned: NewVersioned(),
+		VolumeID:  volumeID,
+		NodeID:    nodeID,
+		Winner:    winner,
+		Reason:    "volume revoked in favour of " + winner,
+	}
+	for {
+		instanceID, err := NotifyFenced(ctx, conn, notice, timeout)
+		if !FenceNoticeRetryable(err) || conn.IsClosed() {
+			return instanceID, err
+		}
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("fence notice for %s never acknowledged: %w: %w", volumeID, ctx.Err(), err)
+		case <-time.After(every):
+		}
+	}
+}
+
 // HoldsUnsealedWrites answers that none are held: every write lands in this
 // process's map, which is the backend, before it is acknowledged.
 func (m *MemoryProvider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {

spinifex/providers/ebs/nats.go

@@ -35,8 +35,9 @@ const (
 const defaultRequestTimeout = 30 * time.Second
 
 // dataMovementTimeout bounds the subjects whose duration is a function of the
-// volume rather than of the control plane. Matches unmountSealTimeout, this
+// volume rather than of the control plane. It must exceed viperblockd's 120s
-// codebase's existing budget for an EBS operation the caller cannot bound.
+// kill grace plus a seal allowance, or an unpublish that uses the whole grace
+// could never be seen to succeed.
 const dataMovementTimeout = 180 * time.Second
 
 // dataMovementVerbs names every operation that references or moves real data,
@@ -88,11 +89,10 @@ func SnapshotCompletionSubject(snapshotID string) (string, error) {
 
 // Last tokens of the per-node subjects: ebs.provider.v1.<node>.<verb>.
 const (
-	publishVerb          = "mount"
+	publishVerb     = "mount"
-	unpublishVerb        = "unmount"
+	unpublishVerb   = "unmount"
-	unsealedVerb         = "unsealed"
+	unsealedVerb    = "unsealed"
-	nodeCapabilitiesVerb = "capabilities"
+	fenceNoticeVerb = "fenced"
-	fenceNoticeVerb      = "fenced"
 )
 
 // nodeSubjectWildcard is the subject matching verb on every node, for a server
@@ -101,10 +101,9 @@ func nodeSubjectWildcard(verb string) string { return subjectPrefix + "*." + ver
 
 // Wildcards over every node for each per-node subject.
 var (
-	PublishSubjectWildcard          = nodeSubjectWildcard(publishVerb)
+	PublishSubjectWildcard        = nodeSubjectWildcard(publishVerb)
-	UnpublishSubjectWildcard        = nodeSubjectWildcard(unpublishVerb)
+	UnpublishSubjectWildcard      = nodeSubjectWildcard(unpublishVerb)
-	UnsealedWritesSubjectWildcard   = nodeSubjectWildcard(unsealedVerb)
+	UnsealedWritesSubjectWildcard = nodeSubjectWildcard(unsealedVerb)
-	NodeCapabilitiesSubjectWildcard = nodeSubjectWildcard(nodeCapabilitiesVerb)
 )
 
 func nodeSubject(nodeID, verb string) (string, error) {
@@ -133,13 +132,6 @@ func UnsealedWritesSubject(nodeID string) (string, error) {
 	return nodeSubject(nodeID, unsealedVerb)
 }
 
-// NodeCapabilitiesSubject returns nodeID's per-node GetCapabilities subject
-// (ebs.provider.v1.<node>.capabilities). Unlike CapabilitiesSubject, which
-// any node's provider answers, only nodeID's provider answers it.
-func NodeCapabilitiesSubject(nodeID string) (string, error) {
-	return nodeSubject(nodeID, nodeCapabilitiesVerb)
-}
-
 // FenceNoticeSubject returns nodeID's fence-notice subject
 // (ebs.provider.v1.<node>.fenced), on which a provider that revoked a live
 // export there sends a FenceNotice for that node's daemon to acknowledge.
@@ -218,6 +210,38 @@ func validateSubjectToken(value string) error {
 	return nil
 }
 
+// A publish answered without an error but without a usable export. The
+// provider may have started the export anyway, so a caller must not assume
+// nothing is left to tear down.
+var (
+	ErrNoPublishedVolume = errors.New("returned no published volume")
+	ErrInvalidExport     = errors.New("returned an invalid export")
+)
+
+// ErrUnreadableReply is a reply that arrived but could not be decoded, or
+// carried a schema version this build does not serve. The provider did
+// receive the request, so whatever it asked for may have happened.
+var ErrUnreadableReply = errors.New("provider reply could not be read")
+
+// AccountIDHeader carries the account a contract request acts for. It is the
+// header the control plane's other NATS hops use, so one name attributes a
+// request from the gateway through to the provider.
+const AccountIDHeader = "X-Account-ID"
+
+type accountIDKey struct{}
+
+// WithAccountID returns ctx carrying the account the contract requests made
+// under it act for. An empty accountID leaves them unattributed.
+func WithAccountID(ctx context.Context, accountID string) context.Context {
+	return context.WithValue(ctx, accountIDKey{}, accountID)
+}
+
+// AccountIDFromContext returns the account WithAccountID put on ctx, or "".
+func AccountIDFromContext(ctx context.Context) string {
+	accountID, _ := ctx.Value(accountIDKey{}).(string)
+	return accountID
+}
+
 // NATSProvider drives a provider daemon through the versioned ebs.* contract.
 type NATSProvider struct {
 	conn           *nats.Conn
@@ -239,26 +263,11 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	return p.getCapabilities(ctx, CapabilitiesSubject, req)
-}
-
-// GetNodeCapabilities asks nodeID's own provider what it supports, over
-// NodeCapabilitiesSubject. A provider that predates that subject gets no
-// responder, which a caller must count as no evidence.
-func (p *NATSProvider) GetNodeCapabilities(ctx context.Context, nodeID string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	subject, err := NodeCapabilitiesSubject(nodeID)
-	if err != nil {
-		return nil, err
-	}
-	return p.getCapabilities(ctx, subject, req)
-}
-
-func (p *NATSProvider) getCapabilities(ctx context.Context, subject string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
-	if err := p.request(ctx, subject, req, &response); err != nil {
+	if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil {
 		return nil, err
 	}
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
@@ -373,7 +382,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	if err != nil {
 		return nil, err
 	}
-	if p.conn == nil || !p.conn.IsConnected() {
+	if !p.connected() {
 		return nil, nats.ErrConnectionClosed
 	}
 
@@ -413,7 +422,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	RecordResponseError(waitSpan, msg.Data)
 	var completed CreateSnapshotResponse
 	if err := json.Unmarshal(msg.Data, &completed); err != nil {
-		return nil, fmt.Errorf("decode snapshot completion: %w", err)
+		return nil, fmt.Errorf("decode snapshot completion: %w: %w", ErrUnreadableReply, err)
 	}
 	if err := responseError(completed.SchemaVersion, completed.Error); err != nil {
 		return nil, err
@@ -496,11 +505,11 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 			return nil, err
 		}
 		if response.Published == nil {
-			return nil, fmt.Errorf("%s returned no published volume", subject)
+			return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
 		}
 		published := PublishedVolumeFromV1(response.Published)
 		if err := published.Export.Validate(); err != nil {
-			return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
+			return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
 		}
 		return published, nil
 	}
@@ -512,10 +521,10 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 		return nil, err
 	}
 	if response.Published == nil {
-		return nil, fmt.Errorf("%s returned no published volume", subject)
+		return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
 	}
 	if err := response.Published.Export.Validate(); err != nil {
-		return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
+		return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
 	}
 	return response.Published, nil
 }
@@ -557,31 +566,64 @@ func (p *NATSProvider) HoldsUnsealedWrites(ctx context.Context, nodeID string, v
 	return response.Held, nil
 }
 
+// connected reports whether a request can be sent now. A reconnecting
+// connection would buffer it and send it on reconnect, after its caller may
+// have given up, so the provider would act with nobody waiting on the result.
+func connected(conn *nats.Conn) bool {
+	return conn != nil && conn.IsConnected()
+}
+
+func (p *NATSProvider) connected() bool { return connected(p.conn) }
+
 func (p *NATSProvider) request(ctx context.Context, subject string, input, output any) error {
-	if p.conn == nil || !p.conn.IsConnected() {
+	reply, err := requestReply(ctx, p.conn, subject, p.timeoutFor(subject), input)
-		return nats.ErrConnectionClosed
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(reply, output); err != nil {
+		return fmt.Errorf("decode %s response: %w: %w", subject, ErrUnreadableReply, err)
+	}
+	return nil
+}
+
+// requestReply sends input on subject as ctx's account, waits up to timeout
+// and returns the raw reply. Its client span records a transport error, an
+// unreadable reply, or the error the reply carries; decoding is the caller's.
+func requestReply(ctx context.Context, conn *nats.Conn, subject string, timeout time.Duration, input any) ([]byte, error) {
+	if !connected(conn) {
+		return nil, nats.ErrConnectionClosed
 	}
 	payload, err := json.Marshal(input)
 	if err != nil {
-		return fmt.Errorf("encode %s request: %w", subject, err)
+		return nil, fmt.Errorf("encode %s request: %w", subject, err)
 	}
 
 	request := &nats.Msg{Subject: subject, Data: payload, Header: nats.Header{}}
-	ctx, span := startClientSpan(ctx, subject, request.Header)
+	accountID := AccountIDFromContext(ctx)
+	if accountID != "" {
+		request.Header.Set(AccountIDHeader, accountID)
+	}
+	ctx, span := startClientSpan(ctx, subject, accountID, request.Header)
 	defer span.End()
 
-	requestCtx, cancel := context.WithTimeout(ctx, p.timeoutFor(subject))
+	requestCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	msg, err := p.conn.RequestMsgWithContext(requestCtx, request)
+	msg, err := conn.RequestMsgWithContext(requestCtx, request)
 	if err != nil {
 		RecordSpanError(span, err)
-		return fmt.Errorf("request %s: %w", subject, err)
+		return nil, fmt.Errorf("request %s: %w", subject, err)
-	}
-	if err := json.Unmarshal(msg.Data, output); err != nil {
-		RecordSpanError(span, err)
-		return fmt.Errorf("decode %s response: %w", subject, err)
 	}
 	RecordResponseError(span, msg.Data)
+	return msg.Data, nil
+}
+
+// ValidateNodeID reports whether nodeID can address this node's contract
+// subjects. A node that fails it has no fence-notice subject, so a revoked
+// export there would leave its guest running.
+func ValidateNodeID(nodeID string) error {
+	if err := validateSubjectToken(nodeID); err != nil {
+		return fmt.Errorf("node ID cannot address EBS provider subjects: %w", err)
+	}
 	return nil
 }
 
@@ -598,7 +640,7 @@ func (p *NATSProvider) requestOwnerFirst(ctx context.Context, ownerSubject, queu
 
 func responseError(version uint16, providerErr *ProviderError) error {
 	if err := checkVersion(version); err != nil {
-		return err
+		return fmt.Errorf("%w: %w", ErrUnreadableReply, err)
 	}
 	if providerErr != nil {
 		return providerErr

spinifex/providers/ebs/nats_reconnect_test.go

@@ -0,0 +1,58 @@
+package ebsprovider
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A request on a connection that is down fails at once rather than buffering:
+// one flushed on reconnect would run after its caller gave up on it.
+func TestNATSProviderRequestRefusesWhileDisconnected(t *testing.T) {
+	connections := map[string]func(t *testing.T) *nats.Conn{
+		"Reconnecting": func(t *testing.T) *nats.Conn {
+			ns := startNATS(t)
+			nc, err := nats.Connect(ns.ClientURL(), nats.MaxReconnects(-1), nats.ReconnectWait(time.Hour))
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
+			ns.Shutdown()
+			require.Eventually(t, func() bool { return !nc.IsConnected() }, 5*time.Second, 5*time.Millisecond)
+			require.False(t, nc.IsClosed(), "the connection must still be trying to reconnect")
+			return nc
+		},
+		"Closed": func(t *testing.T) *nats.Conn {
+			nc, err := nats.Connect(startNATS(t).ClientURL())
+			require.NoError(t, err)
+			nc.Close()
+			return nc
+		},
+	}
+	for name, connect := range connections {
+		t.Run(name, func(t *testing.T) {
+			provider := NewNATSProvider(connect(t), 5*time.Second)
+			ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+			defer cancel()
+
+			_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
+				Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: "node-down",
+			})
+			require.ErrorIs(t, err, nats.ErrConnectionClosed)
+			assert.NotErrorIs(t, err, context.DeadlineExceeded, "the request must be refused, not left waiting")
+		})
+	}
+}
+
+func startNATS(t *testing.T) *server.Server {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(ns.Shutdown)
+	return ns
+}

spinifex/providers/ebs/nats_test.go

@@ -475,6 +475,7 @@ func TestNATSProviderRejectsEmptySuccessResponse(t *testing.T) {
 		respondWith(t, conn, subject, PublishVolumeResponse{Versioned: empty})
 		_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-empty", NodeID: "node-empty"})
 		require.EqualError(t, err, subject+" returned no published volume")
+		require.ErrorIs(t, err, ErrNoPublishedVolume)
 	})
 
 	t.Run("CreateSnapshot", func(t *testing.T) {
@@ -512,6 +513,7 @@ func TestNATSProviderRejectsInvalidExport(t *testing.T) {
 			})
 			published, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: tt.node})
 			require.EqualError(t, err, subject+" returned an invalid export: "+tt.wantErr)
+			require.ErrorIs(t, err, ErrInvalidExport)
 			assert.Nil(t, published)
 		})
 	}
@@ -718,8 +720,6 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
-	nodeCapabilities, err := NodeCapabilitiesSubject("node-1")
-	require.NoError(t, err)
 
 	// A delete costs one round trip per object, so its duration is a function
 	// of the volume. On prod a 100 GiB volume's delete took 52 s against a 30 s
@@ -740,7 +740,7 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
-		publish, unsealed, nodeCapabilities,
+		publish, unsealed,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -126,8 +126,6 @@ func wireVerbs(t *testing.T) []wireVerb {
 	require.NoError(t, err)
 	unsealedSubject, err := ebsprovider.UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
-	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject("node-1")
-	require.NoError(t, err)
 
 	return []wireVerb{
 		{name: "capabilities", subject: ebsprovider.CapabilitiesSubject,
@@ -156,8 +154,6 @@ func wireVerbs(t *testing.T) []wireVerb {
 			request: ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
 		{name: "unsealed", subject: unsealedSubject,
 			request: ebsprovider.HoldsUnsealedWritesRequest{Versioned: ebsprovider.NewVersioned(), NodeID: "node-1", VolumeIDs: []string{"vol-1"}}},
-		{name: "node.capabilities", subject: nodeCapabilitiesSubject,
-			request: ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}},
 	}
 }
 
@@ -737,22 +733,6 @@ func TestServeNodeIDScopesMountSubjects(t *testing.T) {
 	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's mount subject must go unanswered")
 }
 
-// TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode is why the per-node subject
-// exists: the shared one is answered by whichever node's provider is fastest,
-// which says nothing about the provider on the node a caller will mount on.
-func TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode(t *testing.T) {
-	conn := serveProvider(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node-1"})
-	client := ebsprovider.NewNATSProvider(conn, requestTimeout)
-	request := ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}
-
-	resp, err := client.GetNodeCapabilities(t.Context(), "node-1", request)
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
-
-	_, err = client.GetNodeCapabilities(t.Context(), "node-2", request)
-	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's capabilities subject must go unanswered")
-}
-
 // TestServeRejectsUnroutableNodeID covers the subject-build failure path: an
 // unsubscribable node ID must fail Serve outright rather than leave a server
 // running with the shared subjects subscribed and no mount subject at all.

spinifex/providers/ebs/natsserve/server.go

@@ -36,9 +36,9 @@ type Options struct {
 	// across workers does not apply.
 	NoQueueGroup bool
 
-	// NodeID scopes the per-node subjects (publish, unpublish, unsealed writes
+	// NodeID scopes the per-node subjects (publish, unpublish and unsealed
-	// and per-node capabilities) to this node, as a production daemon serves
+	// writes) to this node, as a production daemon serves them. Left empty,
-	// them. Left empty, Serve subscribes their wildcards and answers for any node.
+	// Serve subscribes their wildcards and answers for any node.
 	NodeID string
 }
 
@@ -135,7 +135,6 @@ func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider,
 		{ebsprovider.PublishSubjectWildcard, ebsprovider.PublishSubject, handlePublishVolume(provider)},
 		{ebsprovider.UnpublishSubjectWildcard, ebsprovider.UnpublishSubject, handleUnpublishVolume(provider)},
 		{ebsprovider.UnsealedWritesSubjectWildcard, ebsprovider.UnsealedWritesSubject, handleHoldsUnsealedWrites(provider)},
-		{ebsprovider.NodeCapabilitiesSubjectWildcard, ebsprovider.NodeCapabilitiesSubject, handleCapabilities(provider)},
 	}
 	for _, step := range nodeSteps {
 		subject := step.wildcard

spinifex/providers/ebs/natsserve/server_test.go

@@ -81,3 +81,34 @@ func TestServerSpan_SurvivesAnUntracedCaller(t *testing.T) {
 	assert.Equal(t, "ebs.volume.list", serverSpan.Name())
 	assert.False(t, serverSpan.Parent().IsValid(), "an untraced caller must not leave a dangling parent")
 }
+
+// TestServerSpan_CarriesTheCallersAccount attributes provider work to the
+// account the request acts for, on both halves of the hop, and leaves an
+// unattributed request unattributed.
+func TestServerSpan_CarriesTheCallersAccount(t *testing.T) {
+	recorder := testutil.RecordSpans(t)
+	client := serveMemoryProvider(t)
+
+	ctx := ebsprovider.WithAccountID(t.Context(), "000000000042")
+	_, err := client.ListVolumes(ctx, ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	clientSpan, serverSpan := spansByKind(t, recorder)
+	assert.Equal(t, "000000000042", spanAccount(clientSpan))
+	assert.Equal(t, "000000000042", spanAccount(serverSpan))
+
+	recorder = testutil.RecordSpans(t)
+	_, err = client.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	clientSpan, serverSpan = spansByKind(t, recorder)
+	assert.Empty(t, spanAccount(clientSpan))
+	assert.Empty(t, spanAccount(serverSpan))
+}
+
+func spanAccount(span sdktrace.ReadOnlySpan) string {
+	for _, attr := range span.Attributes() {
+		if attr.Key == "aws.account_id" {
+			return attr.Value.AsString()
+		}
+	}
+	return ""
+}

spinifex/providers/ebs/provider.go

@@ -40,14 +40,13 @@ type Versioned struct {
 	SchemaVersion uint16 `json:"schema_version"`
 }
 
-// NewVersioned returns the oldest representation guaranteed to interoperate
+// NewVersioned returns LegacySchemaVersion, the oldest representation a
-// during an N/N-1 rollout. Callers may use NewLatestVersioned only after a
+// provider serves, for a request whose shape is the same at every version.
-// capability exchange proves the target understands SchemaVersion.
 func NewVersioned() Versioned { return Versioned{SchemaVersion: LegacySchemaVersion} }
 
-// NewLatestVersioned returns the newest representation this build understands.
+// NewLatestVersioned returns SchemaVersion. A cluster runs one contract
-// It must not be used merely because this binary is new: a separately released
+// version, which every provider in it serves, so a request that needs a field
-// peer may still be serving LegacySchemaVersion.
+// only the newest version carries is sent at it without asking first.
 func NewLatestVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} }
 
 // SupportsSchemaVersion reports whether this build can safely decode version.

spinifex/providers/ebs/telemetry.go

@@ -3,6 +3,7 @@ package ebsprovider
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"maps"
 	"slices"
 	"strings"
@@ -30,6 +31,9 @@ const (
 	attrNodeID      = "ebs.provider.node_id"
 	attrOwnerRouted = "ebs.provider.owner_routed"
 	attrErrorCode   = "ebs.provider.error_code"
+	// attrAccountID matches the key the gateway and the control plane's own
+	// NATS spans use, so one field follows a request to the provider.
+	attrAccountID = "aws.account_id"
 )
 
 // VerbUnknown is the verb reported for a subject that is not part of the
@@ -115,8 +119,6 @@ func parseSubject(subject string) subjectTarget {
 		return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID}
 	case unsealedVerb:
 		return subjectTarget{verb: verbUnsealedWrites, nodeID: nodeID}
-	case nodeCapabilitiesVerb:
-		return subjectTarget{verb: verbCapabilities, nodeID: nodeID}
 	case fenceNoticeVerb:
 		return subjectTarget{verb: verbFenceNotice, nodeID: nodeID}
 	default:
@@ -124,11 +126,16 @@ func parseSubject(subject string) subjectTarget {
 	}
 }
 
-func (t subjectTarget) attributes(subject string) []attribute.KeyValue {
+// attributes describes the target; an empty accountID is left off, as work
+// with no caller must not be credited to anyone.
+func (t subjectTarget) attributes(subject, accountID string) []attribute.KeyValue {
 	attrs := []attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String(attrSubject, subject),
 	}
+	if accountID != "" {
+		attrs = append(attrs, attribute.String(attrAccountID, accountID))
+	}
 	if t.volumeID != "" {
 		attrs = append(attrs, attribute.String(attrVolumeID, t.volumeID))
 	}
@@ -156,11 +163,11 @@ func (c headerCarrier) Keys() []string {
 
 // startClientSpan opens a span for an outbound request and injects the trace
 // context into hdr so the serving provider joins the same trace.
-func startClientSpan(ctx context.Context, subject string, hdr nats.Header) (context.Context, trace.Span) {
+func startClientSpan(ctx context.Context, subject, accountID string, hdr nats.Header) (context.Context, trace.Span) {
 	target := parseSubject(subject)
 	ctx, span := otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindClient),
-		trace.WithAttributes(target.attributes(subject)...))
+		trace.WithAttributes(target.attributes(subject, accountID)...))
 	otel.GetTextMapPropagator().Inject(ctx, headerCarrier(hdr))
 	return ctx, span
 }
@@ -178,16 +185,18 @@ func startCompletionSpan(ctx context.Context, verb, completionSubject string) (c
 }
 
 // StartServerSpan joins the trace carried in msg's headers and opens a span
-// for the verb its subject names. Callers must End the returned span and
+// for the verb its subject names, attributed to the account the request
-// should pass the returned context into the work the handler does.
+// carries. Callers must End the span and pass the context into the handler.
 func StartServerSpan(ctx context.Context, msg *nats.Msg) (context.Context, trace.Span) {
 	target := parseSubject(msg.Subject)
+	var accountID string
 	if msg.Header != nil {
 		ctx = otel.GetTextMapPropagator().Extract(ctx, headerCarrier(msg.Header))
+		accountID = msg.Header.Get(AccountIDHeader)
 	}
 	return otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindServer),
-		trace.WithAttributes(target.attributes(msg.Subject)...))
+		trace.WithAttributes(target.attributes(msg.Subject, accountID)...))
 }
 
 // RecordSpanError marks span failed for a transport or handler error.
@@ -210,9 +219,9 @@ func RecordProviderError(span trace.Span, providerErr *ProviderError) {
 	span.SetStatus(codes.Error, providerErr.Message)
 }
 
-// RecordResponseError tags span with the error an encoded response carries.
+// RecordResponseError tags span with the error an encoded response carries, or
-// The response types share no accessor for it, so this re-reads the one field
+// fails it for a response that is not readable. The response types share no
-// rather than making every verb implement one; it runs only when recording.
+// accessor, so this re-reads the one field; it runs only when recording.
 func RecordResponseError(span trace.Span, data []byte) {
 	if !span.IsRecording() {
 		return
@@ -221,6 +230,7 @@ func RecordResponseError(span trace.Span, data []byte) {
 		Error *ProviderError `json:"error"`
 	}
 	if err := json.Unmarshal(data, &envelope); err != nil {
+		RecordSpanError(span, fmt.Errorf("%w: %w", ErrUnreadableReply, err))
 		return
 	}
 	RecordProviderError(span, envelope.Error)

spinifex/providers/ebs/telemetry_test.go

@@ -32,8 +32,6 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject(nodeID)
 	require.NoError(t, err)
-	nodeCapabilities, err := NodeCapabilitiesSubject(nodeID)
-	require.NoError(t, err)
 	fenced, err := FenceNoticeSubject(nodeID)
 	require.NoError(t, err)
 
@@ -54,7 +52,6 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 		publish:               verbVolumePublish,
 		unpublish:             verbVolumeUnpublish,
 		unsealed:              verbUnsealedWrites,
-		nodeCapabilities:      verbCapabilities,
 		fenced:                verbFenceNotice,
 	}
 

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
+	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
 )
@@ -53,24 +54,49 @@ func requireExportTools(t *testing.T) string {
 	return pluginPath
 }
 
+// queueOnlyNode stands a node up whose provider subjects are served on a NATS
+// of its own, with the cluster's lease, dirty and liveness state, so its
+// client reaches it on the queue subjects alone, as when an owner is silent.
+func queueOnlyNode(t *testing.T, node *Config, clusterNC *nats.Conn) ebsprovider.EBSProvider {
+	t.Helper()
+	dirty, err := newVolumeDirty(t.Context(), clusterNC, node.leaseOwner())
+	require.NoError(t, err)
+	node.dirty = dirty
+	node.liveness = newHeartbeatLiveness(clusterNC)
+
+	_, privateURL := setupEmbeddedNATS(t)
+	return ebsprovider.NewNATSProvider(startProviderSubjects(t, node, privateURL), 60*time.Second)
+}
+
+// providerHarness is what inProcessProvider stands up: a client for the
+// cluster, both node names, and the hooks the conformance suite takes.
+type providerHarness struct {
+	newProvider      func(t *testing.T) ebsprovider.EBSProvider
+	nodeID           string
+	otherNodeID      string
+	killExport       func(*testing.T, *ebsprovider.PublishedVolume)
+	revokeExport     func(*testing.T, *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
+	queueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
+}
+
 // inProcessProvider stands two nodes of the provider handlers up over one
-// embedded NATS and one predastore, and returns a client for them plus both
+// embedded NATS and one predastore. Everything is torn down with the test.
-// node names. Everything is torn down with the test.
 //
 // Two nodes rather than one because the exclusion this provider advertises is
 // cluster-scoped, and a single node cannot exercise it: the second publish
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
-// The last return kills a published export's nbdkit behind the handlers' back.
+// The hooks kill a published export's nbdkit behind the handlers' back, and
-func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string, func(*testing.T, *ebsprovider.PublishedVolume)) {
+// revoke one as a lost lease does.
+func inProcessProvider(t *testing.T) providerHarness {
 	t.Helper()
 	pluginPath := requireExportTools(t)
 
 	fixture := testpredastore.Start(t)
 	_, natsURL := setupEmbeddedNATS(t)
 
-	newNode := func(name string) *Config {
+	newNode := func(t *testing.T, name string) *Config {
 		cfg := setupTestConfig(t, natsURL)
 		cfg.S3Host = "https://" + fixture.Host
 		cfg.Bucket = testpredastore.DefaultBucket
@@ -88,8 +114,8 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 		return cfg
 	}
 
-	first := newNode("node-1")
+	first := newNode(t, "node-1")
-	second := newNode("node-2")
+	second := newNode(t, "node-2")
 	nc := startProviderSubjects(t, first, natsURL)
 	startProviderSubjects(t, second, natsURL)
 
@@ -100,21 +126,42 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
-	killExport := func(t *testing.T, published *ebsprovider.PublishedVolume) {
+	holder := func(t *testing.T, published *ebsprovider.PublishedVolume) (*Config, MountedVolume) {
 		t.Helper()
 		for _, node := range []*Config{first, second} {
 			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
-				require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+				return node, mv
-				return
 			}
 		}
 		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+		return nil, MountedVolume{}
 	}
 
-	return func(t *testing.T) ebsprovider.EBSProvider {
+	return providerHarness{
-		t.Helper()
+		newProvider: func(t *testing.T) ebsprovider.EBSProvider {
-		return ebsprovider.NewNATSProvider(nc, 60*time.Second)
+			t.Helper()
-	}, first.NodeName, second.NodeName, killExport
+			return ebsprovider.NewNATSProvider(nc, 60*time.Second)
+		},
+		nodeID:      first.NodeName,
+		otherNodeID: second.NodeName,
+		killExport: func(t *testing.T, published *ebsprovider.PublishedVolume) {
+			t.Helper()
+			_, mv := holder(t, published)
+			require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+		},
+		revokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			node, _ := holder(t, published)
+			return conformance.CaptureFenceNotice(t, nc, published.NodeID, func() error {
+				node.onVolumeLeaseLost(t.Context(), published.VolumeID, leaseLostStalled)
+				return nil
+			})
+		},
+		queueOnOtherNode: func(t *testing.T) ebsprovider.EBSProvider {
+			t.Helper()
+			return queueOnlyNode(t, newNode(t, "node-3"), nc)
+		},
+	}
 }
 
 // runPrefix names one run's volumes so it cannot meet an earlier run's
@@ -126,12 +173,14 @@ func runPrefix() string {
 // TestViperblockdConformance judges the provider handlers by the same suite
 // MemoryProvider and qemunbdd answer to.
 func TestViperblockdConformance(t *testing.T) {
-	newProvider, nodeID, otherNodeID, killExport := inProcessProvider(t)
+	h := inProcessProvider(t)
-	conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{
+	conformance.RunSuiteWithConfig(t, h.newProvider, conformance.SuiteConfig{
-		NamePrefix:  runPrefix(),
+		NamePrefix:       runPrefix(),
-		NodeID:      nodeID,
+		NodeID:           h.nodeID,
-		OtherNodeID: otherNodeID,
+		OtherNodeID:      h.otherNodeID,
-		KillExport:  killExport,
+		KillExport:       h.killExport,
+		RevokeExport:     h.revokeExport,
+		QueueOnOtherNode: h.queueOnOtherNode,
 	})
 }
 
@@ -139,9 +188,9 @@ func TestViperblockdConformance(t *testing.T) {
 // cannot: whether the nbdkit export viperblockd publishes is usable by libnbd,
 // which knows nothing about this codebase.
 func TestViperblockdNBDClient(t *testing.T) {
-	newProvider, nodeID, _, _ := inProcessProvider(t)
+	h := inProcessProvider(t)
-	conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{
+	conformance.RunNBDClientSuiteWithConfig(t, h.newProvider, conformance.NBDClientConfig{
-		NodeID:       nodeID,
+		NodeID:       h.nodeID,
 		VolumePrefix: "vol-" + runPrefix() + "nbd",
 	})
 }

spinifex/providers/ebs/viperblock/provider_contract_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
@@ -176,23 +177,6 @@ func TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion(t *test
 	}
 }
 
-// TestProviderHandlers_NodeCapabilitiesListsServedVersions pins the per-node
-// subject to the same contract versions this build serves.
-func TestProviderHandlers_NodeCapabilitiesListsServedVersions(t *testing.T) {
-	_, natsURL := setupEmbeddedNATS(t)
-	cfg := setupTestConfig(t, natsURL)
-	nc := startProviderSubjects(t, cfg, natsURL)
-
-	subject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
-	require.NoError(t, err)
-	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}))
-
-	var resp ebsprovider.GetCapabilitiesResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	require.Nil(t, resp.Error)
-	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
-}
-
 // TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused covers a caller
 // newer than this build: a no_seal it cannot vouch for is refused, and the
 // export is neither torn down nor sealed.
@@ -252,6 +236,9 @@ func TestPublishError_Codes(t *testing.T) {
 			assert.Equal(t, tc.want, got.Code)
 		})
 	}
+	// Callers still recognise held-elsewhere by its text, so the message keeps it.
+	heldElsewhere := publishError(ebsprovider.NoSealSchemaVersion, "vol-publisherror01", leaseHeld)
+	assert.Contains(t, heldElsewhere.Message, ec2v1.VolumeHeldElsewhereText)
 	assert.True(t, ebsprovider.ErrorCodeUnavailable.Retryable(), "a backing store that is not ready must read as retry-later")
 }
 

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -53,12 +53,12 @@ var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore {
 // viperblock engine construction out of the EC2 control-plane handlers and
 // into the storage daemon that owns BaseDir and the mounted-volume registry.
 //
-// PublishVolume/UnpublishVolume are node-addressed (ebsprovider.PublishSubject
+// It refuses a cfg.NodeName that cannot address the node-addressed subjects.
-// / UnpublishSubject already route to one node), so they are only registered
-// when cfg.NodeName is set; there is no queue-group fallback the way the
-// legacy ebs.mount/ebs.unmount subjects have.
 // Exported so harnesses can put a real provider behind the control plane.
 func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	// The lease store has to exist before any subject is served: a handler
 	// that reaches an engine open without one refuses, and refusing every
 	// publish is a worse failure than not starting.
@@ -114,11 +114,6 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", snapshotCreateWildcard, err)
 	}
 
-	if cfg.NodeName == "" {
-		slog.Warn("ebs.provider: NodeName is empty, this node cannot serve PublishVolume/UnpublishVolume attachments")
-		return nil
-	}
-
 	publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName)
 	if err != nil {
 		return fmt.Errorf("build publish subject: %w", err)
@@ -149,16 +144,6 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", unsealedSubject, err)
 	}
 
-	// The queue subject is answered by any node's provider; this one only by
-	// this node's, so a daemon can learn what its own provider serves.
-	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
-	if err != nil {
-		return fmt.Errorf("build node capabilities subject: %w", err)
-	}
-	if _, err := nc.Subscribe(nodeCapabilitiesSubject, tracedProviderHandler(handleProviderCapabilities)); err != nil {
-		return fmt.Errorf("subscribe to %s: %w", nodeCapabilitiesSubject, err)
-	}
-
 	return nil
 }
 
@@ -1141,13 +1126,9 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 		},
 	})
 
-	// The caller is responsible for draining the volume (flushing guest
+	// The drain runs in the background with the snapshot, so only the accept
-	// writes to the live checkpoint) before requesting a snapshot; that
+	// reply is freed of it; a caller waiting on completion still waits for it.
-	// coordination goes through the guest over ec2.cmd.<instanceID> and
+	// WithoutCancel keeps the trace context but drops the request's deadline.
-	// stays a control-plane concern this provider boundary does not reach.
-	// WithoutCancel keeps the trace context while dropping the deadline: the
-	// work has to outlive the request it was accepted by, and it still has to
-	// be attributable to it.
 	go completeCreateSnapshot(context.WithoutCancel(ctx), cfg, nc, req, operationID, completionSubject)
 }
 
@@ -1188,7 +1169,7 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 		return ebsprovider.ErrorCodeNotFound
 	case errors.Is(err, errSnapshotExistsElsewhere):
 		return ebsprovider.ErrorCodeAlreadyExists
-	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore):
+	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore), errors.Is(err, errExportSettling):
 		// Exclusive access could not be established. The holder gives the
 		// volume up on detach, so this is worth repeating rather than the
 		// permanent failure an internal error reads as.
@@ -1216,26 +1197,36 @@ func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) error {
 }
 
 // snapshotVolumeEngine creates snapshotID off volumeID's live checkpoint,
-// preferring an already-mounted VB over opening a second engine on the same
+// preferring an already-mounted VB, drained first so the checkpoint is
-// volume. Draining the volume so the checkpoint is current is the caller's
+// current, over opening a second engine on the same volume.
-// responsibility (see handleCreateSnapshot).
-//
-// The unmounted branch opens through openVolumeVB, so it holds the volume's
-// cluster-wide lease for as long as the engine is open: a snapshot taken
-// beside a live writer on another node is two engines on one volume.
 func snapshotVolumeEngine(ctx context.Context, cfg *Config, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) {
 	if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil {
+		if err := drainMountedExport(ctx, cfg, volumeID); err != nil {
+			return nil, err
+		}
 		return createSnapshotWithVB(mv.VB, volumeID, snapshotID)
 	}
 
-	vb, lease, err := openVolumeVB(ctx, cfg, volumeID)
+	// The lease refuses a volume another node holds. It is reentrant here, so
+	// an export this node runs but has not registered is excluded or drained
+	// separately, and the engine opens only once that is settled.
+	lease, err := cfg.acquireVolumeLease(ctx, volumeID)
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
+	defer cfg.releaseVolumeLease(ctx, lease)
-		vb.Detach()
+
-		cfg.releaseVolumeLease(ctx, lease)
+	lock, err := excludeLocalWriter(ctx, cfg, volumeID)
-	}()
+	if err != nil {
+		return nil, err
+	}
+	defer releaseWriterLock(ctx, lock, volumeID)
+
+	vb, err := openLeasedVolumeVB(ctx, cfg, volumeID, lease)
+	if err != nil {
+		return nil, err
+	}
+	defer vb.Detach()
 	return createSnapshotWithVB(vb, volumeID, snapshotID)
 }
 
@@ -1813,8 +1804,8 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 
 // handlePublishVolume serves ebs.provider.v1.<node>.mount, the provider-neutral
 // front for the same nbdkit mount path ebs.mount uses. It is node-addressed
-// (RegisterProviderSubjects only subscribes it when cfg.NodeName is set), so
+// (subscribed on cfg.NodeName's subject), so unlike the legacy handler there
-// unlike the legacy handler there is no queue-group fallback to reason about.
+// is no queue-group fallback to reason about.
 func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.PublishVolumeRequest
 	err := json.Unmarshal(msg.Data, &req)
@@ -1929,7 +1920,7 @@ func tearDownPublication(ctx context.Context, cfg *Config, req ebsprovider.Unpub
 	if err != nil {
 		// Nothing mounted here is already the state unpublish asks for, so a
 		// retry after a request that completed server-side converges rather
-		// than erroring, matching the legacy path's unmountResponseError.
+		// than erroring.
 		if unmountResp.NotFound {
 			return unmountResp.Reaped, nil
 		}

spinifex/providers/ebs/viperblock/provider_handlers_test.go

@@ -154,7 +154,6 @@ func providerSubjectCases() []providerSubjectCase {
 			"node_id":    "test-node",
 			"volume_ids": []string{"vol-testunsealed01"},
 		}},
-		{"node_capabilities", "ebs.provider.v1.test-node.capabilities", map[string]any{}},
 	}
 }
 
@@ -1007,9 +1006,8 @@ func TestProviderHandlers_UnpublishVolume_RemovesMountedVolume(t *testing.T) {
 }
 
 // TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent covers a volume this
-// node never mounted. The legacy path treats that as a successful seal (see
+// node never mounted, which counts as a successful seal: a retry after a
-// unmountResponseError), and so must this one: a retry after a request that
+// request that timed out client-side but completed server-side must converge.
-// timed out client-side but completed server-side has to converge.
 func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
 	cfg := setupTestConfig(t, natsURL)
@@ -1029,29 +1027,29 @@ func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	assert.Nil(t, resp.Error, "unpublishing a volume this node never mounted must succeed")
 }
 
-// TestProviderHandlers_PublishUnpublish_RequireNodeName covers
+// TestProviderHandlers_RefuseAnUnaddressableNodeName keeps viperblockd from
-// RegisterProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must
+// serving where the node-addressed subjects, the fence notice among them, have
-// not be registered when cfg.NodeName is empty (PublishSubject has no node to
+// no subject: a fence there would leave its guest running.
-// address), while every other ebs.provider.v1.* subject still is.
+func TestProviderHandlers_RefuseAnUnaddressableNodeName(t *testing.T) {
-func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
-	cfgNoNode := setupTestConfig(t, natsURL)
+	for _, node := range []string{"", "node.a", "node-*"} {
-	cfgNoNode.NodeName = ""
+		t.Run(node, func(t *testing.T) {
-	ncNoNode := startProviderSubjects(t, cfgNoNode, natsURL)
+			nc, err := nats.Connect(natsURL)
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
 
-	_, natsURL2 := setupEmbeddedNATS(t)
+			register := setupTestConfig(t, natsURL)
-	cfgNamed := setupTestConfig(t, natsURL2)
+			register.NodeName = node
-	ncNamed := startProviderSubjects(t, cfgNamed, natsURL2)
+			err = RegisterProviderSubjects(register, nc)
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "viperblockd node name")
 
-	assert.Equal(t, ncNamed.NumSubscriptions()-4, ncNoNode.NumSubscriptions(),
+			run := setupTestConfig(t, natsURL)
-		"an empty NodeName must skip exactly the publish, unpublish, unsealed-writes and node capabilities subscriptions")
+			run.NodeName = node
-
+			require.ErrorIs(t, run.Run(nc), ebsprovider.ErrInvalidArgument)
-	// The other provider subjects must still work without a node name.
+			assert.Zero(t, nc.NumSubscriptions(), "a refused start must serve nothing")
-	body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion})
+		})
-	msg := requestProvider(t, ncNoNode, ebsprovider.CapabilitiesSubject, body)
+	}
-	var resp ebsprovider.GetCapabilitiesResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	assert.Nil(t, resp.Error)
 }
 
 // TestProviderHandlers_CreateVolume_SeedValidation covers the guards that must

spinifex/providers/ebs/viperblock/snapshot_drain.go

@@ -0,0 +1,129 @@
+package viperblock
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/viperblock/viperblock"
+)
+
+const (
+	// exportDrainDialTimeout bounds the connect to the plugin's drain socket.
+	// The socket is local, so a slow connect is a wedged plugin.
+	exportDrainDialTimeout = time.Second
+
+	// exportDrainAckTimeout bounds the plugin's flush of its WAL and live
+	// checkpoint to the backend, which scales with the dirty set.
+	exportDrainAckTimeout = 30 * time.Second
+
+	// maxUnixSocketPath is sun_path's size. The plugin cannot listen on a
+	// longer path, so a drain socket there never exists.
+	maxUnixSocketPath = 108
+)
+
+// errExportDrainFailed is a drain the plugin was asked for and did not
+// confirm, so the live checkpoint cannot be shown to be current.
+var errExportDrainFailed = errors.New("export drain failed")
+
+// errExportSettling is an export whose client is connecting or disconnecting:
+// it holds the volume lock without serving the drain socket, so its writes are
+// neither drainable nor yet sealed. It clears once the client settles.
+var errExportSettling = errors.New("export is between connections")
+
+// drainMountedExport asks the nbdkit plugin serving volumeID on this node to
+// flush its WAL and live checkpoint to the backend before a snapshot reads
+// them. The plugin serves the socket only while a client is connected.
+func drainMountedExport(ctx context.Context, cfg *Config, volumeID string) error {
+	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
+	if len(sockPath) >= maxUnixSocketPath {
+		return requireNoConnectedClient(ctx, cfg, volumeID, "")
+	}
+	start := time.Now()
+	conn, err := net.DialTimeout("unix", sockPath, exportDrainDialTimeout)
+	if errors.Is(err, fs.ErrNotExist) {
+		return requireNoConnectedClient(ctx, cfg, volumeID, sockPath)
+	}
+	if err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	defer conn.Close()
+
+	_ = conn.SetDeadline(time.Now().Add(exportDrainAckTimeout))
+	buf := make([]byte, 8)
+	n, err := conn.Read(buf)
+	if err != nil {
+		return fmt.Errorf("%w: %s: read ack: %w", errExportDrainFailed, volumeID, err)
+	}
+	if !strings.HasPrefix(string(buf[:n]), "OK") {
+		return fmt.Errorf("%w: %s: ack %q", errExportDrainFailed, volumeID, string(buf[:n]))
+	}
+	slog.InfoContext(ctx, "ebs.provider.snapshot.create: drained export", "volume", volumeID, "drain_ms", otelsetup.Millis(time.Since(start)))
+	return nil
+}
+
+// requireNoConnectedClient proves an export serving no drain socket has nothing
+// buffered. sockPath is empty when the socket path is too long for the plugin
+// to listen on, so a held lock there never clears.
+func requireNoConnectedClient(ctx context.Context, cfg *Config, volumeID, sockPath string) error {
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	if errors.Is(err, viperblock.ErrVolumeLocked) {
+		if sockPath == "" {
+			return fmt.Errorf("%w: %s: client connected with no drain socket", errExportDrainFailed, volumeID)
+		}
+		slog.WarnContext(ctx, "ebs.provider.snapshot.create: volume locked with no drain socket, refusing as retry-later",
+			"volume", volumeID, "socket", sockPath)
+		return fmt.Errorf("%w: %s", errExportSettling, volumeID)
+	}
+	if err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	return requireLastClientSealed(cfg, volumeID)
+}
+
+// requireLastClientSealed refuses an export whose last client left WAL behind
+// without a seal receipt newer than it. The plugin releases the lock after a
+// failed seal too, so a free lock alone does not show the writes reached the backend.
+func requireLastClientSealed(cfg *Config, volumeID string) error {
+	if isAuxVolume(volumeID) || !volumeHoldsLocalWAL(cfg, volumeID) || sealReceiptCoversWAL(cfg, volumeID) {
+		return nil
+	}
+	return fmt.Errorf("%w: %s: last client left unsealed WAL", errExportDrainFailed, volumeID)
+}
+
+// excludeLocalWriter proves no export on this node holds writes the backend
+// lacks before a snapshot opens its own engine. The returned lock keeps a new
+// writer out until the snapshot is read; it is nil when a live export was drained.
+func excludeLocalWriter(ctx context.Context, cfg *Config, volumeID string) (*os.File, error) {
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	if errors.Is(err, viperblock.ErrVolumeLocked) {
+		return nil, drainMountedExport(ctx, cfg, volumeID)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	if err := requireLastClientSealed(cfg, volumeID); err != nil {
+		releaseWriterLock(ctx, lock, volumeID)
+		return nil, err
+	}
+	return lock, nil
+}
+
+// releaseWriterLock gives up a lock excludeLocalWriter took. A failed release
+// leaves only the kernel to drop it at exit, so it is logged, not returned.
+func releaseWriterLock(ctx context.Context, lock *os.File, volumeID string) {
+	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
+		slog.WarnContext(ctx, "ebs.provider.snapshot.create: failed to release volume lock", "volume", volumeID, "err", err)
+	}
+}

spinifex/providers/ebs/viperblock/snapshot_drain_test.go

@@ -0,0 +1,450 @@
+package viperblock
+
+import (
+	"encoding/json"
+	"io/fs"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/viperblock/viperblock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountedSnapshotFixture is a node with volumeID mounted over a real engine,
+// whose base dir is where testutil's drain socket stand-in listens.
+func mountedSnapshotFixture(t *testing.T, volumeID string) (cfg *Config, dataDir string, vb *viperblock.VB) {
+	t.Helper()
+	dataDir = testutil.SocketTempDir(t)
+	cfg, vb = mountedAt(t, volumeID, filepath.Join(dataDir, "viperblock"))
+	return cfg, dataDir, vb
+}
+
+// mountedAt is a node with volumeID mounted over a real engine under baseDir.
+func mountedAt(t *testing.T, volumeID, baseDir string) (*Config, *viperblock.VB) {
+	t.Helper()
+	vb := createTestVBWithState(t, volumeID)
+	t.Cleanup(func() {
+		vb.StopChunkUploader()
+		vb.StopWALSyncer()
+	})
+	cfg := &Config{
+		NodeName:       "node-a",
+		BaseDir:        baseDir,
+		MountedVolumes: []MountedVolume{{Name: volumeID, VB: vb}},
+	}
+	require.NoError(t, os.MkdirAll(cfg.BaseDir, 0o750))
+	return cfg, vb
+}
+
+// leaveUnsealedWAL stands in for a client whose disconnect seal failed: the
+// plugin keeps the volume's local WAL and writes no receipt.
+func leaveUnsealedWAL(t *testing.T, cfg *Config, volumeID string) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks"), 0o750))
+}
+
+func requireNoSnapshot(t *testing.T, vb *viperblock.VB, snapshotID string) {
+	t.Helper()
+	exists, err := snapshotConfigExists(vb, snapshotID)
+	require.NoError(t, err)
+	assert.False(t, exists, "a snapshot that could not be shown current must not be written")
+}
+
+// The snapshot reads the checkpoint the plugin flushes, so it must not start
+// until the plugin has acknowledged the drain.
+func TestSnapshotVolumeEngine_MountedVolumeIsDrainedBeforeTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainfirst", "snap-drainfirst00001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	type result struct {
+		snapshot *ebsprovider.Snapshot
+		err      error
+	}
+	done := make(chan result, 1)
+	go func() {
+		snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+		done <- result{snapshot, err}
+	}()
+
+	select {
+	case <-accepted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("a mounted volume's snapshot never asked the plugin to drain")
+	}
+	requireNoSnapshot(t, vb, snapshotID)
+	release()
+
+	select {
+	case got := <-done:
+		require.NoError(t, got.err)
+		assert.Equal(t, ebsprovider.SnapshotStateCompleted, got.snapshot.State)
+	case <-time.After(10 * time.Second):
+		t.Fatal("snapshot did not complete after the drain acked")
+	}
+}
+
+// The control plane issues no drain of its own, so this one is the only flush
+// an attached volume's snapshot gets, and it must happen exactly once.
+func TestSnapshotVolumeEngine_MountedVolumeIsDrainedOnce(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainonce", "snap-drainonce000001"
+	cfg, dataDir, _ := mountedSnapshotFixture(t, volumeID)
+	drains := testutil.StartDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+	assert.Equal(t, 1, drains())
+}
+
+// A plugin that answers without confirming the flush leaves the checkpoint
+// unproven, so the snapshot fails rather than capturing older data.
+func TestSnapshotVolumeEngine_RefusedDrainFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainerr", "snap-drainerr0000001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	testutil.StartDrainSocket(t, dataDir, volumeID, "ERR\n")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A socket file nobody listens on is a plugin that died mid-connection, whose
+// writes were never drained.
+func TestSnapshotVolumeEngine_StaleDrainSocketFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainstale", "snap-drainstale00001"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	sockDir := filepath.Join(cfg.BaseDir, volumeID)
+	require.NoError(t, os.MkdirAll(sockDir, 0o750))
+	ln, err := net.Listen("unix", filepath.Join(sockDir, "snapshot.sock"))
+	require.NoError(t, err)
+	ln.(*net.UnixListener).SetUnlinkOnClose(false)
+	require.NoError(t, ln.Close())
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// The plugin serves the socket only while a client is connected, and drains
+// on disconnect. An export with no client has nothing buffered to flush.
+func TestSnapshotVolumeEngine_ExportWithoutAClientNeedsNoDrain(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainidle", "snap-drainidle000001"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+}
+
+// A client between connections holds the volume lock without serving the
+// socket: its writes are neither drainable nor sealed yet, which clears soon.
+func TestSnapshotVolumeEngine_ExportBetweenConnectionsIsRetryLater(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsettle", "snap-drainsettle0001"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+	logs := captureLogs(t)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+	// A plugin that never serves its socket refuses every retry, so each
+	// refusal is logged with the socket an operator would look for.
+	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
+	assert.True(t, slices.ContainsFunc(strings.Split(logs.String(), "\n"), func(line string) bool {
+		return strings.Contains(line, "level=WARN") && strings.Contains(line, "volume="+volumeID) &&
+			strings.Contains(line, "socket="+sockPath)
+	}), "the refusal must be logged at WARN with the volume and socket: %s", logs.String())
+}
+
+// The plugin releases the lock even when its disconnect seal fails, so a free
+// lock with WAL left behind and no receipt is writes that never reached the
+// backend, and the checkpoint a snapshot would read predates them.
+func TestSnapshotVolumeEngine_UnsealedLastClientFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainunsealed", "snap-drainunsealed01"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A seal receipt is the plugin's proof the last client's writes reached the
+// backend, so local WAL beside it does not block the snapshot.
+func TestSnapshotVolumeEngine_SealedLastClientProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsealed", "snap-drainsealed0001"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: time.Now()})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+	assert.FileExists(t, sealReceiptPath(cfg.BaseDir, volumeID), "the receipt belongs to the unmount it attests to")
+}
+
+// A socket path the plugin could not have listened on never appears, so a
+// connected client there can never be drained: that is permanent, not retry.
+func TestSnapshotVolumeEngine_ConnectedClientWithUnservableSocketFails(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainlongpath", "snap-drainlongpath01"
+	baseDir := filepath.Join(t.TempDir(), strings.Repeat("d", maxUnixSocketPath))
+	cfg, vb := mountedAt(t, volumeID, baseDir)
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.NotErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A plugin that closes the connection without an ack has not confirmed the
+// flush, so the snapshot fails.
+func TestSnapshotVolumeEngine_DrainClosedWithoutAckFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainnoack", "snap-drainnoack00001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	testutil.StartDrainSocket(t, dataDir, volumeID, "")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// lockProbingS3Host is a backend that refuses every request, recording for
+// each whether a new writer could have taken volumeID's lock at that moment.
+type lockProbingS3Host struct {
+	url      string
+	requests atomic.Int32
+	lockable atomic.Int32
+}
+
+func startLockProbingS3Host(t *testing.T, baseDir, volumeID string) *lockProbingS3Host {
+	t.Helper()
+	probe := &lockProbingS3Host{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		probe.requests.Add(1)
+		if lock, err := viperblock.AcquireVolumeLock(baseDir, volumeID); err == nil {
+			probe.lockable.Add(1)
+			_ = viperblock.ReleaseVolumeLock(lock)
+		}
+		w.WriteHeader(http.StatusBadRequest)
+	}))
+	t.Cleanup(srv.Close)
+	probe.url = srv.URL
+	return probe
+}
+
+// unmountedSnapshotFixture is the volume's own host with nothing registered
+// for it, so a snapshot takes the open-an-engine branch.
+func unmountedSnapshotFixture(t *testing.T, volumeID string) (*Config, string, *lockProbingS3Host) {
+	t.Helper()
+	_, natsURL := setupEmbeddedNATS(t)
+	dataDir := testutil.SocketTempDir(t)
+	baseDir := filepath.Join(dataDir, "viperblock")
+	require.NoError(t, os.MkdirAll(baseDir, 0o750))
+	backend := startLockProbingS3Host(t, baseDir, volumeID)
+	cfg := &Config{
+		NodeName:               "node-a",
+		BaseDir:                baseDir,
+		S3Host:                 backend.url,
+		Bucket:                 "test-bucket",
+		Region:                 "us-east-1",
+		AccessKey:              "test-access-key",
+		SecretKey:              "test-secret-key",
+		loadStateRetryAttempts: 1,
+	}
+	installTestVolumeLeases(t, cfg, natsURL)
+	return cfg, dataDir, backend
+}
+
+// holdLockAsExport stands in for an nbdkit this node runs but never
+// registered, as a failed rebuild in recovery leaves behind.
+func holdLockAsExport(t *testing.T, cfg *Config, volumeID string) {
+	t.Helper()
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+}
+
+// An unregistered export still takes guest writes, so the snapshot must have
+// it drain before an engine reads the backend's checkpoint.
+func TestSnapshotVolumeEngine_UnregisteredExportIsDrainedBeforeTheOpen(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayserving", "snap-strayserving001"
+	cfg, dataDir, backend := unmountedSnapshotFixture(t, volumeID)
+	holdLockAsExport(t, cfg, volumeID)
+	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+		done <- err
+	}()
+
+	select {
+	case <-accepted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("an unregistered export was never asked to drain")
+	}
+	assert.Zero(t, backend.requests.Load(), "the engine must not read the backend before the drain acks")
+	release()
+
+	select {
+	case err := <-done:
+		require.Error(t, err, "the refusing backend fails the open")
+		assert.NotErrorIs(t, err, errExportDrainFailed)
+	case <-time.After(10 * time.Second):
+		t.Fatal("snapshot did not proceed after the drain acked")
+	}
+	assert.Positive(t, backend.requests.Load(), "a drained export lets the engine open")
+}
+
+// An unregistered export with no socket cannot be drained, so the snapshot is
+// refused as retry-later before any engine reads the backend.
+func TestSnapshotVolumeEngine_UnregisteredExportWithoutSocketIsRetryLater(t *testing.T) {
+	const volumeID, snapshotID = "vol-straysettle", "snap-straysettle0001"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+	holdLockAsExport(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
+	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
+}
+
+// A departed export that left unsealed WAL holds writes the backend lacks.
+func TestSnapshotVolumeEngine_UnmountedUnsealedWALFailsBeforeTheOpen(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayunsealed", "snap-strayunsealed01"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
+}
+
+// With no export to drain, the volume lock is held across the open so a
+// writer cannot start between the check and the read.
+func TestSnapshotVolumeEngine_UnmountedOpenHoldsTheVolumeLock(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayidle", "snap-strayidle000001"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+
+	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.Error(t, err, "the refusing backend fails the open")
+	assert.NotErrorIs(t, err, errExportDrainFailed)
+	assert.NotErrorIs(t, err, errExportSettling)
+	require.Positive(t, backend.requests.Load(), "a node with no export and no WAL opens the engine")
+	assert.Zero(t, backend.lockable.Load(), "a writer could take the volume lock while the engine was reading")
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err, "the snapshot must give the lock up when it is done")
+	require.NoError(t, viperblock.ReleaseVolumeLock(lock))
+}
+
+// writeSealReceiptAt leaves the receipt the plugin writes after a seal at sealedAt.
+func writeSealReceiptAt(t *testing.T, cfg *Config, volumeID string, sealedAt time.Time) {
+	t.Helper()
+	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: sealedAt})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
+}
+
+// touchWAL dates every entry of the volume's local WAL to at.
+func touchWAL(t *testing.T, cfg *Config, volumeID string, at time.Time) {
+	t.Helper()
+	dir, err := localWALDir(cfg, volumeID)
+	require.NoError(t, err)
+	require.NoError(t, filepath.WalkDir(dir, func(path string, _ fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		return os.Chtimes(path, at, at)
+	}))
+}
+
+// A receipt is proof only for the WAL that existed when it was written. WAL a
+// later client wrote and failed to seal is newer than it, so it is refused.
+func TestSnapshotVolumeEngine_SealReceiptOlderThanTheWALFails(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainstaleseal", "snap-drainstaleseal1"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	sealedAt := time.Now().Add(-time.Minute)
+	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	require.NoError(t, os.WriteFile(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks", "wal.00000002.bin"), []byte("w"), 0o600))
+	touchWAL(t, cfg, volumeID, sealedAt.Add(time.Second))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A receipt written after every WAL change attests to all of it.
+func TestSnapshotVolumeEngine_SealReceiptNewerThanTheWALProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainfreshseal", "snap-drainfreshseal1"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	walChanged := time.Now().Add(-time.Minute)
+	touchWAL(t, cfg, volumeID, walChanged)
+	writeSealReceiptAt(t, cfg, volumeID, walChanged.Add(time.Second))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+}
+
+// The plugin dates its receipt to the millisecond, after the seal's own WAL
+// changes, so a WAL change inside the receipt's millisecond is not newer.
+func TestSnapshotVolumeEngine_SealReceiptInTheWALsMillisecondProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsamemilli", "snap-drainsamemilli1"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	sealedAt := time.Now().Add(-time.Minute).Truncate(time.Millisecond)
+	touchWAL(t, cfg, volumeID, sealedAt.Add(400*time.Microsecond))
+	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
+
+	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+}

spinifex/providers/ebs/viperblock/viperblock.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
+	"io/fs"
 	"log/slog"
 	"net/http"
 	"os"
@@ -23,6 +24,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/mulgadc/viperblock/viperblock/backends/s3"
@@ -459,12 +461,22 @@ func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperbl
 	if err != nil {
 		return nil, nil, err
 	}
+	vb, err := openLeasedVolumeVB(ctx, cfg, volumeName, lease)
+	if err != nil {
+		cfg.releaseVolumeLease(ctx, lease)
+		return nil, nil, err
+	}
+	return vb, lease, nil
+}
 
+// openLeasedVolumeVB is openVolumeVB under a lease the caller already holds,
+// for a caller with a check to make between the lease and the open. The lease
+// stays the caller's to release, on failure too.
+func openLeasedVolumeVB(ctx context.Context, cfg *Config, volumeName string, lease *volumeLease) (*viperblock.VB, error) {
 	vbconfig, s3cfg := volumeVBConfig(cfg, volumeName)
 	vb, err := viperblock.New(&vbconfig, "s3", s3cfg)
 	if err != nil {
-		cfg.releaseVolumeLease(ctx, lease)
+		return nil, fmt.Errorf("new viperblock: %w", err)
-		return nil, nil, fmt.Errorf("new viperblock: %w", err)
 	}
 
 	// New starts the chunk uploader and WAL syncer, so a failure below must
@@ -473,22 +485,21 @@ func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperbl
 	defer func() {
 		if !opened {
 			vb.Detach()
-			cfg.releaseVolumeLease(ctx, lease)
 		}
 	}()
 
 	if err := vb.Backend.InitCtx(ctx); err != nil {
-		return nil, nil, fmt.Errorf("backend init: %w", err)
+		return nil, fmt.Errorf("backend init: %w", err)
 	}
 	if err := loadStateWithRetry(ctx, cfg, vb, volumeName); err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
 			cfg.clearVolumeDirty(ctx, volumeName, lease.generation)
 		}
-		return nil, nil, fmt.Errorf("load state: %w", err)
+		return nil, fmt.Errorf("load state: %w", err)
 	}
 
 	opened = true
-	return vb, lease, nil
+	return vb, nil
 }
 
 // isAuxVolume reports whether a volume is an -efi auxiliary volume. Auxiliary
@@ -546,6 +557,55 @@ func volumeNeedsSeal(volumeName, baseDir string) bool {
 	return err == nil
 }
 
+// volumeHoldsLocalWAL reports whether an nbdkit plugin has opened the
+// volume's WAL here and not removed it. A successful seal removes it; an
+// export no client has connected to yet never created it.
+func volumeHoldsLocalWAL(cfg *Config, volumeName string) bool {
+	dir, err := localWALDir(cfg, volumeName)
+	if err != nil {
+		return false
+	}
+	_, err = os.Stat(dir)
+	return err == nil
+}
+
+// localWALDir is where the nbdkit plugin keeps volumeName's WAL on this node.
+func localWALDir(cfg *Config, volumeName string) (string, error) {
+	walBase := cfg.WALBaseDir
+	if walBase == "" {
+		walBase = cfg.BaseDir
+	}
+	dir, err := localVolumeDir(walBase, volumeName)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "wal"), nil
+}
+
+// latestWALChange is the newest modification time under the volume's local
+// WAL, the directory itself included, so a file created or removed counts too.
+func latestWALChange(cfg *Config, volumeName string) (time.Time, error) {
+	dir, err := localWALDir(cfg, volumeName)
+	if err != nil {
+		return time.Time{}, err
+	}
+	var latest time.Time
+	err = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		if info.ModTime().After(latest) {
+			latest = info.ModTime()
+		}
+		return nil
+	})
+	return latest, err
+}
+
 // sealReceiptSuffix names the file the nbdkit plugin leaves at
 // baseDir/<volume>.sealed after a successful seal.
 const sealReceiptSuffix = ".sealed"
@@ -584,19 +644,53 @@ func consumeSealReceipt(baseDir, volume string) bool {
 	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
 		slog.Warn("failed to remove seal receipt", "volume", volume, "path", path, "err", err)
 	}
+	_, ok := validSealReceipt(data, volume, path)
+	return ok
+}
 
-	var receipt sealReceipt
+// sealReceiptCoversWAL reports whether the volume's seal receipt postdates
-	if err := json.Unmarshal(data, &receipt); err != nil {
+// every change to its local WAL, leaving the receipt for the unmount it
-		slog.Warn("malformed seal receipt", "volume", volume, "path", path, "err", err)
+// attests to. An older receipt is an earlier client's, not the last one's.
+func sealReceiptCoversWAL(cfg *Config, volume string) bool {
+	path := sealReceiptPath(cfg.BaseDir, volume)
+	data, err := os.ReadFile(path)
+	if err != nil {
 		return false
 	}
-	if receipt.Volume != volume || receipt.SealedAt.IsZero() {
+	receipt, ok := validSealReceipt(data, volume, path)
-		slog.Warn("seal receipt missing required fields", "volume", volume, "path", path, "receipt", receipt)
+	if !ok {
+		return false
+	}
+	latest, err := latestWALChange(cfg, volume)
+	if err != nil {
+		slog.Warn("could not read local WAL to date its seal receipt", "volume", volume, "err", err)
+		return false
+	}
+	// The plugin writes SealedAt to the millisecond, after the seal's own WAL
+	// changes, so a WAL change in the receipt's millisecond is the seal's.
+	if receipt.SealedAt.Before(latest.Truncate(time.Millisecond)) {
+		slog.Warn("seal receipt predates the local WAL, treating the WAL as unsealed",
+			"volume", volume, "sealed_at", receipt.SealedAt, "wal_changed_at", latest)
 		return false
 	}
 	return true
 }
 
+// validSealReceipt decodes data, reporting whether it is a receipt the plugin
+// wrote for volume.
+func validSealReceipt(data []byte, volume, path string) (sealReceipt, bool) {
+	var receipt sealReceipt
+	if err := json.Unmarshal(data, &receipt); err != nil {
+		slog.Warn("malformed seal receipt", "volume", volume, "path", path, "err", err)
+		return sealReceipt{}, false
+	}
+	if receipt.Volume != volume || receipt.SealedAt.IsZero() {
+		slog.Warn("seal receipt missing required fields", "volume", volume, "path", path, "receipt", receipt)
+		return sealReceipt{}, false
+	}
+	return receipt, true
+}
+
 // clearStaleSealReceipt removes any seal receipt left by a previous mount of
 // this volume. Staleness is handled here, at mount, rather than by matching
 // PIDs at unmount: once this runs, any receipt found at the next unmount can
@@ -733,6 +827,9 @@ func (cfg *Config) RecoverMountedVolumes(ctx context.Context, nc *nats.Conn) {
 // readiness: without the round trip a caller can act on the signal and still
 // have its request arrive before the interest does.
 func (cfg *Config) Run(nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	if err := cfg.LoadEncryptionKey(); err != nil {
 		return err
 	}
@@ -746,11 +843,7 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 
 	cfg.RecoverMountedVolumes(context.Background(), nc)
 
-	if cfg.NodeName != "" {
+	slog.Info("Waiting for EBS events", "node", cfg.NodeName)
-		slog.Info("Waiting for EBS events", "node", cfg.NodeName)
-	} else {
-		slog.Info("Waiting for EBS events (single-node mode)")
-	}
 
 	if err := RegisterVolumeLifecycleSubjects(cfg, nc); err != nil {
 		return err
@@ -768,6 +861,15 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 	return nil
 }
 
+// validateNodeName refuses a node name the contract cannot address. Without
+// one the fence notice has no subject, and a fenced guest would keep running.
+func (cfg *Config) validateNodeName() error {
+	if err := ebsprovider.ValidateNodeID(cfg.NodeName); err != nil {
+		return fmt.Errorf("viperblockd node name %q (cluster config node): %w", cfg.NodeName, err)
+	}
+	return nil
+}
+
 // Shutdown flushes each mounted volume's WAL and reaps idle nbdkit processes.
 // It snapshots MountedVolumes and clears it while holding the lock, then
 // flushes/kills outside the lock since VB.Close does heavy I/O.

spinifex/providers/ebs/viperblock/viperblockd_integration_test.go

@@ -839,42 +839,6 @@ func TestIntegration_ServiceGracefulShutdown(t *testing.T) {
 	t.Log("Service is running and responsive")
 }
 
-// TestIntegration_GenericTopicRouting tests that NodeName="" uses generic ebs.unmount with queue group.
-func TestIntegration_GenericTopicRouting(t *testing.T) {
-	t.Parallel()
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	ns, natsURL := setupEmbeddedNATS(t)
-	defer ns.Shutdown()
-
-	cfg := setupTestConfig(t, natsURL)
-	cfg.NodeName = "" // Single-node / generic mode
-	cfg.MountedVolumes = []MountedVolume{
-		{Name: "vol-generic", PID: deadpid.PID},
-	}
-
-	startTestService(t, cfg)
-
-	nc, err := nats.Connect(natsURL)
-	assert.NoError(t, err)
-	defer nc.Close()
-
-	// Send to generic topic (not ebs.{node}.unmount)
-	request := viperblocklegacyv1.EBSRequest{Name: "vol-generic"}
-	requestData, _ := json.Marshal(request)
-
-	msg, err := nc.Request("ebs.unmount", requestData, 3*time.Second)
-	assert.NoError(t, err)
-
-	var response viperblocklegacyv1.EBSUnMountResponse
-	assert.NoError(t, json.Unmarshal(msg.Data, &response))
-	assert.Equal(t, "vol-generic", response.Volume)
-	assert.False(t, response.Mounted)
-	assert.Empty(t, response.Error)
-}
-
 // TestIntegration_MountAuxiliaryVolumeSuffix tests that volumes with -cloudinit suffix
 // reach the mount handler and trigger the auxiliary volume code path.
 func TestIntegration_MountAuxiliaryVolumeSuffix(t *testing.T) {

spinifex/providers/ebs/viperblock/volume_fence.go

@@ -9,8 +9,8 @@ import (
 	"os"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -319,46 +319,40 @@ func (cfg *Config) fenceNoticeRetry() time.Duration {
 }
 
 // publishVolumeFenced tells the local daemon a guest is running against a disk
-// that is gone, until the daemon acknowledges. It gives up on a closed
+// that is gone, until the daemon acknowledges. It gives up on a notice that can
-// connection, or once the volume is exported here again by a newer mount.
+// never be acted on, a closed connection, or the volume being exported again.
 func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner string) {
 	if cfg.nc == nil {
 		return
 	}
-	event := viperblocklegacyv1.VolumeFencedEvent{
+	notice := ebsprovider.FenceNotice{
-		Volume: volumeName,
+		Versioned: ebsprovider.NewVersioned(),
-		Node:   cfg.leaseOwner(),
+		VolumeID:  volumeName,
-		Winner: winner,
+		NodeID:    cfg.NodeName,
-		Reason: fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
+		Winner:    winner,
+		Reason:    fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
 	}
-	payload, err := json.Marshal(event)
-	if err != nil {
-		slog.ErrorContext(ctx, "fence: marshal fenced event", "volume", volumeName, "err", err)
-		return
-	}
-	subject := viperblocklegacyv1.VolumeFencedSubject(cfg.NodeName)
 	began := time.Now()
 
 	for attempt := 1; ; attempt++ {
-		reqCtx, cancel := context.WithTimeout(ctx, fenceNoticeTimeout)
+		instanceID, err := ebsprovider.NotifyFenced(ctx, cfg.nc, notice, fenceNoticeTimeout)
-		reply, err := cfg.nc.RequestWithContext(reqCtx, subject, payload)
-		cancel()
 		if err == nil {
-			var ack viperblocklegacyv1.VolumeFencedAck
+			if attempt > 1 {
-			if jerr := json.Unmarshal(reply.Data, &ack); jerr == nil && ack.Error != "" {
-				slog.ErrorContext(ctx, "fence: the daemon could not act on the fenced volume",
-					"volume", volumeName, "subject", subject, "err", ack.Error)
-			} else if attempt > 1 {
 				slog.WarnContext(ctx, "fence: the daemon acknowledged the fenced volume after retrying",
-					"volume", volumeName, "instance", ack.Instance, "attempts", attempt,
+					"volume", volumeName, "instance", instanceID, "attempts", attempt,
 					"unacknowledged_ms", otelsetup.Millis(time.Since(began)))
 			}
 			return
 		}
+		if !ebsprovider.FenceNoticeRetryable(err) {
+			slog.ErrorContext(ctx, "fence: the daemon cannot act on the fenced volume, the guest is left with dead I/O",
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt, "err", err)
+			return
+		}
 
 		if cfg.nc.IsClosed() {
 			slog.ErrorContext(ctx, "fence: connection closed before the daemon acknowledged the fenced volume, the guest may be left with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt)
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt)
 			return
 		}
 		if _, again := cfg.lookupMountedVolume(volumeName); again {
@@ -368,7 +362,7 @@ func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner s
 		}
 		if attempt == 1 || attempt%fenceNoticeLogEveryTries == 0 {
 			slog.ErrorContext(ctx, "fence: the daemon has not acknowledged the fenced volume, retrying; until it does the guest keeps running with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt,
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt,
 				"unacknowledged_ms", otelsetup.Millis(time.Since(began)), "err", err)
 		}
 

spinifex/providers/ebs/viperblock/volume_fence_invariants_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -133,16 +133,16 @@ func TestS4_FenceHolderReadIsBounded(t *testing.T) {
 // fenceResponder answers fence notices as the daemon does and records them.
 type fenceResponder struct {
 	mu     sync.Mutex
-	events []viperblocklegacyv1.VolumeFencedEvent
+	events []ebsprovider.FenceNotice
 }
 
 func (r *fenceResponder) handle(msg *nats.Msg) {
-	var event viperblocklegacyv1.VolumeFencedEvent
+	var notice ebsprovider.FenceNotice
-	_ = json.Unmarshal(msg.Data, &event)
+	_ = json.Unmarshal(msg.Data, &notice)
 	r.mu.Lock()
-	r.events = append(r.events, event)
+	r.events = append(r.events, notice)
 	r.mu.Unlock()
-	payload, _ := json.Marshal(viperblocklegacyv1.VolumeFencedAck{Instance: "i-fenced"})
+	payload, _ := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion), InstanceID: "i-fenced"})
 	_ = msg.Respond(payload)
 }
 
@@ -185,7 +185,9 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.NoError(t, err)
 	t.Cleanup(daemonConn.Close)
 	responder := &fenceResponder{}
-	_, err = daemonConn.Subscribe(viperblocklegacyv1.VolumeFencedSubject("node-a"), responder.handle)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = daemonConn.Subscribe(subject, responder.handle)
 	require.NoError(t, err)
 	require.NoError(t, daemonConn.Flush())
 
@@ -197,7 +199,10 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.GreaterOrEqualf(t, responder.count(), 1,
 		"ADR-0007:S4: a daemon that subscribed after the first notice never heard the fence, so its guest runs on with no disk. "+
 			"Fix: retry publishVolumeFenced (volume_fence.go) until the daemon acknowledges")
-	assert.Equal(t, volumeName, responder.events[0].Volume)
+	first := responder.events[0]
+	assert.Equal(t, volumeName, first.VolumeID)
+	assert.Equal(t, "node-a", first.NodeID)
+	assert.True(t, ebsprovider.SupportsSchemaVersion(first.SchemaVersion), "the daemon refuses a notice in a version it cannot read")
 
 	seen := responder.count()
 	time.Sleep(5 * cfg.fenceNoticeEvery)
@@ -252,3 +257,93 @@ func TestS4_FenceNoticeStopsWithTheConnection(t *testing.T) {
 		t.Fatal("a fence notice kept retrying on a closed connection")
 	}
 }
+
+// TestFenceNotice_UnavailableDaemonIsAskedAgain keeps a refusal that says
+// "not now" from ending delivery: the guest is still running on a dead disk.
+func TestFenceNotice_UnavailableDaemonIsAskedAgain(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunavailable"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	unavailable, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "not yet"},
+	})
+	require.NoError(t, err)
+	ack, err := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned(), InstanceID: "i-fenced"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			_ = msg.Respond(unavailable)
+			return
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice was still being sent after the daemon acknowledged it")
+	}
+	assert.Equal(t, int32(2), heard.Load(), "ADR-0007:S4: a daemon that was unavailable was never asked again")
+}
+
+// TestFenceNotice_UnsendableNoticeIsNotRetried stops a notice that can never
+// be addressed from spinning until shutdown on a goroutine nobody watches.
+func TestFenceNotice_UnsendableNoticeIsNotRetried(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunsendable"
+	cfg, _ := noticeConfig(t, natsURL, volumeName)
+	cfg.NodeName = ""
+	cfg.MountedVolumes = nil // gone, as after the teardown, so only the error can end the retry
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.publishVolumeFenced(t.Context(), volumeName, "node-b") }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice with no subject to send it on kept retrying")
+	}
+}
+
+// TestFenceNotice_RefusedNoticeIsNotRepeated keeps a notice the daemon heard and
+// could not act on from being resent forever: a refusal is an answer.
+func TestFenceNotice_RefusedNoticeIsNotRepeated(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticerefused"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	var heard atomic.Int32
+	refusal, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: "no"},
+	})
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		heard.Add(1)
+		_ = msg.Respond(refusal)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice kept retrying after the daemon refused it")
+	}
+	assert.Equal(t, int32(1), heard.Load())
+}

spinifex/providers/ebs/viperblock/volume_fence_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -171,15 +172,25 @@ func TestVolumeFence_CompletesOnceTheWriterFinallyExits(t *testing.T) {
 
 	const volumeName = "vol-fencekilllate"
 
-	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
 	cfg.MountedVolumes[0].PID = 0 // rejected by ForceKillProcess, so the kill cannot succeed
 	// Alive for the guard and one watch pass, gone on the next: the fence has to
 	// see the failure persist and then resolve.
 	var probes atomic.Int32
 	cfg.processAlive = func(int) bool { return probes.Add(1) <= 2 }
 	cfg.fenceWatchEvery = 10 * time.Millisecond
+	responder := &fenceResponder{}
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, responder.handle)
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
 
 	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
+	require.Equalf(t, 1, responder.count(),
+		"ADR-0007:S4: a fence that completed late never told the daemon, so its guest runs on with no disk. "+
+			"Fix: publishVolumeFenced once awaitFencedWriterExit sees the writer gone (volume_fence.go)")
+	assert.Equal(t, volumeName, responder.events[0].VolumeID)
 
 	cfg.mu.Lock()
 	mounted := len(cfg.MountedVolumes)

spinifex/runtime/compute/vm/console_log_reaper.go

@@ -0,0 +1,228 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+)
+
+const (
+	// consoleTrimAbove is the size past which a console log is cut back.
+	consoleTrimAbove = 4 << 20
+	// consoleKeepBytes is how much of the newest output survives a cut.
+	consoleKeepBytes = 1 << 20
+	// consoleRemoveAfter is how long a retired console file lingers so a late
+	// GetConsoleOutput still sees it.
+	consoleRemoveAfter = time.Hour
+	// consoleIdleAfter is how long a live console file must sit unchanged, with
+	// no QEMU on this node holding it open, before it is removed.
+	consoleIdleAfter = time.Hour
+)
+
+// consoleTerminatedSuffix marks the console file of an instance that is gone
+// for good. The log shipper only follows console-*.log, so a renamed file is
+// no longer shipped, and the reaper removes it once it is old enough.
+const consoleTerminatedSuffix = ".terminated"
+
+// RetiredConsolePath is where the console file at path moves when its instance
+// is terminated.
+func RetiredConsolePath(path string) string { return path + consoleTerminatedSuffix }
+
+// retireConsoleLog renames an ended instance's console file so the reaper can
+// age it out. The file is final by now: QEMU is down or being force-finalized.
+func retireConsoleLog(path string) {
+	if path == "" {
+		return
+	}
+	if err := os.Rename(path, RetiredConsolePath(path)); err != nil && !os.IsNotExist(err) {
+		slog.Warn("vm: retire console log failed", "path", path, "err", err)
+	}
+}
+
+// ConsoleLogReaper caps the serial console files under dir and removes the
+// retired ones. A live console-<id>.log goes only after an idle hour with no
+// QEMU on this node holding it open, decided from /proc alone and never from
+// instance state.
+type ConsoleLogReaper struct {
+	dir      string
+	procRoot string
+	now      func() time.Time
+}
+
+var _ Reaper = (*ConsoleLogReaper)(nil)
+
+// NewConsoleLogReaper builds a reaper over the console files in dir.
+func NewConsoleLogReaper(dir string) *ConsoleLogReaper {
+	return &ConsoleLogReaper{dir: dir, procRoot: qemuProcRoot, now: time.Now}
+}
+
+func (r *ConsoleLogReaper) Class() string      { return "console-log" }
+func (r *ConsoleLogReaper) Scope() ReaperScope { return ScopeNodeLocal }
+
+// NewLocalConsoleLogReaper builds the reaper over this node's runtime directory.
+func (m *Manager) NewLocalConsoleLogReaper() *ConsoleLogReaper {
+	return NewConsoleLogReaper(hostprocess.RuntimeDir())
+}
+
+// Sweep removes idle live console files nothing holds open, cuts oversized ones
+// back to their newest consoleKeepBytes, and removes retired ones that have sat
+// for over an hour.
+func (r *ConsoleLogReaper) Sweep(context.Context) (int, error) {
+	reaped := 0
+	live, err := filepath.Glob(filepath.Join(r.dir, "console-*.log"))
+	if err != nil {
+		return 0, err
+	}
+	now := r.now()
+	warned := false
+	for _, p := range live {
+		info, err := os.Stat(p)
+		if err != nil || !info.Mode().IsRegular() {
+			continue
+		}
+		if now.Sub(info.ModTime()) > consoleIdleAfter && r.removeIfUnheld(p, &warned) {
+			reaped++
+			continue
+		}
+		if info.Size() <= consoleTrimAbove {
+			continue
+		}
+		if err := trimConsoleLog(p); err != nil {
+			slog.Warn("vm/gc: trim console log failed", "path", p, "err", err)
+			continue
+		}
+		reaped++
+	}
+	retired, err := filepath.Glob(filepath.Join(r.dir, "console-*.log"+consoleTerminatedSuffix))
+	if err != nil {
+		return reaped, err
+	}
+	for _, p := range retired {
+		info, err := os.Stat(p)
+		if err != nil || !info.Mode().IsRegular() || now.Sub(info.ModTime()) <= consoleRemoveAfter {
+			continue
+		}
+		if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
+			slog.Warn("vm/gc: remove retired console log failed", "path", p, "err", err)
+			continue
+		}
+		reaped++
+	}
+	return reaped, nil
+}
+
+// removeIfUnheld removes path when no QEMU process on this node has it open. If
+// the process table cannot be read in full it removes nothing, since a guest
+// that is merely quiet must keep its log.
+func (r *ConsoleLogReaper) removeIfUnheld(path string, warned *bool) bool {
+	held, err := qemuOpenFiles(r.procRoot, r.dir)
+	if err != nil {
+		if !*warned {
+			slog.Warn("vm/gc: idle console logs kept, cannot tell whether QEMU holds them", "err", err)
+			*warned = true
+		}
+		return false
+	}
+	if resolved, err := filepath.EvalSymlinks(path); err == nil {
+		path = resolved
+	}
+	if held[path] {
+		return false
+	}
+	if err := os.Remove(path); err != nil {
+		if !os.IsNotExist(err) {
+			slog.Warn("vm/gc: remove idle console log failed", "path", path, "err", err)
+		}
+		return false
+	}
+	return true
+}
+
+// qemuOpenFiles returns the files every QEMU process under procRoot has open.
+// Any unreadable descriptor table is an error, never a partial answer. The scan
+// is complete only while the daemon keeps CAP_SYS_ADMIN under ProtectProc=invisible,
+// so an instance pidfile naming a live pid the scan missed is an error too.
+func qemuOpenFiles(procRoot, runtimeDir string) (map[string]bool, error) {
+	pids, err := scanLiveQEMUPIDs(procRoot)
+	if err != nil {
+		return nil, err
+	}
+	if err := checkPidFilesVisible(runtimeDir, pids); err != nil {
+		return nil, err
+	}
+	held := make(map[string]bool)
+	for _, pid := range pids {
+		fdDir := filepath.Join(procRoot, strconv.Itoa(pid), "fd")
+		fds, err := os.ReadDir(fdDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		for _, fd := range fds {
+			target, err := os.Readlink(filepath.Join(fdDir, fd.Name()))
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				return nil, err
+			}
+			held[target] = true
+		}
+	}
+	return held, nil
+}
+
+// checkPidFilesVisible fails when an instance pidfile names a process that is
+// alive but absent from seen. Signal 0 answers even when /proc hides the pid.
+func checkPidFilesVisible(runtimeDir string, seen []int) error {
+	visible := make(map[int]bool, len(seen))
+	for _, pid := range seen {
+		visible[pid] = true
+	}
+	for pid, id := range pidFileOwners(runtimeDir) {
+		if !strings.HasPrefix(id, "i-") || visible[pid] {
+			continue
+		}
+		if err := syscall.Kill(pid, 0); err == nil || err == syscall.EPERM {
+			return fmt.Errorf("pidfile for %s names live pid %d that the process scan cannot see", id, pid)
+		}
+	}
+	return nil
+}
+
+// trimConsoleLog rewrites the file in place as its newest consoleKeepBytes. The
+// guest's writer keeps its descriptor (O_APPEND), so output written while the
+// copy runs may be lost but the writer is never blocked.
+func trimConsoleLog(path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if info.Size() <= consoleTrimAbove {
+		return nil
+	}
+	tail := make([]byte, consoleKeepBytes)
+	if _, err := f.ReadAt(tail, info.Size()-consoleKeepBytes); err != nil && err != io.EOF {
+		return err
+	}
+	if _, err := f.WriteAt(tail, 0); err != nil {
+		return err
+	}
+	return f.Truncate(consoleKeepBytes)
+}

spinifex/runtime/compute/vm/console_log_reaper_test.go

@@ -0,0 +1,298 @@
+package vm
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// consoleBody returns n bytes of 16-byte numbered lines, so any slice of it
+// identifies which part of the original it came from.
+func consoleBody(n int) []byte {
+	var b bytes.Buffer
+	for i := 0; b.Len() < n; i++ {
+		fmt.Fprintf(&b, "l%014d\n", i)
+	}
+	return b.Bytes()[:n]
+}
+
+func newConsoleReaper(t *testing.T, now time.Time) (*ConsoleLogReaper, string) {
+	t.Helper()
+	dir := t.TempDir()
+	r := NewConsoleLogReaper(dir)
+	r.procRoot = t.TempDir()
+	r.now = func() time.Time { return now }
+	return r, dir
+}
+
+func writeConsole(t *testing.T, dir, id string, body []byte, mtime time.Time) string {
+	t.Helper()
+	p := filepath.Join(dir, "console-"+id+".log")
+	require.NoError(t, os.WriteFile(p, body, 0o644))
+	require.NoError(t, os.Chtimes(p, mtime, mtime))
+	return p
+}
+
+func TestConsoleLogReaper_TrimsToNewestMiB(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	body := consoleBody(5 << 20)
+	p := writeConsole(t, dir, "i-big", body, now)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+
+	got, err := os.ReadFile(p)
+	require.NoError(t, err)
+	gotSize := len(got)
+	require.Equal(t, consoleKeepBytes, gotSize, "a trim keeps exactly the newest MiB")
+	assert.True(t, bytes.Equal(body[len(body)-consoleKeepBytes:], got), "the newest MiB must survive, in order")
+}
+
+func TestConsoleLogReaper_LeavesFilesAtOrBelowTheLimit(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	small := consoleBody(3 << 20)
+	limit := consoleBody(consoleTrimAbove)
+	ps := writeConsole(t, dir, "i-small", small, now)
+	pl := writeConsole(t, dir, "i-limit", limit, now)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+
+	for p, want := range map[string][]byte{ps: small, pl: limit} {
+		got, err := os.ReadFile(p)
+		require.NoError(t, err)
+		assert.True(t, bytes.Equal(want, got), "%s must not change", filepath.Base(p))
+	}
+}
+
+// QEMU writes through a descriptor it keeps open, so the trim must leave the
+// file usable for that writer: no hole of NULs, and later output at the end.
+func TestConsoleLogReaper_AppendWriterKeepsWritingAfterTrim(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	p := filepath.Join(dir, "console-i-live.log")
+	w, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+	require.NoError(t, err)
+	defer w.Close()
+	_, err = w.Write(consoleBody(5 << 20))
+	require.NoError(t, err)
+
+	_, err = r.Sweep(context.Background())
+	require.NoError(t, err)
+
+	marker := []byte("after-trim-marker\n")
+	_, err = w.Write(marker)
+	require.NoError(t, err)
+
+	got, err := os.ReadFile(p)
+	require.NoError(t, err)
+	assert.Zero(t, bytes.Count(got, []byte{0}), "no NUL hole from a stale write offset")
+	assert.True(t, bytes.HasSuffix(got, marker), "the writer's next line lands at the end")
+	assert.LessOrEqual(t, len(got), consoleKeepBytes+len(marker))
+}
+
+func TestConsoleLogReaper_RemovesOnlyAgedRetiredFiles(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	old := now.Add(-consoleRemoveAfter - time.Minute)
+	recent := now.Add(-consoleRemoveAfter + time.Minute)
+	retiredOld := writeConsole(t, dir, "i-gone-old", []byte("x\n"), old)
+	retiredNew := writeConsole(t, dir, "i-gone-new", []byte("x\n"), recent)
+	require.NoError(t, os.Rename(retiredOld, RetiredConsolePath(retiredOld)))
+	require.NoError(t, os.Rename(retiredNew, RetiredConsolePath(retiredNew)))
+	require.NoError(t, os.Chtimes(RetiredConsolePath(retiredOld), old, old))
+	require.NoError(t, os.Chtimes(RetiredConsolePath(retiredNew), recent, recent))
+	// A live file that is not yet idle for an hour is kept.
+	stopped := writeConsole(t, dir, "i-stopped", []byte("boot\n"), recent)
+	other := filepath.Join(dir, "serial-i-x.sock.log.terminated")
+	require.NoError(t, os.WriteFile(other, []byte("x"), 0o644))
+	require.NoError(t, os.Chtimes(other, old, old))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped, "only the retired file older than an hour")
+
+	_, err = os.Stat(RetiredConsolePath(retiredOld))
+	assert.ErrorIs(t, err, os.ErrNotExist, "retired over an hour ago: removed")
+	_, err = os.Stat(RetiredConsolePath(retiredNew))
+	assert.NoError(t, err, "retired recently: kept for a late GetConsoleOutput")
+	_, err = os.Stat(stopped)
+	assert.NoError(t, err, "a live file idle for under an hour is kept")
+	_, err = os.Stat(other)
+	assert.NoError(t, err, "files that are not console logs are never touched")
+}
+
+func TestRetireConsoleLog(t *testing.T) {
+	dir := t.TempDir()
+	p := writeConsole(t, dir, "i-end", []byte("last words\n"), time.Now())
+
+	retireConsoleLog(p)
+	_, err := os.Stat(p)
+	assert.ErrorIs(t, err, os.ErrNotExist, "the live name no longer matches the shipper's glob")
+	got, err := os.ReadFile(RetiredConsolePath(p))
+	require.NoError(t, err)
+	assert.Equal(t, "last words\n", string(got))
+
+	assert.NotPanics(t, func() { retireConsoleLog(p) }, "a repeat finalize is a no-op")
+	assert.NotPanics(t, func() { retireConsoleLog("") }, "an instance that never got a console path is a no-op")
+}
+
+func TestConsoleLogReaper_TrimsOnlyLiveFiles(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	big := writeConsole(t, dir, "i-big", consoleBody(5<<20), now)
+	retired := writeConsole(t, dir, "i-gone", consoleBody(5<<20), now)
+	require.NoError(t, os.Rename(retired, RetiredConsolePath(retired)))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	info, err := os.Stat(big)
+	require.NoError(t, err)
+	assert.Equal(t, int64(consoleKeepBytes), info.Size())
+	info, err = os.Stat(RetiredConsolePath(retired))
+	require.NoError(t, err)
+	assert.Equal(t, int64(5<<20), info.Size(), "a retired file is final and is only aged out")
+}
+
+func TestExecute_ConsoleAlwaysAppends(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+		want string
+	}{
+		{"socket first boot", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c,logappend=on"},
+		{"microvm first boot", Config{MachineType: "microvm", ConsoleLogPath: "/c"}, "file,id=console0,path=/c,append=on"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := minimalConfig()
+			cfg.MachineType, cfg.SerialSocket = tc.cfg.MachineType, tc.cfg.SerialSocket
+			cfg.ConsoleLogPath = tc.cfg.ConsoleLogPath
+			cmd, err := cfg.Execute()
+			require.NoError(t, err)
+			var chardev string
+			for i, a := range cmd.Args {
+				if a == "-chardev" && i+1 < len(cmd.Args) {
+					chardev = cmd.Args[i+1]
+				}
+			}
+			assert.Contains(t, chardev, tc.want, "a truncating open would let an in-place trim leave a NUL hole")
+		})
+	}
+}
+
+// fakeQEMUProc fabricates procRoot/<pid> with a QEMU comm and the given open files.
+func fakeQEMUProc(t *testing.T, procRoot string, pid int, open ...string) {
+	t.Helper()
+	pdir := filepath.Join(procRoot, strconv.Itoa(pid))
+	require.NoError(t, os.MkdirAll(filepath.Join(pdir, "fd"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	for i, f := range open {
+		require.NoError(t, os.Symlink(f, filepath.Join(pdir, "fd", strconv.Itoa(i+3))))
+	}
+}
+
+func TestConsoleLogReaper_RemovesIdleLiveFileNothingHolds(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-stopped-then-terminated", []byte("boot\n"), now.Add(-consoleIdleAfter-time.Minute))
+	fresh := writeConsole(t, dir, "i-fresh", []byte("boot\n"), now.Add(-consoleIdleAfter+time.Minute))
+	fakeQEMUProc(t, r.procRoot, 4242)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	_, err = os.Stat(idle)
+	assert.ErrorIs(t, err, os.ErrNotExist, "idle over an hour and unopened: removed")
+	_, err = os.Stat(fresh)
+	assert.NoError(t, err, "written within the hour: kept")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileHeldByRealProcess(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	held := writeConsole(t, dir, "i-quiet-guest", []byte("boot\n"), now.Add(-48*time.Hour))
+	// A real child holds the file open, as QEMU holds its chardev log.
+	cmd := exec.Command("sh", "-c", "exec 3<"+held+"; echo ready; read _")
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+	out, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() { _ = stdin.Close(); _ = cmd.Wait() })
+	line := make([]byte, 6)
+	_, err = io.ReadFull(out, line)
+	require.NoError(t, err)
+	// Present the child to the reaper as a QEMU process using its real fd table.
+	r.procRoot = t.TempDir()
+	pdir := filepath.Join(r.procRoot, strconv.Itoa(cmd.Process.Pid))
+	require.NoError(t, os.MkdirAll(pdir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	require.NoError(t, os.Symlink(fmt.Sprintf("/proc/%d/fd", cmd.Process.Pid), filepath.Join(pdir, "fd")))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(held)
+	assert.NoError(t, err, "a quiet running guest keeps its log")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileWhenProcessTableUnreadable(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-unprovable", []byte("boot\n"), now.Add(-48*time.Hour))
+	pdir := filepath.Join(r.procRoot, "4242")
+	require.NoError(t, os.MkdirAll(pdir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	// A QEMU whose fd table cannot be read: a dangling fd entry that is a file, not a directory.
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "fd"), nil, 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(idle)
+	assert.NoError(t, err, "cannot prove it is unused: kept")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileWhenPidFileNamesLivePidTheScanMissed(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-quiet", []byte("boot\n"), now.Add(-48*time.Hour))
+	// This test process is alive but is no QEMU, so the fabricated scan never lists it.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "i-quiet.pid"), []byte(strconv.Itoa(os.Getpid())), 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(idle)
+	assert.NoError(t, err, "a live pid the scan cannot see means the scan is incomplete: nothing removed")
+}
+
+func TestConsoleLogReaper_IgnoresPidFileOfDeadProcess(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-gone", []byte("boot\n"), now.Add(-48*time.Hour))
+	cmd := exec.Command("true")
+	require.NoError(t, cmd.Run())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "i-gone.pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	_, err = os.Stat(idle)
+	assert.ErrorIs(t, err, os.ErrNotExist, "a pidfile for a dead process does not block removal")
+}

spinifex/runtime/compute/vm/console_retire_test.go

@@ -0,0 +1,85 @@
+package vm
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func consoleFixture(t *testing.T, id string) (*VM, string) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "console-"+id+".log")
+	require.NoError(t, os.WriteFile(path, []byte("last output\n"), 0o644))
+	return &VM{
+		ID:           id,
+		Status:       StateRunning,
+		InstanceType: "t3.micro",
+		Instance:     &ec2.Instance{},
+		Config:       Config{ConsoleLogPath: path},
+	}, path
+}
+
+func requireRetired(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.ErrorIs(t, err, os.ErrNotExist, "the shipper's console-*.log glob must no longer match")
+	got, err := os.ReadFile(RetiredConsolePath(path))
+	require.NoError(t, err)
+	assert.Equal(t, "last output\n", string(got), "the content survives the rename")
+}
+
+func requireLive(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.NoError(t, err)
+	_, err = os.Stat(RetiredConsolePath(path))
+	require.ErrorIs(t, err, os.ErrNotExist)
+}
+
+func TestTerminate_RetiresConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-term-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.Terminate(instance.ID))
+
+	requireRetired(t, path)
+}
+
+func TestMarkFailed_RetiresConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-failed-console")
+	instance.FirstLaunch = true
+	instance.Status = StatePending
+	m.Insert(instance)
+
+	m.MarkFailed(context.Background(), instance, "launch failed")
+	m.goroutineWg.Wait()
+
+	requireRetired(t, path)
+}
+
+func TestStop_KeepsConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-stop-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.Stop(instance.ID))
+
+	requireLive(t, path)
+}
+
+func TestStopAll_DrainKeepsConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-drain-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.StopAll())
+
+	requireLive(t, path)
+}

spinifex/runtime/compute/vm/desired_state_test.go

@@ -107,7 +107,7 @@ func TestVMMarshal_LegacyHostfwdNotWrittenBack(t *testing.T) {
 	assert.Contains(t, raw, "hostfwd_port_map")
 }
 
-// The other twelve command booleans had nowhere to go because nothing read them
+// The other eleven command booleans had nowhere to go because nothing read them
 // back. A record must carry none of them, so they are gone rather than unread.
 func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	legacy, err := json.Marshal(map[string]any{
@@ -115,7 +115,6 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 		"attributes": ec2v1.EC2CommandAttributes{
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			StartInstance:               true,
 			RebootInstance:              true,
 			AttachENI:                   true,
@@ -137,7 +136,7 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, field := range []string{
-		"attach_volume", "detach_volume", "drain_volume", "start_instance",
+		"attach_volume", "detach_volume", "start_instance",
 		"reboot_instance", "attach_eni", "detach_eni",
 		"associate_iam_instance_profile", "set_spot_lineage", "set_instance_tags",
 		"remove_instance_tags", "set_instance_monitoring",

spinifex/runtime/compute/vm/errors.go

@@ -32,11 +32,10 @@ var ErrVolumeNotDetachable = errors.New("volume cannot be detached")
 // supplies a device name that does not match the recorded attachment.
 var ErrVolumeDeviceMismatch = errors.New("volume device name mismatch")
 
-// ErrMountAmbiguous is returned by VolumeMounter.MountOne when the
+// ErrMountAmbiguous is returned by VolumeMounter.Mount and MountOne when a sent
-// backend acknowledged the mount but returned an empty URI. The mount may
+// publish got no usable export back (bad reply, timeout, cancelled caller), so
-// or may not have started serving NBD; AttachVolume must invoke UnmountOne
+// the provider may have started one; it is unpublished so it is not orphaned.
-// defensively so a half-started backend mount is not orphaned.
+var ErrMountAmbiguous = errors.New("publish reply carried no usable export")
-var ErrMountAmbiguous = errors.New("ebs.mount succeeded with empty URI")
 
 // ErrENINotAttached is returned by HotUnplugENI when the supplied ENI id
 // is not present in the instance's ENIRequests.AttachedByENIID map.

spinifex/runtime/compute/vm/firmware.go

@@ -1,21 +1,14 @@
 package vm
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
-	"io"
 	"os"
-	"sync"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 )
 
 // FirmwareCandidate is a (CODE, VARS template) pair shipped together by a
-// firmware package. CODE is mounted readonly as pflash unit 0; VARS seeds the
+// firmware package. CODE is mounted readonly as pflash unit 0; VARS is copied
-// instance's variable store, which is mounted as pflash unit 1 so EFI
+// into a per-VM viperblock volume on first boot and mounted as pflash unit 1
-// variables (BootOrder, BootNext, secure-boot state) survive reboots.
+// so EFI variables (BootOrder, BootNext, secure-boot state) survive reboots.
 type FirmwareCandidate struct {
 	Code         string
 	VarsTemplate string
@@ -55,67 +48,3 @@ func FirmwarePaths(arch string) (code, varsTemplate string, varsSize int64, err
 	}
 	return "", "", 0, fmt.Errorf("no UEFI firmware found for architecture %q; install ovmf (x86_64) or qemu-efi-aarch64 (arm64)", arch)
 }
-
-// NodeFirmware is the UEFI firmware this node runs for one architecture: the
-// candidate FirmwarePaths matched and the identity a variable store is
-// checked against.
-type NodeFirmware struct {
-	Code         string
-	VarsTemplate string
-	Identity     uefistore.FirmwareIdentity
-}
-
-// codeHashKey identifies one version of a CODE blob on disk, so a package
-// update that replaces the file is hashed again.
-type codeHashKey struct {
-	path    string
-	size    int64
-	modTime time.Time
-}
-
-// codeHashes caches CODE blob hashes by codeHashKey. The arm64 blob is 64 MiB,
-// and every UEFI launch needs its hash.
-var codeHashes sync.Map
-
-// NodeFirmwareFor returns the firmware this node runs for arch, with the
-// SHA-256 of its CODE blob.
-func NodeFirmwareFor(arch string) (NodeFirmware, error) {
-	code, vars, varsSize, err := FirmwarePaths(arch)
-	if err != nil {
-		return NodeFirmware{}, err
-	}
-	sum, err := codeSHA256(code)
-	if err != nil {
-		return NodeFirmware{}, err
-	}
-	return NodeFirmware{
-		Code:         code,
-		VarsTemplate: vars,
-		Identity:     uefistore.FirmwareIdentity{Arch: arch, VarsSize: varsSize, CodeSHA256: sum},
-	}, nil
-}
-
-func codeSHA256(path string) (string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return "", fmt.Errorf("open UEFI CODE %s: %w", path, err)
-	}
-	defer f.Close()
-	info, err := f.Stat()
-	if err != nil {
-		return "", fmt.Errorf("stat UEFI CODE %s: %w", path, err)
-	}
-	key := codeHashKey{path: path, size: info.Size(), modTime: info.ModTime()}
-	if cached, ok := codeHashes.Load(key); ok {
-		if sum, ok := cached.(string); ok {
-			return sum, nil
-		}
-	}
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("hash UEFI CODE %s: %w", path, err)
-	}
-	sum := hex.EncodeToString(h.Sum(nil))
-	codeHashes.Store(key, sum)
-	return sum, nil
-}

spinifex/runtime/compute/vm/guest_poweroff.go

@@ -0,0 +1,27 @@
+package vm
+
+import "log/slog"
+
+// HandleQEMUExit decides what a finished QEMU process means once startup was
+// confirmed. A wait error is a crash. A clean exit after the heartbeat saw the
+// guest power itself off is a customer stop, kept in the stopped bucket so
+// StartInstances can bring it back. Any other clean exit is left to the
+// heartbeat's crash detection.
+func (m *Manager) HandleQEMUExit(instance *VM, waitErr error) {
+	if waitErr != nil {
+		if m.deps.CrashHandler != nil {
+			m.deps.CrashHandler(instance, waitErr)
+		}
+		return
+	}
+	if !instance.guestPoweroff.Load() {
+		slog.Info("VM process exited cleanly", "instance", instance.ID)
+		return
+	}
+
+	slog.Info("Guest powered itself off, stopping instance", "instance", instance.ID)
+	m.UpdateState(instance.ID, func(v *VM) { v.DesiredState = DesiredStopped })
+	if err := m.Stop(instance.ID); err != nil {
+		slog.Error("Stop after guest poweroff failed", "instance", instance.ID, "err", err)
+	}
+}

spinifex/runtime/compute/vm/guest_poweroff_test.go

@@ -0,0 +1,155 @@
+package vm
+
+import (
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func guestPoweroffManager(t *testing.T) (*Manager, *fakeStateStore, *recordedTransitions, *[]string) {
+	t.Helper()
+	m, store, _, _, rt := shutdownTestManager(t)
+	var (
+		mu   sync.Mutex
+		down []string
+	)
+	m.SetDeps(Deps{
+		NodeID:          "test-node",
+		StateStore:      store,
+		VolumeMounter:   &fakeVolumeMounter{},
+		InstanceCleaner: &recordingInstanceCleaner{},
+		TransitionState: rt.apply,
+		ShutdownSignal:  func() bool { return false },
+		CrashHandler:    m.HandleCrash,
+		Hooks: ManagerHooks{
+			OnInstanceDown: func(id string) {
+				mu.Lock()
+				down = append(down, id)
+				mu.Unlock()
+			},
+		},
+	})
+	return m, store, rt, &down
+}
+
+func TestQMPHeartbeatPoll_MarksGuestPoweroff(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status  string
+		running bool
+		want    bool
+	}{
+		"guest shut down": {qmpStatusShutdown, false, true},
+		"guest running":   {"running", true, false},
+	} {
+		t.Run(name, func(t *testing.T) {
+			qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
+				if cmd.Execute == "query-status" {
+					return qmpStatusResponse(tc.status, tc.running)
+				}
+				return nil
+			})
+			defer cancel()
+
+			m := NewManager()
+			instance := &VM{ID: "i-1", Status: StateRunning, QMPClient: qmpClient}
+			m.Insert(instance)
+
+			require.True(t, m.qmpHeartbeatPoll(instance))
+			assert.Equal(t, tc.want, instance.guestPoweroff.Load())
+		})
+	}
+}
+
+// A guest that powered itself off is a stop: stopping then stopped, released
+// to the stopped bucket so StartInstances can bring it back, and never a crash.
+func TestHandleQEMUExit_GuestPoweroffStopsInstance(t *testing.T) {
+	m, store, rt, down := guestPoweroffManager(t)
+	scheduled := countScheduledRestarts(t)
+
+	instance := &VM{
+		ID:           "i-poweroff",
+		Status:       StateRunning,
+		InstanceType: "t3.micro",
+		Instance:     &ec2.Instance{},
+	}
+	instance.guestPoweroff.Store(true)
+	m.Insert(instance)
+
+	started := time.Now()
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Less(t, time.Since(started), qmpHeartbeatInterval,
+		"the stop must not wait for the next heartbeat")
+	assert.Equal(t, []InstanceState{StateStopping, StateStopped}, rt.targets(instance.ID),
+		"a guest poweroff goes stopping then stopped and never through error")
+	assert.Zero(t, scheduled.Load(), "a guest poweroff must not be restarted")
+	assert.Equal(t, []string{instance.ID}, *down)
+	stored, err := store.LoadStoppedInstance(instance.ID)
+	require.NoError(t, err)
+	require.NotNil(t, stored, "the instance must be startable from the stopped bucket")
+	assert.Equal(t, StateStopped, stored.Status)
+}
+
+// Without the guest-poweroff mark a clean exit is not ours to call a stop.
+func TestHandleQEMUExit_CleanExitWithoutMarkIsLeftToCrashRecovery(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+
+	instance := &VM{ID: "i-clean", Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+	m.Insert(instance)
+
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Empty(t, rt.targets(instance.ID), "no stop without the mark")
+	assert.Equal(t, StateRunning, m.Status(instance))
+}
+
+// A wait error is a crash even when the mark is set: the mark only vouches for
+// the quit we sent, not for a process that died some other way.
+func TestHandleQEMUExit_WaitErrorIsACrashEvenWithMark(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+	countScheduledRestarts(t)
+
+	instance := &VM{ID: "i-killed", Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+	instance.guestPoweroff.Store(true)
+	m.Insert(instance)
+
+	m.HandleQEMUExit(instance, errors.New("signal: killed"))
+
+	assert.Contains(t, rt.targets(instance.ID), StateError)
+	assert.NotContains(t, rt.targets(instance.ID), StateStopped)
+}
+
+// One heartbeat poll is enough to see the poweroff and end QEMU, and the exit
+// then stops the instance. It never goes through the failure count that
+// precedes crash recovery, which takes three polls to reach. The interval is a
+// constant, so "within one interval" is a single poll with no waiting.
+func TestGuestPoweroff_ReachesStoppedFromOnePollWithoutCrashCounting(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+	scheduled := countScheduledRestarts(t)
+
+	qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
+		if cmd.Execute == "query-status" {
+			return qmpStatusResponse(qmpStatusShutdown, false)
+		}
+		return nil
+	})
+	defer cancel()
+	instance := &VM{
+		ID: "i-one-poll", Status: StateRunning, InstanceType: "t3.micro",
+		Instance: &ec2.Instance{}, QMPClient: qmpClient,
+	}
+	m.Insert(instance)
+
+	require.True(t, m.qmpHeartbeatPoll(instance), "the poll keeps running until QEMU exits")
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Equal(t, []InstanceState{StateStopping, StateStopped}, rt.targets(instance.ID))
+	assert.Zero(t, instance.Health.QMPConsecutiveFailures, "a powered-off guest is not a failed heartbeat")
+	assert.Zero(t, scheduled.Load(), "no crash restart")
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -92,9 +92,6 @@ func (m *Manager) Start(ctx context.Context, id string) error {
 	if !ok {
 		return fmt.Errorf("%w: %s", ErrInstanceNotFound, id)
 	}
-	if err := CheckUEFIVarStore(instance); err != nil {
-		return err
-	}
 	// StateError is rejected by launchStillValid; move to Pending first so
 	// launch proceeds (resource re-allocation is the caller's responsibility).
 	if m.Status(instance) == StateError && m.deps.TransitionState != nil {
@@ -130,13 +127,6 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		return nil
 	}
 
-	// Before the reason below is cleared, so a refused start changes nothing.
-	if err := CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
-			"instanceId", instance.ID, "err", err)
-		return err
-	}
-
 	// A reason left by the stop, or by a restore that found this node could not
 	// schedule the instance, describes a state the instance is now leaving.
 	// Clearing it here keeps the node the only writer of the field.
@@ -158,16 +148,6 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		}
 	}
 
-	// Before the mount, so a store this node cannot boot fails the launch
-	// with nothing to roll back.
-	uefiCtx, uefiSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.uefi_var_store")
-	uefiErr := m.prepareUEFIVarStore(uefiCtx, instance)
-	endSpanWithError(uefiSpan, uefiErr)
-	if uefiErr != nil {
-		slog.ErrorContext(ctx, "Failed to prepare the UEFI variable store", "instanceId", instance.ID, "err", uefiErr)
-		return uefiErr
-	}
-
 	_, mountSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.mount_volumes")
 	mountErr := m.deps.VolumeMounter.Mount(ctx, instance)
 	endSpanWithError(mountSpan, mountErr)
@@ -196,7 +176,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 	}
 
 	_, qemuSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.start_qemu")
-	qemuErr := m.startQEMU(instance, false)
+	qemuErr := m.startQEMU(instance)
 	endSpanWithError(qemuSpan, qemuErr)
 	if qemuErr != nil {
 		slog.ErrorContext(ctx, "Failed to launch instance", "err", qemuErr)
@@ -302,9 +282,9 @@ func qemuStderrLevel(line string) slog.Level {
 }
 
 // startQEMU launches the QEMU process for instance and waits for startup to
-// confirm. appendConsole keeps the console log of the process it replaces, so a
+// confirm. The console log is opened in append mode, so a replacement process
-// reboot does not erase what the guest printed before it.
+// keeps what the guest printed before it.
-func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
+func (m *Manager) startQEMU(instance *VM) error {
 	pidFile, err := hostprocess.GeneratePidFile(instance.ID)
 	if err != nil {
 		slog.Error("Failed to generate PID file", "err", err)
@@ -344,14 +324,6 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 		instance.Config.IOThreads = append(instance.Config.IOThreads, driveCfg.IOThreads...)
 		instance.Config.Devices = append(instance.Config.Devices, driveCfg.Devices...)
 		instance.Config.Blockdevs = append(instance.Config.Blockdevs, driveCfg.Blockdevs...)
-
-		varsDrive, err := m.uefiVarsDrive(instance)
-		if err != nil {
-			return err
-		}
-		if varsDrive != nil {
-			instance.Config.Drives = append(instance.Config.Drives, *varsDrive)
-		}
 	}
 
 	if instance.DirectBoot {
@@ -532,7 +504,6 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 		}
 		launchCfg = &composed
 	}
-	launchCfg.ConsoleLogAppend = appendConsole
 
 	// An NBD export is waited on until it accepts connections; a block export
 	// is local to the host and only has to exist. An invalid request has no
@@ -635,13 +606,7 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 			slog.Info("Replaced VM process exited", "instance", instance.ID, "pid", cmd.Process.Pid, "err", waitErr)
 			return
 		}
-		if waitErr != nil {
+		m.HandleQEMUExit(instance, waitErr)
-			if m.deps.CrashHandler != nil {
-				m.deps.CrashHandler(instance, waitErr)
-			}
-		} else {
-			slog.Info("VM process exited cleanly", "instance", instance.ID)
-		}
 	}()
 
 	pid := <-processChan
@@ -996,6 +961,7 @@ func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
 	if err == nil && qmpStatus.Status == qmpStatusShutdown &&
 		m.Status(instance) == StateRunning {
 		slog.InfoContext(ctx, "Guest powered itself off, ending QEMU", "instance", instance.ID)
+		instance.guestPoweroff.Store(true)
 		if _, quitErr := sendQMPCommand(ctx, instance.QMPClient,
 			qmp.QMPCommand{Execute: "quit"}, instance.ID); quitErr != nil {
 			slog.WarnContext(ctx, "QMP quit failed after guest shutdown",
@@ -1350,7 +1316,7 @@ func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serial
 		CPUCount:       vCPUs,
 		Architecture:   architecture,
 		InstanceType:   instanceType,
-		UseUEFI:        isUEFIBootMode(bootMode),
+		UseUEFI:        bootMode == "uefi" || bootMode == "uefi-preferred",
 
 		// Present EC2-shaped DMI so stock cloud-init activates the Ec2 datasource.
 		SMBIOSUUID:         ec2SMBIOSUUID(instanceID),
@@ -1443,8 +1409,7 @@ func buildDiskBootDrives(requests []EBSRequest, cpuCount int, machineType string
 //
 // The boot volume is a -blockdev node named "os" under a bootindex=1 device
 // that carries werror/rerror; it is never detached, so it keeps fixed names.
-// EFI volumes emit pflash unit=1, as uefiVarsDrive does for a variable store
+// EFI volumes emit pflash unit=1; Config.Execute adds the CODE blob (unit=0).
-// pointer; Config.Execute adds the CODE blob (unit=0).
 //
 // Every other (data) volume is given the same named node-name/device-id/
 // iothread-id shape AttachVolume's hot-plug path uses, via a -blockdev entry

spinifex/runtime/compute/vm/lifecycle_test.go

@@ -1042,7 +1042,7 @@ func TestStartQEMU_DirectBoot_PrimaryTapError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:01",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "setup tap device")
@@ -1070,7 +1070,7 @@ func TestStartQEMU_DirectBoot_BridgeEnsureError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:04",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "ensure IMDS bridge")
@@ -1105,7 +1105,7 @@ func TestStartQEMU_DirectBoot_ExtraENITapError(t *testing.T) {
 		},
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "extra ENI")
@@ -1126,7 +1126,7 @@ func TestStartQEMU_DirectBoot_MgmtTapError(t *testing.T) {
 		MgmtMAC:      "02:aa:bb:cc:dd:ff",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1152,7 +1152,7 @@ func TestStartQEMU_DiskBoot_MgmtTapCreated(t *testing.T) {
 	}
 	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: "nbd:unix:/tmp/root.sock", Boot: true}}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1174,7 +1174,7 @@ func TestStartQEMU_DirectBoot_InstanceTypeNotFound(t *testing.T) {
 		DirectBoot:   true,
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "x9.enormous")
@@ -1190,7 +1190,7 @@ func TestStartQEMU_BlockDeviceMissing(t *testing.T) {
 	instance := directBootInstance("i-blk-missing",
 		EBSRequest{Name: "vol-blk", BlockDevice: missing, BlockSizeBytes: 1 << 20})
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "block device not present for vol-blk")
@@ -2089,12 +2089,12 @@ func TestStartQEMU_DirectBoot_LaunchesMountedDataVolumeWithoutSavingIt(t *testin
 	second := nbdListener(t)
 
 	instance := directBootInstance("i-db-vol", EBSRequest{Name: "vol-data-a", NBDURI: first})
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	// A later launch from the same saved base, after the volume was remounted
 	// on a different endpoint.
 	instance.EBSRequests.Requests[0].NBDURI = second
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 2)
@@ -2136,7 +2136,7 @@ func TestStartQEMU_DirectBoot_LaunchesTheEndpointItProbed(t *testing.T) {
 		instance.EBSRequests.Mu.Unlock()
 		return orig(uri, timeout)
 	}
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	assert.Equal(t, []string{probedEP}, probed)
 	got := launches()
@@ -2151,7 +2151,7 @@ func TestStartQEMU_DirectBoot_NoVolumesLaunchesTheBase(t *testing.T) {
 	launches := withFakeQEMU(t)
 
 	instance := directBootInstance("i-db-novol")
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 1)
@@ -2176,7 +2176,7 @@ func TestStartQEMU_DirectBoot_InvalidDataRequestIsNotLaunched(t *testing.T) {
 			launches := withFakeQEMU(t)
 
 			instance := directBootInstance("i-db-bad", tt.req)
-			err := m.startQEMU(instance, false)
+			err := m.startQEMU(instance)
 
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), tt.wantErr)
@@ -2197,7 +2197,7 @@ func TestStartQEMU_DiskBoot_DataVolumeStaysOnThePCIPath(t *testing.T) {
 		{Name: "vol-data-a", NBDURI: ep},
 	}
 	require.NoError(t, os.WriteFile(strings.TrimPrefix(instance.EBSRequests.Requests[0].NBDURI, "nbd:unix:"), nil, 0o600))
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 1)
@@ -2252,7 +2252,7 @@ func TestStartQEMU_DiskBoot_BootVolumeIsABlockdevNode(t *testing.T) {
 			req := tt.req(t)
 			instance := &VM{ID: "i-boot-blockdev", InstanceType: "t3.nano"}
 			instance.EBSRequests.Requests = []EBSRequest{req}
-			require.NoError(t, m.startQEMU(instance, false))
+			require.NoError(t, m.startQEMU(instance))
 
 			got := launches()
 			require.Len(t, got, 1)
@@ -2283,7 +2283,7 @@ func TestStartQEMU_DiskBoot_NoBootVolumeIsNotLaunched(t *testing.T) {
 
 			instance := &VM{ID: "i-no-boot", InstanceType: "t3.nano"}
 			instance.EBSRequests.Requests = tt.requests
-			err := m.startQEMU(instance, false)
+			err := m.startQEMU(instance)
 
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), "no boot volume among the instance's EBS requests")

spinifex/runtime/compute/vm/manager.go

@@ -87,9 +87,6 @@ type Deps struct {
 	// (predastore + viperblock) is confirmed up. Nil is treated as ready,
 	// preserving behaviour for tests and nodes with storage unwired.
 	BackingStoreReady func() bool
-
-	// UEFIVarStore is where UEFI variable stores are read from and kept.
-	UEFIVarStore UEFIVarStoreDeps
 }
 
 // Manager owns the in-memory map of running VMs on this node and every

spinifex/runtime/compute/vm/reboot.go

@@ -52,7 +52,7 @@ const (
 // startRelaunchedQEMU starts the replacement process. A var so tests can stand
 // in for QEMU.
 var startRelaunchedQEMU = func(m *Manager, instance *VM) error {
-	return m.startQEMU(instance, true)
+	return m.startQEMU(instance)
 }
 
 // errRelaunchFailed marks a reboot that ended the old QEMU and could not start

spinifex/runtime/compute/vm/reboot_test.go

@@ -465,21 +465,18 @@ func TestThreadStatFields(t *testing.T) {
 	assert.Empty(t, threadStatFields("garbage"))
 }
 
-func TestExecute_ConsoleLogAppendOnlyWhenAsked(t *testing.T) {
+func TestExecute_ConsoleLogAppendOnEveryBoot(t *testing.T) {
 	for _, tc := range []struct {
-		name   string
+		name string
-		cfg    Config
+		cfg  Config
-		want   string
+		want string
-		absent string
 	}{
-		{"socket fresh", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c", "logappend"},
+		{"socket", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c,logappend=on"},
-		{"socket reboot", Config{SerialSocket: "/s", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "logfile=/c,logappend=on", ""},
+		{"file", Config{MachineType: "microvm", ConsoleLogPath: "/c"}, "file,id=console0,path=/c,append=on"},
-		{"file reboot", Config{MachineType: "microvm", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "file,id=console0,path=/c,append=on", ""},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			cfg := minimalConfig()
-			cfg.MachineType, cfg.SerialSocket = tc.cfg.MachineType, tc.cfg.SerialSocket
+			cfg.MachineType, cfg.SerialSocket, cfg.ConsoleLogPath = tc.cfg.MachineType, tc.cfg.SerialSocket, tc.cfg.ConsoleLogPath
-			cfg.ConsoleLogPath, cfg.ConsoleLogAppend = tc.cfg.ConsoleLogPath, tc.cfg.ConsoleLogAppend
 			cmd, err := cfg.Execute()
 			require.NoError(t, err)
 			var chardev string
@@ -489,9 +486,6 @@ func TestExecute_ConsoleLogAppendOnlyWhenAsked(t *testing.T) {
 				}
 			}
 			assert.Contains(t, chardev, tc.want)
-			if tc.absent != "" {
-				assert.NotContains(t, chardev, tc.absent)
-			}
 		})
 	}
 }

spinifex/runtime/compute/vm/shutdown.go

@@ -389,6 +389,8 @@ func (m *Manager) finalizeTerminated(instance *VM) error {
 		instance.TerminatedAt = time.Now()
 	}
 
+	retireConsoleLog(instance.Config.ConsoleLogPath)
+
 	if m.deps.StateStore != nil {
 		if err := m.deps.StateStore.WriteTerminatedInstance(instance.ID, instance); err != nil {
 			slog.Error("Failed to write terminated instance to KV, keeping in local state for retry",

spinifex/runtime/compute/vm/uefi_varstore.go

@@ -1,390 +0,0 @@
-package vm
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-const (
-	// uefiStoreDirName is the directory under DataDir that holds one
-	// subdirectory per instance with its live variable store.
-	uefiStoreDirName   = "uefi"
-	uefiVarsFileName   = "vars.fd"
-	uefiSeededFileName = "seeded.json"
-
-	// uefiStoreGetTimeout bounds the one get a launch makes for a fixed-size
-	// object, so a wedged object store fails the start rather than holding it.
-	uefiStoreGetTimeout = 10 * time.Second
-
-	// uefiStoreBodySlack is how far a compressed body may exceed its region
-	// before it is refused; gzip grows incompressible data by far less.
-	uefiStoreBodySlack = 1 << 20
-
-	// uefiStateReasonCode is the StateReason a launch refused over its
-	// variable store records, as EC2 reports an internal launch failure.
-	uefiStateReasonCode = "Server.InternalError"
-)
-
-// ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
-// store and carries no EFI volume, so it has nothing to boot its variables from.
-var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
-
-// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
-// store and also carries an EFI volume, so either one would be ignored.
-var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
-
-// ErrUEFIFirmwareMismatch is a variable store written under firmware whose
-// architecture or VARS size differs from this node's, which pflash cannot load.
-var ErrUEFIFirmwareMismatch = errors.New("UEFI variable store firmware does not match this node")
-
-// ErrUEFIVarStoreUnusable is a variable store object that is missing or does
-// not decode to a region of this node's VARS size. Retrying does not help.
-var ErrUEFIVarStoreUnusable = errors.New("UEFI variable store object is unusable")
-
-// UEFIVarStoreDeps is where a node reads instance variable stores from and
-// keeps the live copy QEMU opens. DataDir is the daemon's persistent data
-// directory, so the copy survives a daemon restart and a host reboot.
-type UEFIVarStoreDeps struct {
-	Objects objectstore.ObjectStore
-	Bucket  string
-	DataDir string
-}
-
-// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
-type uefiSeed struct {
-	Key      string    `json:"key"`
-	SeededAt time.Time `json:"seeded_at"`
-}
-
-// isUEFIBootMode reports whether bootMode boots the guest through UEFI firmware.
-func isUEFIBootMode(bootMode string) bool {
-	return bootMode == "uefi" || bootMode == "uefi-preferred"
-}
-
-// usesUEFIVarStore reports whether v boots through UEFI and so needs a
-// variable store. A direct-boot microvm boots qboot and has none.
-func usesUEFIVarStore(v *VM) bool {
-	return !v.DirectBoot && isUEFIBootMode(v.BootMode)
-}
-
-// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
-// two, before anything about it is changed. An EFI volume still counts as a
-// store until RunInstances seeds pointers instead. The caller owns v.
-func CheckUEFIVarStore(v *VM) error {
-	if !usesUEFIVarStore(v) {
-		return nil
-	}
-	v.EBSRequests.Mu.Lock()
-	hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
-	v.EBSRequests.Mu.Unlock()
-
-	switch {
-	case v.UEFIVarStore == nil && !hasEFIVolume:
-		return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
-	case v.UEFIVarStore != nil && hasEFIVolume:
-		return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
-	}
-	return nil
-}
-
-// uefiInstanceDir is the directory holding instanceID's vars.fd and
-// seeded.json. An ID that is not a single path segment is refused.
-func uefiInstanceDir(dataDir, instanceID string) (string, error) {
-	if !strings.HasPrefix(instanceID, "i-") || strings.ContainsAny(instanceID, `/\`) || instanceID != filepath.Clean(instanceID) {
-		return "", fmt.Errorf("instance ID %q cannot name a UEFI store directory", instanceID)
-	}
-	return filepath.Join(dataDir, uefiStoreDirName, instanceID), nil
-}
-
-// uefiVarsPath is the live variable store QEMU opens as pflash unit 1.
-func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
-	dir, err := uefiInstanceDir(m.deps.UEFIVarStore.DataDir, instanceID)
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(dir, uefiVarsFileName), nil
-}
-
-// uefiVarsDrive is pflash unit 1 from this node's copy of the store the
-// record points at, or nil with no pointer. Launch seeds the copy and a
-// reboot reuses it, so a missing one fails rather than booting without it.
-func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
-	hasPointer := false
-	m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
-	if !hasPointer || !usesUEFIVarStore(instance) {
-		return nil, nil
-	}
-	path, err := m.uefiVarsPath(instance.ID)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := os.Stat(path); err != nil {
-		return nil, fmt.Errorf("UEFI variable store for %s: %w", instance.ID, err)
-	}
-	return &Drive{File: path, Format: "raw", If: "pflash", Unit: 1}, nil
-}
-
-// prepareUEFIVarStore leaves vars.fd holding the store the record points at,
-// reusing this node's copy when seeded.json names the pointer and fetching
-// the object otherwise. The object and the record's pointer are only read.
-func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
-	var ref *UEFIVarStoreRef
-	m.Inspect(instance, func(v *VM) {
-		if v.UEFIVarStore != nil {
-			cp := *v.UEFIVarStore
-			ref = &cp
-		}
-	})
-	if ref == nil || !usesUEFIVarStore(instance) {
-		return nil
-	}
-
-	deps := m.deps.UEFIVarStore
-	if deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" {
-		return errors.New("UEFI variable store access is not wired on this node")
-	}
-	if name, ok := strings.CutPrefix(ref.Key, uefistore.KeyPrefix(instance.ID)); !ok || name == "" || strings.Contains(name, "/") {
-		return fmt.Errorf("%w: pointer %q is not under %s", ErrUEFIVarStoreUnusable, ref.Key, uefistore.KeyPrefix(instance.ID))
-	}
-	if m.deps.InstanceTypes == nil {
-		return errors.New("InstanceTypeResolver not wired")
-	}
-	spec, ok := m.deps.InstanceTypes.Resolve(instance.InstanceType)
-	if !ok {
-		return fmt.Errorf("instance type %s not found", instance.InstanceType)
-	}
-	fw, err := NodeFirmwareFor(spec.Architecture)
-	if err != nil {
-		return err
-	}
-	if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
-		return err
-	}
-
-	dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
-	if err != nil {
-		return err
-	}
-	if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
-		recordUEFIStoreSeed("local")
-		slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
-		return nil
-	}
-
-	start := time.Now()
-	region, err := fetchUEFIVars(ctx, deps, ref.Key, fw.Identity.VarsSize)
-	if err != nil {
-		return err
-	}
-	if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
-		return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
-	}
-	seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
-	if err != nil {
-		return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
-	}
-	if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
-		return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
-	}
-	recordUEFIStoreSeed("object")
-	slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
-		"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
-	return nil
-}
-
-// checkUEFIFirmware fails a store whose arch or VARS size differs from this
-// node's, naming both firmwares in the log and the StateReason. A different
-// CODE build boots, because a package update changes it on every node.
-func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeFirmware) error {
-	stored, node := ref.Firmware, fw.Identity
-	var kind string
-	switch {
-	case stored.Arch != node.Arch:
-		kind = "arch"
-	case stored.VarsSize != node.VarsSize:
-		kind = "vars_size"
-	case stored.CodeSHA256 != node.CodeSHA256:
-		recordUEFIFirmwareMismatch("code")
-		slog.Warn("UEFI variable store was written under a different CODE build; booting it",
-			"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode,
-			"storeCodeSha256", stored.CodeSHA256, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
-		return nil
-	default:
-		return nil
-	}
-
-	recordUEFIFirmwareMismatch(kind)
-	msg := fmt.Sprintf("%s: the UEFI variable store %s was written under firmware arch=%s vars_size=%d code_sha256=%s, and this node runs arch=%s vars_size=%d code_sha256=%s (%s)",
-		uefiStateReasonCode, ref.Key, stored.Arch, stored.VarsSize, stored.CodeSHA256,
-		node.Arch, node.VarsSize, node.CodeSHA256, fw.Code)
-	m.Inspect(instance, func(v *VM) {
-		if v.Instance != nil {
-			v.Instance.StateReason = &ec2.StateReason{Code: aws.String(uefiStateReasonCode), Message: aws.String(msg)}
-		}
-	})
-	slog.Error("UEFI variable store firmware does not match this node; refusing the launch",
-		"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode, "mismatch", kind,
-		"storeArch", stored.Arch, "storeVarsSize", stored.VarsSize, "storeCodeSha256", stored.CodeSHA256,
-		"nodeArch", node.Arch, "nodeVarsSize", node.VarsSize, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
-	return fmt.Errorf("%w: %s", ErrUEFIFirmwareMismatch, msg)
-}
-
-// reuseUEFIVars reports whether dir already holds the store key names: a
-// seeded.json naming key beside a vars.fd of the region's size.
-func reuseUEFIVars(dir, key string, varsSize int64) bool {
-	data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
-	if err != nil {
-		return false
-	}
-	var seed uefiSeed
-	if err := json.Unmarshal(data, &seed); err != nil || seed.Key != key {
-		return false
-	}
-	info, err := os.Stat(filepath.Join(dir, uefiVarsFileName))
-	if err != nil || info.Size() != varsSize {
-		slog.Warn("UEFI variable store seeded here has no usable vars.fd; fetching it again", "dir", dir, "key", key)
-		return false
-	}
-	return true
-}
-
-// fetchUEFIVars gets key and returns its decompressed region, which must be
-// exactly varsSize bytes. A failed get is retryable; a missing or malformed
-// object is not.
-func fetchUEFIVars(ctx context.Context, deps UEFIVarStoreDeps, key string, varsSize int64) ([]byte, error) {
-	getCtx, cancel := context.WithTimeout(ctx, uefiStoreGetTimeout)
-	defer cancel()
-
-	out, err := deps.Objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(deps.Bucket), Key: aws.String(key)})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return nil, fmt.Errorf("%w: %s does not exist", ErrUEFIVarStoreUnusable, key)
-		}
-		return nil, fmt.Errorf("get UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
-	}
-	defer out.Body.Close()
-
-	limit := varsSize + uefiStoreBodySlack
-	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
-	if err != nil {
-		return nil, fmt.Errorf("read UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
-	}
-	if int64(len(raw)) > limit {
-		return nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrUEFIVarStoreUnusable, key, limit)
-	}
-
-	gz, err := gzip.NewReader(bytes.NewReader(raw))
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s is not gzip: %w", ErrUEFIVarStoreUnusable, key, err)
-	}
-	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s does not decompress: %w", ErrUEFIVarStoreUnusable, key, err)
-	}
-	if int64(len(region)) != varsSize {
-		return nil, fmt.Errorf("%w: %s holds a %d-byte region, and this node's VARS template is %d bytes",
-			ErrUEFIVarStoreUnusable, key, len(region), varsSize)
-	}
-	return region, nil
-}
-
-// writeFileAtomic replaces dir/name with data by temporary file and rename,
-// syncing both, so a crash leaves the old file or the new one.
-func writeFileAtomic(dir, name string, data []byte) error {
-	if err := os.MkdirAll(dir, 0o700); err != nil {
-		return err
-	}
-	tmp, err := os.CreateTemp(dir, "."+name+".*")
-	if err != nil {
-		return err
-	}
-	tmpName := tmp.Name()
-	if _, err := tmp.Write(data); err != nil {
-		_ = tmp.Close()
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := tmp.Sync(); err != nil {
-		_ = tmp.Close()
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := tmp.Close(); err != nil {
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := os.Rename(tmpName, filepath.Join(dir, name)); err != nil {
-		_ = os.Remove(tmpName)
-		return err
-	}
-	d, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-	defer d.Close()
-	return d.Sync()
-}
-
-// uefiStoreMeterName is the instrumentation scope for variable store launches.
-const uefiStoreMeterName = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/uefistore"
-
-var (
-	uefiStoreMetricsOnce   sync.Once
-	uefiStoreSeedCounter   metric.Int64Counter
-	uefiFirmwareMismatches metric.Int64Counter
-)
-
-func uefiStoreMetrics() {
-	uefiStoreMetricsOnce.Do(func() {
-		meter := otel.Meter(uefiStoreMeterName)
-		var err error
-		uefiStoreSeedCounter, err = meter.Int64Counter("mulga.vm.uefi_store.seeds",
-			metric.WithDescription("UEFI launches by where their variable store came from: this node's copy or the object store."),
-			metric.WithUnit("{launch}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-		uefiFirmwareMismatches, err = meter.Int64Counter("mulga.vm.uefi_store.firmware_mismatches",
-			metric.WithDescription("UEFI launches whose variable store was written under different firmware, by what differed. Only kind=code boots."),
-			metric.WithUnit("{launch}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-	})
-}
-
-func recordUEFIStoreSeed(source string) {
-	uefiStoreMetrics()
-	if uefiStoreSeedCounter != nil {
-		uefiStoreSeedCounter.Add(context.Background(), 1, metric.WithAttributes(attribute.String("source", source)))
-	}
-}
-
-func recordUEFIFirmwareMismatch(kind string) {
-	uefiStoreMetrics()
-	if uefiFirmwareMismatches != nil {
-		uefiFirmwareMismatches.Add(context.Background(), 1, metric.WithAttributes(attribute.String("kind", kind)))
-	}
-}

spinifex/runtime/compute/vm/uefi_varstore_test.go

@@ -1,539 +0,0 @@
-package vm
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
-	"go.opentelemetry.io/otel/sdk/metric/metricdata"
-)
-
-const (
-	testUEFIBucket   = "instances"
-	testUEFIVarsSize = 4096
-	testUEFIInstance = "i-0123456789abcdef0"
-)
-
-var (
-	uefiMetricsReaderOnce sync.Once
-	uefiMetricsReader     *sdkmetric.ManualReader
-)
-
-// uefiCounter returns the sum of the named counter's points whose attribute
-// key has value.
-func uefiCounter(t *testing.T, name, key, value string) int64 {
-	t.Helper()
-	uefiMetricsReaderOnce.Do(func() {
-		uefiMetricsReader = sdkmetric.NewManualReader()
-		otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(uefiMetricsReader)))
-	})
-	var rm metricdata.ResourceMetrics
-	require.NoError(t, uefiMetricsReader.Collect(context.Background(), &rm))
-	var total int64
-	for _, sm := range rm.ScopeMetrics {
-		for _, m := range sm.Metrics {
-			sum, ok := m.Data.(metricdata.Sum[int64])
-			if m.Name != name || !ok {
-				continue
-			}
-			for _, dp := range sum.DataPoints {
-				if v, ok := dp.Attributes.Value(attribute.Key(key)); ok && v.AsString() == value {
-					total += dp.Value
-				}
-			}
-		}
-	}
-	return total
-}
-
-// uefiFixture is a node with one x86_64 firmware candidate, an in-memory
-// object store and a data directory.
-type uefiFixture struct {
-	m        *Manager
-	mounter  *fakeVolumeMounter
-	objects  *objectstore.MemoryObjectStore
-	dataDir  string
-	codePath string
-	identity uefistore.FirmwareIdentity
-}
-
-func newUEFIFixture(t *testing.T) *uefiFixture {
-	t.Helper()
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	fwDir := t.TempDir()
-	code := filepath.Join(fwDir, "OVMF_CODE.fd")
-	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
-	codeBytes := []byte("ovmf code build 1")
-	require.NoError(t, os.WriteFile(code, codeBytes, 0o600))
-	require.NoError(t, os.WriteFile(vars, make([]byte, testUEFIVarsSize), 0o600))
-	orig := FirmwarePathCandidates
-	FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
-	t.Cleanup(func() { FirmwarePathCandidates = orig })
-
-	sum := sha256.Sum256(codeBytes)
-	f := &uefiFixture{
-		mounter:  &fakeVolumeMounter{},
-		objects:  objectstore.NewMemoryObjectStore(),
-		dataDir:  t.TempDir(),
-		codePath: code,
-		identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
-	}
-	f.m = NewManagerWithDeps(Deps{
-		VolumeMounter: f.mounter,
-		InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
-		UEFIVarStore:  UEFIVarStoreDeps{Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir},
-	})
-	return f
-}
-
-// region is a VARS region of the fixture's size whose bytes say which store it is.
-func region(tag string) []byte {
-	b := make([]byte, testUEFIVarsSize)
-	copy(b, tag)
-	return b
-}
-
-func gzipped(t *testing.T, data []byte) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	_, err := zw.Write(data)
-	require.NoError(t, err)
-	require.NoError(t, zw.Close())
-	return buf.Bytes()
-}
-
-func (f *uefiFixture) putObject(t *testing.T, key string, body []byte) {
-	t.Helper()
-	_, err := f.objects.PutObject(context.Background(), &s3.PutObjectInput{
-		Bucket: aws.String(testUEFIBucket), Key: aws.String(key), Body: bytes.NewReader(body),
-	})
-	require.NoError(t, err)
-}
-
-func (f *uefiFixture) objectBody(t *testing.T, key string) []byte {
-	t.Helper()
-	out, err := f.objects.GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(testUEFIBucket), Key: aws.String(key)})
-	require.NoError(t, err)
-	defer out.Body.Close()
-	b, err := io.ReadAll(out.Body)
-	require.NoError(t, err)
-	return b
-}
-
-func (f *uefiFixture) instanceDir() string {
-	return filepath.Join(f.dataDir, "uefi", testUEFIInstance)
-}
-
-func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, vars []byte) {
-	t.Helper()
-	require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
-	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "vars.fd"), vars, 0o600))
-	seed, err := json.Marshal(uefiSeed{Key: seededKey, SeededAt: time.Now()})
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), seed, 0o600))
-}
-
-func (f *uefiFixture) localVars(t *testing.T) []byte {
-	t.Helper()
-	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "vars.fd"))
-	require.NoError(t, err)
-	return b
-}
-
-func (f *uefiFixture) seededKey(t *testing.T) string {
-	t.Helper()
-	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "seeded.json"))
-	require.NoError(t, err)
-	var seed uefiSeed
-	require.NoError(t, json.Unmarshal(b, &seed))
-	return seed.Key
-}
-
-func (f *uefiFixture) instance(key string, fw uefistore.FirmwareIdentity) *VM {
-	v := &VM{
-		ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Status: StateStopped,
-		Instance: &ec2.Instance{},
-	}
-	if key != "" {
-		v.UEFIVarStore = &UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-a", WrittenAt: time.Unix(1700000000, 0).UTC()}
-	}
-	f.m.Insert(v)
-	return v
-}
-
-func keyFor(name string) string { return uefistore.KeyPrefix(testUEFIInstance) + name }
-
-func TestPrepareUEFIVarStore_FetchesTheObjectAndRecordsTheSeed(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("11111111-1111-1111-1111-111111111111")
-	f.putObject(t, key, gzipped(t, region("converted")))
-	instance := f.instance(key, f.identity)
-	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-	assert.Equal(t, region("converted"), f.localVars(t), "vars.fd must hold the decompressed region")
-	assert.Equal(t, key, f.seededKey(t))
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object"))
-	assert.Equal(t, key, instance.UEFIVarStore.Key, "seeding only reads the pointer")
-}
-
-// A start on the node that seeded the pointer boots that node's copy, which
-// can be newer than the object, and makes no get at all.
-func TestPrepareUEFIVarStore_ReusesTheLocalCopyWhenSeededNamesThePointer(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("22222222-2222-2222-2222-222222222222")
-	f.writeLocal(t, key, region("guest wrote this"))
-	instance := f.instance(key, f.identity)
-	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance), "no object exists, so a get would fail")
-
-	assert.Equal(t, region("guest wrote this"), f.localVars(t))
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
-}
-
-// Once the pointer has moved past what this node seeded, the local copy is
-// stale and the pointed object replaces it.
-func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing.T) {
-	tests := []struct {
-		name  string
-		local func(f *uefiFixture, t *testing.T)
-	}{
-		{name: "seeded from an older pointer", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("older"), region("stale"))
-		}},
-		{name: "vars.fd of the wrong size", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), []byte("short"))
-		}},
-		{name: "seeded.json unreadable", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), region("stale"))
-			require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte("{"), 0o600))
-		}},
-		{name: "no vars.fd", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), region("stale"))
-			require.NoError(t, os.Remove(filepath.Join(f.instanceDir(), "vars.fd")))
-		}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("current")
-			tt.local(f, t)
-			f.putObject(t, key, gzipped(t, region("restored")))
-			instance := f.instance(key, f.identity)
-
-			require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-			assert.Equal(t, region("restored"), f.localVars(t))
-			assert.Equal(t, key, f.seededKey(t))
-		})
-	}
-}
-
-// A store pflash cannot load fails the launch before any volume is mounted,
-// names both firmwares, and leaves the object, the pointer and the local
-// copy as they were.
-func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
-	tests := []struct {
-		name   string
-		kind   string
-		mutate func(*uefistore.FirmwareIdentity)
-		want   []string
-	}{
-		{name: "arch", kind: "arch", mutate: func(fw *uefistore.FirmwareIdentity) { fw.Arch = "arm64" },
-			want: []string{"arch=arm64", "arch=x86_64"}},
-		{name: "vars size", kind: "vars_size", mutate: func(fw *uefistore.FirmwareIdentity) { fw.VarsSize = 67108864 },
-			want: []string{"vars_size=67108864", "vars_size=4096"}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("mismatch")
-			body := gzipped(t, region("other firmware"))
-			f.putObject(t, key, body)
-			f.writeLocal(t, keyFor("older"), region("local"))
-			stored := f.identity
-			tt.mutate(&stored)
-			instance := f.instance(key, stored)
-			wantRef := *instance.UEFIVarStore
-			before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind)
-
-			err := f.m.Run(t.Context(), instance)
-
-			require.ErrorIs(t, err, ErrUEFIFirmwareMismatch)
-			require.NotErrorIs(t, err, ErrMountRetryable, "a mismatch does not clear with time")
-			assert.Empty(t, f.mounter.mounted, "nothing is mounted for a store that cannot boot")
-			require.NotNil(t, instance.Instance.StateReason)
-			assert.Equal(t, "Server.InternalError", aws.StringValue(instance.Instance.StateReason.Code))
-			for _, s := range append(tt.want, f.identity.CodeSHA256, f.codePath, key) {
-				assert.Contains(t, aws.StringValue(instance.Instance.StateReason.Message), s)
-			}
-			assert.Equal(t, wantRef, *instance.UEFIVarStore, "the pointer is unchanged")
-			assert.Equal(t, body, f.objectBody(t, key), "the object is unchanged")
-			assert.Equal(t, region("local"), f.localVars(t), "the local copy is unchanged")
-			assert.Equal(t, keyFor("older"), f.seededKey(t))
-			assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind))
-		})
-	}
-}
-
-// An ovmf package update changes the CODE hash on every node, so a store
-// written under another build boots and the difference is counted.
-func TestPrepareUEFIVarStore_DifferentCodeBuildBootsAndIsCounted(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("code")
-	f.putObject(t, key, gzipped(t, region("older ovmf")))
-	stored := f.identity
-	stored.CodeSHA256 = strings.Repeat("0", 64)
-	instance := f.instance(key, stored)
-	before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-	assert.Equal(t, region("older ovmf"), f.localVars(t))
-	assert.Nil(t, instance.Instance.StateReason)
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code"))
-}
-
-// failingGetStore fails every get, standing in for an object store that
-// does not answer.
-type failingGetStore struct {
-	*objectstore.MemoryObjectStore
-
-	err error
-}
-
-func (s *failingGetStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
-	return nil, s.err
-}
-
-func TestPrepareUEFIVarStore_FailedGetIsRetryable(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("connection refused")}
-	instance := f.instance(keyFor("down"), f.identity)
-
-	err := f.m.prepareUEFIVarStore(t.Context(), instance)
-
-	require.ErrorIs(t, err, ErrMountRetryable, "restore backs off on a get as it does on a mount")
-	assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
-}
-
-// An object that is missing or does not decode to this node's region is
-// refused for good, and nothing is written locally.
-func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
-	tests := []struct {
-		name string
-		body []byte
-	}{
-		{name: "missing"},
-		{name: "not gzip", body: region("raw, uncompressed")},
-		{name: "short region", body: gzipped(t, make([]byte, testUEFIVarsSize-1))},
-		{name: "long region", body: gzipped(t, make([]byte, testUEFIVarsSize+1))},
-		{name: "truncated gzip", body: gzipped(t, region("cut"))[:20]},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("bad")
-			if tt.body != nil {
-				f.putObject(t, key, tt.body)
-			}
-			instance := f.instance(key, f.identity)
-
-			err := f.m.prepareUEFIVarStore(t.Context(), instance)
-
-			require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
-			require.NotErrorIs(t, err, ErrMountRetryable)
-			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "vars.fd"))
-			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
-		})
-	}
-}
-
-func TestPrepareUEFIVarStore_PointerOutsideTheInstancePrefixIsRefused(t *testing.T) {
-	for _, key := range []string{"uefi/i-0aaaaaaaaaaaaaaaa/other", keyFor("a/b"), uefistore.KeyPrefix(testUEFIInstance)} {
-		t.Run(key, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			f.putObject(t, key, gzipped(t, region("x")))
-			instance := f.instance(key, f.identity)
-
-			require.ErrorIs(t, f.m.prepareUEFIVarStore(t.Context(), instance), ErrUEFIVarStoreUnusable)
-		})
-	}
-}
-
-// A BIOS instance, a microvm and a legacy instance with no pointer touch
-// neither the object store nor the data directory.
-func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("must not be called")}
-	bios := f.instance(keyFor("bios"), f.identity)
-	bios.BootMode = ""
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), bios))
-
-	legacy := f.instance("", f.identity)
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), legacy))
-	assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
-}
-
-// A UEFI instance with no pointer, or two stores, is refused before the
-// launch clears or changes anything on its record.
-func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
-	efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
-	tests := []struct {
-		name    string
-		pointer bool
-		efi     bool
-		want    error
-	}{
-		{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
-		{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := ""
-			if tt.pointer {
-				key = keyFor("p")
-			}
-			instance := f.instance(key, f.identity)
-			if tt.efi {
-				instance.EBSRequests.Requests = []EBSRequest{efiVolume}
-			}
-			stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
-			instance.Instance.StateReason = stale
-			before := instance.Record()
-
-			err := f.m.Run(t.Context(), instance)
-
-			require.ErrorIs(t, err, tt.want)
-			assert.Empty(t, f.mounter.mounted)
-			assert.Equal(t, before, instance.Record(), "the record is unchanged")
-		})
-	}
-}
-
-// Start from StateError transitions to pending before launching; a refused
-// UEFI instance must not take that transition either.
-func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testing.T) {
-	f := newUEFIFixture(t)
-	transitions := 0
-	f.m.deps.TransitionState = func(v *VM, s InstanceState) error { transitions++; v.Status = s; return nil }
-	instance := f.instance("", f.identity)
-	instance.Status = StateError
-
-	err := f.m.Start(t.Context(), instance.ID)
-
-	require.ErrorIs(t, err, ErrUEFIVarStoreMissing)
-	assert.Equal(t, 0, transitions)
-	assert.Equal(t, StateError, instance.Status)
-}
-
-// Until RunInstances seeds pointers, an instance whose variables are on an
-// EFI volume still launches through the volume, and a BIOS or microvm
-// guest needs no store at all.
-func TestCheckUEFIVarStore_Accepts(t *testing.T) {
-	tests := []struct {
-		name string
-		v    *VM
-	}{
-		{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
-		{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
-		{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
-		{name: "no boot mode", v: &VM{}},
-		{name: "microvm", v: &VM{BootMode: "uefi", DirectBoot: true}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.NoError(t, CheckUEFIVarStore(tt.v))
-		})
-	}
-}
-
-// A launch from a pointer opens this node's copy as pflash unit 1 with
-// format=raw, after the read-only CODE blob.
-func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
-	launches := withFakeQEMU(t)
-	key := keyFor("boot")
-	f.writeLocal(t, key, region("seeded"))
-	instance := f.instance(key, f.identity)
-	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
-
-	require.NoError(t, f.m.startQEMU(instance, false))
-
-	got := launches()
-	require.Len(t, got, 1)
-	vars := filepath.Join(f.instanceDir(), "vars.fd")
-	assert.Equal(t, []string{
-		"file=" + f.codePath + ",format=raw,if=pflash,unit=0,readonly=on",
-		"file=" + vars + ",format=raw,if=pflash,unit=1",
-	}, allArgValues(got[0], "-drive"))
-}
-
-// A reboot reuses the copy the launch seeded; without one there is nothing
-// safe to boot, so QEMU is not started.
-func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
-	launches := withFakeQEMU(t)
-	instance := f.instance(keyFor("boot"), f.identity)
-	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
-
-	err := f.m.startQEMU(instance, false)
-
-	require.ErrorIs(t, err, os.ErrNotExist)
-	assert.Empty(t, launches())
-}
-
-func TestUEFIInstanceDir_RefusesIDsThatAreNotOnePathSegment(t *testing.T) {
-	for _, id := range []string{"", "..", "i-../x", "i-a/b", `i-a\b`, "vol-1"} {
-		_, err := uefiInstanceDir("/data", id)
-		assert.Error(t, err, id)
-	}
-	dir, err := uefiInstanceDir("/data", testUEFIInstance)
-	require.NoError(t, err)
-	assert.Equal(t, "/data/uefi/"+testUEFIInstance, dir)
-}
-
-// The CODE hash is what a store records; a package update that replaces the
-// blob must be hashed again rather than served from the cache.
-func TestNodeFirmwareFor_HashesTheCodeBlobAgainAfterItChanges(t *testing.T) {
-	f := newUEFIFixture(t)
-	first, err := NodeFirmwareFor("x86_64")
-	require.NoError(t, err)
-	assert.Equal(t, f.identity, first.Identity)
-
-	updated := []byte("ovmf code build 2, a longer blob")
-	require.NoError(t, os.WriteFile(f.codePath, updated, 0o600))
-	second, err := NodeFirmwareFor("x86_64")
-	require.NoError(t, err)
-
-	sum := sha256.Sum256(updated)
-	assert.Equal(t, hex.EncodeToString(sum[:]), second.Identity.CodeSHA256)
-	assert.Equal(t, f.codePath, second.Code)
-}

spinifex/runtime/compute/vm/vm.go

@@ -137,6 +137,10 @@ type VM struct {
 	// claiming the flag serialises reboots of one instance.
 	rebooting atomic.Bool `json:"-"`
 
+	// guestPoweroff is set once the heartbeat has seen the guest power itself off
+	// and asked QEMU to quit, so the exit that follows is a stop, not a crash.
+	guestPoweroff atomic.Bool `json:"-"`
+
 	// relaunchMu is held while a reboot swaps this guest's QEMU process, so a
 	// stop or terminate never unmounts volumes under a process being started.
 	relaunchMu sync.Mutex `json:"-"`
@@ -387,7 +391,7 @@ type Device struct {
 }
 
 // Drive is one QEMU -drive entry, used for the pflash firmware drives (the
-// CODE blob and the variable store). Each set field becomes a key=value option.
+// CODE blob and the EFI volume). Each set field becomes a key=value option.
 type Drive struct {
 	File   string `json:"file"`
 	Format string `json:"format"`
@@ -396,7 +400,7 @@ type Drive struct {
 	ID     string `json:"id"`
 	Cache  string `json:"cache,omitempty"`
 	// Unit selects the pflash slot when If=="pflash": 0 is the CODE blob,
-	// 1 is the per-VM variable store. Ignored for non-pflash drives.
+	// 1 is the per-VM VARS volume. Ignored for non-pflash drives.
 	Unit int `json:"unit,omitempty"`
 	// ReadOnly emits readonly=on; used for the pflash CODE blob.
 	ReadOnly bool `json:"readonly,omitempty"`
@@ -432,13 +436,10 @@ type Config struct {
 	NoGraphic          bool   `json:"no_graphic"`
 	MachineType        string `json:"machine_type"`
 	ConsoleLogPath     string `json:"console_log_path,omitempty"`
-	// ConsoleLogAppend keeps the existing console log rather than truncating
+	SerialSocket       string `json:"serial_socket,omitempty"`
-	// it, for a reboot that replaces the process.
+	CPUType            string `json:"cpu_type"`
-	ConsoleLogAppend bool   `json:"-"`
+	CPUCount           int    `json:"cpu_count"`
-	SerialSocket     string `json:"serial_socket,omitempty"`
+	Memory             int    `json:"memory"`
-	CPUType          string `json:"cpu_type"`
-	CPUCount         int    `json:"cpu_count"`
-	Memory           int    `json:"memory"`
 
 	Drives    []Drive    `json:"drives"`
 	IOThreads []IOThread `json:"io_threads,omitempty"`
@@ -552,11 +553,11 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		// -serial chardev:console0 wires it to the existing device (not a new one).
 		// Do NOT use -device isa-serial here — that adds a second device (ttyS1)
 		// while the kernel talks to ttyS0.
-		args = append(args, "-chardev", fmt.Sprintf("file,id=console0,path=%s%s", cfg.ConsoleLogPath, appendOpt(",append", cfg.ConsoleLogAppend)))
+		args = append(args, "-chardev", fmt.Sprintf("file,id=console0,path=%s,append=on", cfg.ConsoleLogPath))
 		args = append(args, "-serial", "chardev:console0")
 	} else if cfg.SerialSocket != "" && cfg.ConsoleLogPath != "" {
-		chardevOpts := fmt.Sprintf("socket,id=console0,path=%s,server=on,wait=off,logfile=%s%s",
+		chardevOpts := fmt.Sprintf("socket,id=console0,path=%s,server=on,wait=off,logfile=%s,logappend=on",
-			cfg.SerialSocket, cfg.ConsoleLogPath, appendOpt(",logappend", cfg.ConsoleLogAppend))
+			cfg.SerialSocket, cfg.ConsoleLogPath)
 		args = append(args, "-chardev", chardevOpts)
 		args = append(args, "-serial", "chardev:console0")
 	}
@@ -588,8 +589,8 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 			return nil, fwErr
 		}
 		// pflash CODE must come before VARS so QEMU loads firmware into the
-		// readonly slot first; startQEMU emits the VARS drive (unit 1) from
+		// readonly slot first; buildDrives emits the VARS drive (unit 1) from
-		// the node's copy of the variable store or the EFI volume.
+		// the per-VM EFI viperblock volume.
 		drives = append([]Drive{{If: "pflash", Format: "raw", Unit: 0, ReadOnly: true, File: codePath}}, drives...)
 	}
 
@@ -693,12 +694,3 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 
 	return cmd, nil
 }
-
-// appendOpt renders a chardev append option, or nothing when off, so a
-// fresh launch keeps the command line it always had.
-func appendOpt(key string, on bool) string {
-	if !on {
-		return ""
-	}
-	return key + "=on"
-}

spinifex/runtime/compute/vm/vm_test.go

@@ -463,7 +463,7 @@ func TestExecute_MicrovmFileChardev(t *testing.T) {
 
 	args := cmd.Args[1:]
 	chardev := argValue(args, "-chardev")
-	assert.Equal(t, "file,id=console0,path=/var/log/console.log", chardev)
+	assert.Equal(t, "file,id=console0,path=/var/log/console.log,append=on", chardev)
 	// -serial chardev:console0 attaches to the isa-serial=on device (ttyS0).
 	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
 	// No explicit isa-serial device addition (would create ttyS1, not ttyS0).
@@ -699,6 +699,6 @@ func TestExecute_MicrovmWithPCIEFileChardev(t *testing.T) {
 	require.NoError(t, err)
 
 	args := cmd.Args[1:]
-	assert.Equal(t, "file,id=console0,path=/var/log/console.log", argValue(args, "-chardev"))
+	assert.Equal(t, "file,id=console0,path=/var/log/console.log,append=on", argValue(args, "-chardev"))
 	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
 }

spinifex/runtime/compute/vm/volume_mounter.go

@@ -5,8 +5,8 @@ import (
 )
 
 // VolumeMounter mounts and unmounts the EBS volumes attached to a VM. The
-// real implementation routes ebs.mount / ebs.unmount NATS requests; the
+// real implementation publishes and unpublishes them through the EBS provider
-// abstraction keeps NATS out of the manager.
+// over NATS; the abstraction keeps NATS out of the manager.
 //
 // Every method takes the caller's context so the ebs.* span joins the trace
 // that asked for the work rather than rooting one of its own. Shutdown and
@@ -14,9 +14,9 @@ import (
 // honest answer rather than a missing one.
 type VolumeMounter interface {
 	// Mount mounts every attached volume in v.EBSRequests.Requests, recording
-	// the resolved NBDURI back onto each request entry.
+	// the resolved export back onto each request entry.
 	Mount(ctx context.Context, v *VM) error
-	// Unmount sends ebs.unmount for each attached volume, which drives the
+	// Unmount unmounts each attached volume, which drives the
 	// synchronous block-map seal. Every volume is attempted; the seal
 	// failures are aggregated and returned, and mean the data is not durable.
 	Unmount(ctx context.Context, v *VM) error
@@ -31,12 +31,12 @@ type VolumeMounter interface {
 	// anywhere.
 	Abandon(ctx context.Context, v *VM, reason string) error
 
-	// MountOne sends ebs.mount for a single request and writes the resolved
+	// MountOne mounts a single request and writes the resolved export back
-	// NBDURI back into req.NBDURI on success. Used by hot-attach. accountID
+	// into req on success. Used by hot-attach. accountID
 	// names the owner, which a lone request carries no instance to supply.
 	MountOne(ctx context.Context, accountID string, req *EBSRequest) error
-	// UnmountOne sends ebs.unmount for a single request and returns any error.
+	// UnmountOne unmounts a single request and returns any error. The
-	// ebs.unmount drives the synchronous block-map seal to predastore, so hot
+	// unmount drives the synchronous block-map seal to the backend, so hot
 	// detach gates the volume's available transition on the returned error.
 	UnmountOne(ctx context.Context, accountID string, req EBSRequest) error
 }

spinifex/runtime/compute/vm/volumes.go

@@ -25,7 +25,7 @@ const blockdevDelMaxAttempts = 20
 // object-del steps), so a wedged-but-responsive QEMU cannot stack per-step
 // timeouts into an effectively unbounded hang. It comfortably covers those
 // steps' own budgets (qmpCommandTimeout + the bounded blockdev-del retry) plus
-// margin; the ebs.unmount seal keeps its separate unmountSealTimeout.
+// margin; the volume seal is not bound by it and keeps its own budget.
 const detachAggregateTimeout = 3 * time.Minute
 
 // ErrHotAttachUnsupported is returned by AttachVolume for a direct-boot microvm,
@@ -119,9 +119,9 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 		return "", fmt.Errorf("VolumeMounter not wired")
 	}
 	if err := m.deps.VolumeMounter.MountOne(ctx, instance.AccountID, &ebsRequest); err != nil {
-		slog.ErrorContext(ctx, "AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err)
+		slog.ErrorContext(ctx, "AttachVolume: publish failed", "volumeId", volumeID, "err", err)
-		// Empty-URI response leaves backend NBD state ambiguous; unmount
+		// A publish with no usable reply may have started an export, so it is
-		// defensively to avoid orphaning a half-started mount.
+		// unpublished defensively rather than orphaned.
 		if errors.Is(err, ErrMountAmbiguous) {
 			_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
 		}
@@ -300,7 +300,7 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 }
 
 // DetachVolume hot-unplugs a volume via the QMP pipeline (device_del →
-// blockdev-del → object-del → ebs.unmount). Returns the recorded AWS-API device
+// blockdev-del → object-del → unpublish). Returns the recorded AWS-API device
 // name. force=true tolerates device_del failure; blockdev-del is always required.
 func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string, force bool) (string, error) {
 	instance, ok := m.Get(id)
@@ -344,8 +344,8 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// Bound the ctx-driven portion of the hot-unplug chain so a wedged-but-
 	// responsive QEMU cannot stack per-step QMP timeouts into an unbounded hang
-	// that pins attachMu forever. The ebs.unmount seal below keeps its own
+	// that pins attachMu forever. The volume seal below is not bound by it and
-	// unmountSealTimeout. This deadline cannot preempt a plain-mutex QMP call,
+	// keeps its own budget. This deadline cannot preempt a plain-mutex QMP call,
 	// which is why a fully dead QEMU is short-circuited outright below.
 	ctx, cancel := context.WithTimeout(ctx, detachAggregateTimeout)
 	defer cancel()
@@ -359,7 +359,7 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// A confirmed-dead QEMU took every block node and guest device down with it,
 	// so the QMP unplug steps are moot — and worse, issuing them only risks
-	// hanging on the wedged process. Skip straight to the ebs.unmount seal and
+	// hanging on the wedged process. Skip straight to the unpublish seal and
 	// the attachment clear. This is the detach-wedges-when-QEMU-dies fix: the
 	// context deadline above cannot interrupt a plain-mutex QMP path, so a
 	// provably-gone process must bypass it rather than wait on it. A missing PID
@@ -422,15 +422,15 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 			"volumeId", volumeID, "instanceId", instance.ID)
 	}
 
-	// ebs.unmount drives the synchronous block-map seal to predastore. On
+	// Unpublish drives the synchronous block-map seal to the backend. On
 	// failure the volume's local WAL is retained, so keep the volume attached
 	// and return the error; an AWS-CLI retry re-drives the seal and same-node
 	// reattach still works meanwhile.
 	if m.deps.VolumeMounter != nil {
 		if err := m.deps.VolumeMounter.UnmountOne(ctx, instance.AccountID, ebsReq); err != nil {
-			slog.ErrorContext(ctx, "DetachVolume: ebs.unmount seal failed, leaving volume attached",
+			slog.ErrorContext(ctx, "DetachVolume: unpublish seal failed, leaving volume attached",
 				"volumeId", volumeID, "err", err)
-			return "", fmt.Errorf("ebs.unmount seal: %w", err)
+			return "", fmt.Errorf("unpublish seal: %w", err)
 		}
 	}
 
@@ -554,7 +554,7 @@ func (m *Manager) tryBlockdevDel(ctx context.Context, instance *VM, nodeName str
 			return nil
 		}
 		// A prior detach already removed the block node (e.g. an AWS-CLI retry
-		// after the ebs.unmount seal failed): treat as success so the retry
+		// after the unpublish seal failed): treat as success so the retry
 		// resumes through object-del to the seal instead of wedging on a
 		// now-absent node.
 		if isQMPNodeNotFound(err) {

spinifex/runtime/compute/vm/volumes_test.go

@@ -543,7 +543,7 @@ func TestDetachVolume_DeviceGuards(t *testing.T) {
 }
 
 // TestDetachVolume_SealFailureGatesAvailable covers the durability gate in the
-// detach-seal fix: when ebs.unmount (the synchronous block-map seal) fails,
+// detach-seal fix: when unpublish (the synchronous block-map seal) fails,
 // DetachVolume must propagate the error, must NOT flip the volume to "available",
 // and must leave it in EBSRequests so the volume stays attached/retryable and its
 // local WAL is retained. A regression back to fire-and-forget unmount would let a
@@ -661,7 +661,7 @@ func TestAttachVolume_MountAmbiguous_TriggersUnmountOne(t *testing.T) {
 // MountOne failures (NATS timeout, backend explicit Error, marshal errors)
 // do NOT trigger UnmountOne — only the empty-URI ambiguous case rolls back.
 func TestAttachVolume_GenericMountError_DoesNotUnmount(t *testing.T) {
-	mounter := &fakeVolumeMounter{mountOneErr: errors.New("ebs.mount NATS request: timeout")}
+	mounter := &fakeVolumeMounter{mountOneErr: errors.New("publish request: timeout")}
 	m, _ := attachVolumeRunningInstance(t, nil, mounter)
 
 	_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
@@ -1387,7 +1387,7 @@ func TestDetachVolume_DeviceDeletedTimeout_FallsBackToRetry(t *testing.T) {
 // detach-wedges-when-QEMU-dies fix: when QEMU is provably gone (PID file
 // present, process dead), DetachVolume issues no QMP command — those steps
 // would only hang on the wedged process — and proceeds straight to the
-// ebs.unmount seal and the attachment clear.
+// unpublish seal and the attachment clear.
 func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
@@ -1418,7 +1418,7 @@ func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "/dev/sdf", device)
 	assert.False(t, qmpCalled.Load(), "a confirmed-dead QEMU must receive no QMP command")
-	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the ebs.unmount seal must still run")
+	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the unpublish seal must still run")
 	calls := updater.snapshot()
 	require.Len(t, calls, 1, "the stale attachment must be cleared exactly once")
 	assert.Equal(t, "available", calls[0].State, "the volume must be returned to available")

spinifex/runtime/daemon/daemon.go

@@ -41,7 +41,6 @@ import (
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/preflight"
 	acmdomain "github.com/mulgadc/spinifex/spinifex/domains/acm"
@@ -152,7 +151,6 @@ type Daemon struct {
 	natsConn          *nats.Conn
 	resourceMgr       *ResourceManager
 	instanceService   *ec2instance.InstanceServiceImpl
-	instanceObjects   objectstore.ObjectStore
 	dnsWriter         *dns.Writer
 	dnsReconciler     *dns.Reconciler
 	dnsBaseDomain     string
@@ -734,6 +732,11 @@ func (d *Daemon) SetConfigPath(path string) {
 
 // NewDaemon creates a new daemon instance.
 func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) {
+	// The fence notice is node-addressed: a daemon that cannot subscribe to it
+	// would leave a guest running on a volume whose export was revoked.
+	if err := ebsprovider.ValidateNodeID(cfg.Node); err != nil {
+		return nil, fmt.Errorf("cluster config node %q: %w", cfg.Node, err)
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 
 	// If WalDir is not set, use BaseDir
@@ -937,6 +940,8 @@ func (d *Daemon) subscribeAll() error {
 		// Cluster-wide on purpose: the QMP detach is instance-scoped, so a
 		// volume held by an unreachable host has no other way out.
 		{"ec2.ForceDetachVolume", handleNATSRequest(d.node, d.volumeService.ForceDetachVolume), "spinifex-workers"},
+		{"ec2.DetachVolumeFromStopped", handleNATSRequest(d.node, d.instanceService.DetachVolumeFromStopped), "spinifex-workers"},
+		{"ec2.AttachVolumeToStopped", handleNATSRequest(d.node, d.instanceService.AttachVolumeToStopped), "spinifex-workers"},
 		{"ec2.DescribeVolumeStatus", handleNATSRequest(d.node, d.volumeService.DescribeVolumeStatus), "spinifex-workers"},
 		{"ec2.DescribeVolumesModifications", handleNATSRequest(d.node, d.volumeService.DescribeVolumesModifications), "spinifex-workers"},
 		{"ec2.CreateSnapshot", handleNATSRequest(d.node, d.snapshotService.CreateSnapshot), "spinifex-workers"},
@@ -1354,10 +1359,8 @@ func (d *Daemon) subscribeAll() error {
 		// here, so the reconcile request flows daemon-ward.
 		natsSub{dhcp.TopicLeaseChanged, d.handleDHCPLeaseChanged, "spinifex-workers"},
 		natsSub{dhcp.TopicOwnerCheck, d.handleDHCPOwnerCheck, "spinifex-workers"},
-		// Node-addressed with no queue group: the fenced guest is on this host,
-		// so a worker on another node would find nothing to stop.
-		natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
 	)
+	subs = append(subs, d.fenceNoticeSubs()...)
 
 	// Per-node, no queue group: a second daemon claiming this node ID also
 	// answers, so the teardown probe sees a duplicate and refuses to seal. An
@@ -2018,7 +2021,6 @@ func (d *Daemon) startCluster() error {
 
 	// Create services before loading/launching instances, since LaunchInstance depends on them
 	store := objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(d.config.Predastore.Host), d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey)
-	d.instanceObjects = store
 	d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, d.resourceMgr.instanceTypes, d.natsConn, store, d.vmMgr, d.resourceMgr, d.jsManager)
 	d.dnsWriter = dns.NewWriter(d.config, d.clusterConfig, d.natsConn)
 	d.dnsReconciler = dns.NewReconciler(d.config, d.clusterConfig, d.natsConn, d.dnsWriter, d.dnsDesiredSet, d.dnsWatchSources()...)
@@ -2573,6 +2575,7 @@ func (d *Daemon) startCluster() error {
 			d.vmMgr.NewTerminatedTeardownReaper(),
 			d.vmMgr.NewOrphanQEMUReaper(),
 			d.vmMgr.NewStuckTerminateReaper(),
+			d.vmMgr.NewLocalConsoleLogReaper(),
 		}
 		if eniRec := d.newENIReconciler(); eniRec != nil {
 			reapers = append(reapers, eniRec)
@@ -2624,9 +2627,9 @@ const ebsProviderRequestTimeout = 30 * time.Second
 // a missing responder must be reported promptly, and the check is advisory.
 const ebsProviderProbeTimeout = 2 * time.Second
 
-// configureEBSProvider wires a single NATSProvider into every EBS-adjacent
+// configureEBSProvider wires a NATSProvider into the EBS-adjacent services and
-// service. The control plane never constructs a provider implementation
+// the volume mounter. The control plane never constructs a provider
-// itself, so this always runs regardless of which daemon answers on the wire.
+// implementation, so this runs whichever daemon answers on the wire.
 func (d *Daemon) configureEBSProvider() error {
 	provider := ebsprovider.NewNATSProvider(d.natsConn, ebsProviderRequestTimeout)
 	d.ebsProvider = provider

spinifex/runtime/daemon/daemon_handlers.go

@@ -191,9 +191,6 @@ func handleNATSRequestWithPrincipal[I any, O any](nodeID string, serviceFn func(
 func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 	start := time.Now()
 	action, outcome := d.dispatchEC2Command(msg)
-	if outcome == outcomeDeferred {
-		return
-	}
 	otelsetup.RecordRequest(context.Background(), ec2CmdAction(action), outcome, time.Since(start))
 }
 
@@ -233,21 +230,6 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) {
 		return name, d.handleAttachVolume(ctx, msg, command, instance)
 	case command.Attributes.DetachVolume:
 		return name, d.handleDetachVolume(ctx, msg, command, instance)
-	case command.Attributes.DrainVolume:
-		// A drain flushes the volume's whole dirty set to S3, so it is bounded by
-		// the dirty set rather than by a unit of work. nats.go delivers a
-		// subscription's messages serially, so running it inline would stall
-		// every other command for this instance — stop, terminate, hot-plug —
-		// for the duration. It touches no VM state, so it is safe off-thread.
-		//
-		// It therefore records its own point: timing the dispatch would call
-		// every drain an instant success.
-		started := time.Now()
-		go func() {
-			outcome := d.handleDrainVolume(ctx, msg, command, instance)
-			otelsetup.RecordRequest(context.Background(), ec2CmdAction(name), outcome, time.Since(started))
-		}()
-		return name, outcomeDeferred
 	case command.Attributes.AttachENI:
 		return name, d.handleAttachNetworkInterface(ctx, msg, command, instance)
 	case command.Attributes.DetachENI:
@@ -314,8 +296,6 @@ func ec2CommandName(command ec2v1.EC2InstanceCommand) string {
 		return "AttachVolume"
 	case command.Attributes.DetachVolume:
 		return "DetachVolume"
-	case command.Attributes.DrainVolume:
-		return "DrainVolume"
 	case command.Attributes.AttachENI:
 		return "AttachNetworkInterface"
 	case command.Attributes.DetachENI:

spinifex/runtime/daemon/daemon_handlers_console.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 )
 
@@ -49,6 +50,11 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) {
 
 	if logPath != "" {
 		info, err := os.Stat(logPath)
+		if err != nil && os.IsNotExist(err) {
+			// A terminated instance's file is renamed, not removed, until it ages out.
+			logPath = vm.RetiredConsolePath(logPath)
+			info, err = os.Stat(logPath)
+		}
 		if err == nil {
 			modTime = info.ModTime()
 

spinifex/runtime/daemon/daemon_handlers_fence.go

@@ -3,56 +3,77 @@ package daemon
 import (
 	"context"
 	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"fmt"
 	"log/slog"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/nats-io/nats.go"
 )
 
-// handleVolumeFenced stops the local guest whose volume viperblockd has just
+// fenceNoticeSubs is this node's fence-notice subscription, whatever the EBS
-// torn the export out from under.
+// provider. Node-addressed with no queue group: the fenced guest is on this
+// host, so a worker on another node would find nothing to stop.
+func (d *Daemon) fenceNoticeSubs() []natsSub {
+	subject, err := ebsprovider.FenceNoticeSubject(d.node)
+	if err != nil {
+		slog.Error("Not serving EBS fence notices: node ID is not a single NATS subject token", "node", d.node, "err", err)
+		return nil
+	}
+	return []natsSub{{subject, d.handleFenceNotice, ""}}
+}
+
+// handleFenceNotice stops the local guest whose volume the EBS provider has
+// just revoked the export of.
 //
 // By the time this arrives the disk is already gone, so the guest is doing no
 // useful work and its writes cannot reach the backend. What is left is a QEMU
 // process holding memory and taps, and an instance record claiming this node
 // runs something that runs somewhere else now.
-func (d *Daemon) handleVolumeFenced(msg *nats.Msg) string {
+func (d *Daemon) handleFenceNotice(msg *nats.Msg) string {
-	ctx, span := natsmsg.StartConsumerSpan(msg)
+	ctx, span := ebsprovider.StartServerSpan(context.Background(), msg)
 	defer span.End()
 
-	var event viperblocklegacyv1.VolumeFencedEvent
+	var notice ebsprovider.FenceNotice
-	if err := json.Unmarshal(msg.Data, &event); err != nil {
+	decodeErr := json.Unmarshal(msg.Data, &notice)
-		slog.ErrorContext(ctx, "volume fenced: bad event", "err", err)
+	ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion)}
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "bad event: " + err.Error()})
+	refuse := func(code ebsprovider.ErrorCode, message string) string {
+		slog.ErrorContext(ctx, "volume fenced: refused the notice", "volumeId", notice.VolumeID, "code", code, "err", message)
+		ack.Error = &ebsprovider.ProviderError{Code: code, Message: message}
+		ebsprovider.RecordProviderError(span, ack.Error)
+		ackFenceNotice(ctx, msg, ack)
 		return outcomeError
 	}
-	if event.Volume == "" {
+	if decodeErr != nil {
-		slog.ErrorContext(ctx, "volume fenced: event named no volume")
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "bad notice: "+decodeErr.Error())
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "event named no volume"})
+	}
-		return outcomeError
+	if !ebsprovider.SupportsSchemaVersion(notice.SchemaVersion) {
+		return refuse(ebsprovider.ErrorCodeUnsupportedVersion,
+			fmt.Sprintf("unsupported schema version %d; ", notice.SchemaVersion)+ebsprovider.SupportedVersionsText())
+	}
+	if notice.VolumeID == "" {
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "notice named no volume")
 	}
 
 	// FenceVolume moves the guest out of running before it returns, so the ack
 	// below is sent only once nothing here will resume it.
-	instanceID := d.vmMgr.FenceVolume(ctx, event.Volume, event.Reason)
+	ack.InstanceID = d.vmMgr.FenceVolume(ctx, notice.VolumeID, notice.Reason)
-	ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Instance: instanceID})
+	ackFenceNotice(ctx, msg, ack)
-	if instanceID == "" {
+	if ack.InstanceID == "" {
 		// The usual case: the volume outlived the guest that was using it, so
 		// there is nothing local left to stop.
 		slog.InfoContext(ctx, "volume fenced with no local guest using it",
-			"volumeId", event.Volume, "winner", event.Winner)
+			"volumeId", notice.VolumeID, "winner", notice.Winner)
 		return outcomeSuccess
 	}
 
 	slog.ErrorContext(ctx, "volume fenced, stopped the guest that was using it",
-		"volumeId", event.Volume, "instanceId", instanceID, "winner", event.Winner)
+		"volumeId", notice.VolumeID, "instanceId", ack.InstanceID, "winner", notice.Winner)
 	return outcomeSuccess
 }
 
-// ackVolumeFenced answers viperblockd, which repeats the notice until it hears
+// ackFenceNotice answers the provider, which repeats the notice until it hears
 // back. A notice with no reply subject was published once and needs no answer.
-func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.VolumeFencedAck) {
+func ackFenceNotice(ctx context.Context, msg *nats.Msg, ack ebsprovider.FenceNoticeAck) {
 	if msg.Reply == "" {
 		return
 	}
@@ -62,6 +83,6 @@ func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.
 		return
 	}
 	if err := msg.Respond(payload); err != nil {
-		slog.WarnContext(ctx, "volume fenced: could not acknowledge, viperblockd will repeat the notice", "err", err)
+		slog.WarnContext(ctx, "volume fenced: could not acknowledge, the provider will repeat the notice", "err", err)
 	}
 }

spinifex/runtime/daemon/daemon_handlers_fence_invariants_test.go

@@ -2,92 +2,313 @@ package daemon
 
 import (
 	"encoding/json"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/codes"
 )
 
-// fenceDaemon is a daemon running one guest on volumeID, answering fence
+// fenceDaemon is a daemon running one guest on volumeID, subscribed to fence
-// notices on a real NATS server as it does in production.
+// notices on a real NATS server through the subscription it makes in production.
 func fenceDaemon(t *testing.T, volumeID string) (*Daemon, *vm.VM, *nats.Conn) {
+	t.Helper()
+	return fenceDaemonWithManager(t, volumeID, vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"}))
+}
+
+func fenceDaemonWithManager(t *testing.T, volumeID string, mgr *vm.Manager) (*Daemon, *vm.VM, *nats.Conn) {
 	t.Helper()
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
 	_, nc := testutil.StartTestNATS(t)
-	d := &Daemon{node: "node-a", vmMgr: vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})}
+	d := &Daemon{
-	guest := &vm.VM{
+		node:              "node-a",
+		vmMgr:             mgr,
+		natsConn:          nc,
+		natsSubscriptions: map[string]*nats.Subscription{},
+	}
+	guest := fencedGuest(volumeID)
+	d.vmMgr.Insert(guest)
+
+	require.NoError(t, d.registerNatsSubs(d.fenceNoticeSubs()))
+	require.NoError(t, nc.Flush())
+	return d, guest, nc
+}
+
+func fencedGuest(volumeID string) *vm.VM {
+	return &vm.VM{
 		ID:          "i-fencedguest",
 		Status:      vm.StateRunning,
 		Instance:    &ec2.Instance{},
 		EBSRequests: vm.EBSRequests{Requests: []vm.EBSRequest{{Name: volumeID}}},
 	}
-	d.vmMgr.Insert(guest)
+}
 
-	_, err := nc.Subscribe(viperblocklegacyv1.VolumeFencedSubject(d.node), func(msg *nats.Msg) { d.handleVolumeFenced(msg) })
+// requestFenceNotice sends raw to node-a's fence-notice subject and decodes the ack.
+func requestFenceNotice(t *testing.T, nc *nats.Conn, raw []byte) ebsprovider.FenceNoticeAck {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
 	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
+	reply, err := nc.Request(subject, raw, 2*time.Second)
-	return d, guest, nc
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not answer a fence notice, so the provider can never tell it was heard. "+
+			"Fix: subscribe fenceNoticeSubs and respond in handleFenceNotice (daemon_handlers_fence.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	return ack
+}
+
+func fenceNoticePayload(t *testing.T, notice ebsprovider.FenceNotice) []byte {
+	t.Helper()
+	payload, err := json.Marshal(notice)
+	require.NoError(t, err)
+	return payload
+}
+
+// memoryProviderPublishing is a non-viperblock provider holding volumeID
+// published to node-a, announcing fences on nc.
+func memoryProviderPublishing(t *testing.T, nc *nats.Conn, volumeID string) *ebsprovider.MemoryProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	provider.NotifyFencesOn(nc)
+	_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(t.Context(), ebsprovider.PublishVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-a",
+	})
+	require.NoError(t, err)
+	return provider
 }
 
 // TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped enforces ADR-0007:S4
-// on the daemon's half of the fence. viperblockd repeats the notice until it is
+// on the daemon's half of the fence. The provider repeats the notice until it is
 // answered, so the answer is what ends delivery, and it must not come before the
 // guest is out of running, or a lost notice is acknowledged for a guest left up.
 func TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedack")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{
+	ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
-		Volume: "vol-fencedack", Node: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
+		Versioned: ebsprovider.NewVersioned(),
-	})
+		VolumeID:  "vol-fencedack", NodeID: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
-	require.NoError(t, err)
+	}))
-
+	assert.Equal(t, guest.ID, ack.InstanceID, "the ack names the guest the fence stopped")
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
+	assert.Nil(t, ack.Error)
-	require.NoErrorf(t, err,
-		"ADR-0007:S4: the daemon did not answer a fence notice, so viperblockd can never tell it was heard. "+
-			"Fix: respond in handleVolumeFenced (daemon_handlers_fence.go)")
-
-	var ack viperblocklegacyv1.VolumeFencedAck
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	assert.Equal(t, guest.ID, ack.Instance, "the ack names the guest the fence stopped")
-	assert.Empty(t, ack.Error)
 	assert.NotEqual(t, vm.StateRunning, d.vmMgr.Status(guest),
 		"ADR-0007:S4: the fence was acknowledged while the guest was still running")
 }
 
+// TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress enforces
+// ADR-0007:S4 on the order of the stop and the ack: with the stop held open,
+// no answer may reach the provider until it completes.
+func TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress(t *testing.T) {
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
+
+	var mgr *vm.Manager
+	mgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a", TransitionState: func(v *vm.VM, target vm.InstanceState) error {
+		close(entered)
+		<-release
+		mgr.Inspect(v, func(v *vm.VM) { v.Status = target })
+		return nil
+	}})
+	_, guest, nc := fenceDaemonWithManager(t, "vol-fenceheld", mgr)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	type answer struct {
+		msg *nats.Msg
+		err error
+	}
+	answers := make(chan answer, 1)
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fenceheld", NodeID: "node-a"})
+	go func() {
+		msg, err := nc.Request(subject, payload, 10*time.Second)
+		answers <- answer{msg, err}
+	}()
+
+	select {
+	case <-entered:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence never reached the guest's stop, so this test proved nothing about the ack order")
+	}
+	select {
+	case got := <-answers:
+		t.Fatalf("ADR-0007:S4: the fence was acknowledged (%v) before the guest's stop finished. "+
+			"Fix: ack only after FenceVolume returns in handleFenceNotice (daemon_handlers_fence.go)", got.err)
+	case <-time.After(300 * time.Millisecond):
+	}
+
+	releaseOnce.Do(func() { close(release) })
+	var got answer
+	select {
+	case got = <-answers:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence was never acknowledged once the stop finished")
+	}
+	require.NoError(t, got.err)
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable keeps the
+// subscription in subscribeAll: a handler nothing subscribes is a fence no
+// daemon hears.
+func TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.node = "node-a"
+	d.vmMgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})
+	guest := fencedGuest("vol-fencedsubscribed")
+	d.vmMgr.Insert(guest)
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	reply, err := caller.Request(subject, fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedsubscribed", NodeID: "node-a",
+	}), 2*time.Second)
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the running daemon does not serve fence notices. Fix: append fenceNoticeSubs in subscribeAll (daemon.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	assert.Nil(t, ack.Error)
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress keeps a daemon from
+// starting where no fence notice can reach it, which would leave a guest
+// running on a revoked export with nothing but a log line to say so.
+func TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress(t *testing.T) {
+	for _, node := range []string{"", "node.a", "node-*", "node>"} {
+		t.Run(node, func(t *testing.T) {
+			d, err := NewDaemon(&config.ClusterConfig{Node: node, Nodes: map[string]config.Config{node: {}}})
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "cluster config node")
+			assert.Nil(t, d)
+		})
+	}
+}
+
+// TestFenceNotice_SpanAttributesTheAccountAndTheRefusal reads the daemon's
+// span for a refused notice: a server span for the fenced verb, carrying the
+// sender's account and the refusal's code rather than a successful round trip.
+func TestFenceNotice_SpanAttributesTheAccountAndTheRefusal(t *testing.T) {
+	recorder := testutil.RecordSpans(t)
+	_, _, nc := fenceDaemon(t, "vol-unused")
+
+	ctx := ebsprovider.WithAccountID(t.Context(), "123456789012")
+	_, err := ebsprovider.NotifyFenced(ctx, nc, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}, VolumeID: "vol-unused", NodeID: "node-a",
+	}, 2*time.Second)
+	require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+
+	_, server := testutil.SpansByKind(t, recorder)
+	assert.Equal(t, "ebs.volume.fenced", server.Name())
+	assert.Equal(t, codes.Error, server.Status().Code)
+	assert.Equal(t, string(ebsprovider.ErrorCodeUnsupportedVersion), testutil.SpanAttribute(t, server, "ebs.provider.error_code").AsString())
+	assert.Equal(t, "123456789012", testutil.SpanAttribute(t, server, "aws.account_id").AsString())
+}
+
+// TestFenceNotice_AckEchoesTheNoticeVersion keeps the answer readable by the
+// sender: an ack in a version other than the notice's is one it may refuse.
+func TestFenceNotice_AckEchoesTheNoticeVersion(t *testing.T) {
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		_, _, nc := fenceDaemon(t, "vol-versioned")
+		ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
+			Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: "vol-versioned", NodeID: "node-a",
+		}))
+		assert.Nil(t, ack.Error)
+		assert.Equalf(t, version, ack.SchemaVersion, "a notice at version %d was answered at another", version)
+	}
+}
+
+// TestS4_FenceNoticeFromAnyProviderStopsTheGuest holds the daemon to the
+// contract rather than to viperblockd: a provider that is not viperblock
+// revoking the export a local guest runs on gets that guest stopped.
+func TestS4_FenceNoticeFromAnyProviderStopsTheGuest(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-memoryfenced")
+	provider := memoryProviderPublishing(t, nc, "vol-memoryfenced")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-memoryfenced", "node-b")
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not take a fence notice from a non-viperblock provider. "+
+			"Fix: subscribe the contract fence-notice subject in fenceNoticeSubs (daemon_handlers_fence.go)")
+	assert.Equal(t, guest.ID, stopped)
+	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest),
+		"ADR-0007:S4: the guest runs on against an export its provider revoked")
+}
+
+// TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing is the ordinary case: a
+// volume fenced after its guest has gone leaves every local guest as it was.
+func TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-inuse")
+	provider := memoryProviderPublishing(t, nc, "vol-noguest")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-noguest", "node-b")
+	require.NoError(t, err, "a fence with nothing to stop is not an error")
+	assert.Empty(t, stopped)
+	assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest), "a guest on another volume must not be stopped")
+}
+
 // TestS4_RepeatedFenceNoticeIsHarmless covers the cost of the retry: the notice
 // can arrive more than once, and a second copy must leave the fenced guest as the
 // first did rather than re-driving it or failing.
 func TestS4_RepeatedFenceNoticeIsHarmless(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedtwice")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{Volume: "vol-fencedtwice", Winner: "node-b"})
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{
-	require.NoError(t, err)
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedtwice", NodeID: "node-a", Winner: "node-b",
-
+	})
 	for range 2 {
-		reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
+		ack := requestFenceNotice(t, nc, payload)
-		require.NoError(t, err)
+		assert.Nil(t, ack.Error, "a repeated notice is not an error")
-		var ack viperblocklegacyv1.VolumeFencedAck
-		require.NoError(t, json.Unmarshal(reply.Data, &ack))
-		assert.Empty(t, ack.Error, "a repeated notice is not an error")
 	}
 	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest))
 }
 
-// TestVolumeFenced_BadEventIsAnsweredWithAnError stops a notice that can never be
+// TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError stops a notice that
-// acted on from being repeated forever.
+// can never be acted on from being repeated forever, and leaves the guest alone.
-func TestVolumeFenced_BadEventIsAnsweredWithAnError(t *testing.T) {
+func TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError(t *testing.T) {
-	d, _, nc := fenceDaemon(t, "vol-unused")
+	cases := map[string]struct {
-
+		raw  []byte
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), []byte(`{"volume":""}`), 2*time.Second)
+		code ebsprovider.ErrorCode
-	require.NoError(t, err)
+	}{
-	var ack viperblocklegacyv1.VolumeFencedAck
+		"undecodable":     {[]byte(`{`), ebsprovider.ErrorCodeInvalidArgument},
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+		"no volume":       {[]byte(`{"schema_version":1,"volume_id":""}`), ebsprovider.ErrorCodeInvalidArgument},
-	assert.NotEmpty(t, ack.Error)
+		"unknown version": {[]byte(`{"schema_version":0,"volume_id":"vol-unused"}`), ebsprovider.ErrorCodeUnsupportedVersion},
+		"future version":  {fmt.Appendf(nil, `{"schema_version":%d,"volume_id":"vol-unused"}`, ebsprovider.SchemaVersion+1), ebsprovider.ErrorCodeUnsupportedVersion},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			d, guest, nc := fenceDaemon(t, "vol-unused")
+			ack := requestFenceNotice(t, nc, tc.raw)
+			require.NotNil(t, ack.Error)
+			assert.Equal(t, tc.code, ack.Error.Code)
+			if tc.code == ebsprovider.ErrorCodeUnsupportedVersion {
+				assert.Contains(t, ack.Error.Message, "unsupported schema version ")
+				assert.Contains(t, ack.Error.Message, ebsprovider.SupportedVersionsText())
+			}
+			assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest))
+		})
+	}
 }

spinifex/runtime/daemon/daemon_handlers_instance_test.go

@@ -38,9 +38,13 @@ func daemonWithFakeStateStore(t *testing.T, store *vmmock.StateStore) *Daemon {
 	t.Helper()
 	d := createTestDaemon(t, sharedNATSURL)
 	d.stateStore = store
+	objects := objectstore.NewMemoryObjectStore()
+	for _, stopped := range store.Stopped {
+		seedBootVolume(t, objects, d.config.Predastore.Bucket, stopped.AccountID, stopped.ID)
+	}
 	d.instanceService = ec2instance.NewInstanceServiceImpl(
 		d.config, d.resourceMgr.instanceTypes, d.natsConn,
-		objectstore.NewMemoryObjectStore(), d.vmMgr, d.resourceMgr, store,
+		objects, d.vmMgr, d.resourceMgr, store,
 	)
 	return d
 }
@@ -71,7 +75,7 @@ func decodeError(t *testing.T, data []byte) map[string]any {
 
 // stoppedVMFixture builds a minimally-valid stopped VM for handler tests.
 func stoppedVMFixture(id, accountID string) *vm.VM {
-	return &vm.VM{
+	return withBootVolume(&vm.VM{
 		ID:           id,
 		Status:       vm.StateStopped,
 		InstanceType: "t3.micro",
@@ -84,7 +88,7 @@ func stoppedVMFixture(id, accountID string) *vm.VM {
 			InstanceId:   aws.String(id),
 			InstanceType: aws.String("t3.micro"),
 		},
-	}
+	})
 }
 
 // --- handleEC2StartStoppedInstance ---

spinifex/runtime/daemon/daemon_handlers_test.go

@@ -1438,6 +1438,85 @@ func TestHandleEC2GetConsoleOutput(t *testing.T) {
 	assert.Contains(t, string(decoded), "Boot complete.")
 }
 
+func TestHandleEC2GetConsoleOutput_ReadsRetiredFileWhenLiveNameIsGone(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	instanceID := "i-console-retired-001"
+	logPath := t.TempDir() + "/console-" + instanceID + ".log"
+	var body bytes.Buffer
+	for i := 0; body.Len() < 200<<10; i++ {
+		fmt.Fprintf(&body, "l%014d\n", i)
+	}
+	require.NoError(t, os.WriteFile(vm.RetiredConsolePath(logPath), body.Bytes(), 0o644))
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateShuttingDown,
+		AccountID: testAccountID,
+		Config:    vm.Config{ConsoleLogPath: logPath},
+	})
+
+	topic := fmt.Sprintf("ec2.%s.GetConsoleOutput", instanceID)
+	sub, err := daemon.natsConn.Subscribe(topic, daemon.handleEC2GetConsoleOutput)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+	reqData, _ := json.Marshal(&ec2.GetConsoleOutputInput{InstanceId: aws.String(instanceID)})
+	reply, err := natsRequest(daemon.natsConn, topic, reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.GetConsoleOutputOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	decoded, err := base64.StdEncoding.DecodeString(*output.Output)
+	require.NoError(t, err)
+	want := body.Bytes()[body.Len()-64*1024:]
+	decodedSize := len(decoded)
+	assert.Equal(t, 64*1024, decodedSize)
+	assert.True(t, bytes.Equal(want, decoded), "the newest 64 KiB of the retired file")
+}
+
+func TestHandleEC2GetConsoleOutput_AfterTrimReturnsNewest64KiB(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	instanceID := "i-console-trim-001"
+	dir := t.TempDir()
+	logPath := dir + "/console-" + instanceID + ".log"
+	var body bytes.Buffer
+	for i := 0; body.Len() < 5<<20; i++ {
+		fmt.Fprintf(&body, "l%014d\n", i)
+	}
+	require.NoError(t, os.WriteFile(logPath, body.Bytes(), 0o644))
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateRunning,
+		AccountID: testAccountID,
+		Config:    vm.Config{ConsoleLogPath: logPath},
+	})
+	reaper := vm.NewConsoleLogReaper(dir)
+	_, err := reaper.Sweep(context.Background())
+	require.NoError(t, err)
+	info, err := os.Stat(logPath)
+	require.NoError(t, err)
+	require.Equal(t, int64(1<<20), info.Size(), "the console must have been trimmed before it is read")
+
+	topic := fmt.Sprintf("ec2.%s.GetConsoleOutput", instanceID)
+	sub, err := daemon.natsConn.Subscribe(topic, daemon.handleEC2GetConsoleOutput)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+	reqData, _ := json.Marshal(&ec2.GetConsoleOutputInput{InstanceId: aws.String(instanceID)})
+	reply, err := natsRequest(daemon.natsConn, topic, reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.GetConsoleOutputOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	decoded, err := base64.StdEncoding.DecodeString(*output.Output)
+	require.NoError(t, err)
+	want := body.Bytes()[body.Len()-64*1024:]
+	decodedSize := len(decoded)
+	assert.Equal(t, 64*1024, decodedSize)
+	assert.True(t, bytes.Equal(want, decoded), "the response is the newest 64 KiB of the original console")
+}
+
 func TestHandleEC2GetConsoleOutput_EmptyLog(t *testing.T) {
 	natsURL := sharedNATSURL
 	daemon := createFullTestDaemon(t, natsURL)
@@ -2763,7 +2842,7 @@ func TestHandleEC2DescribeInstanceTypes_NoFilter(t *testing.T) {
 func TestHandleEC2StartStoppedInstance_InstanceTypeNotAvailable(t *testing.T) {
 	daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
 
-	stoppedVM := &vm.VM{
+	stoppedVM := withBootVolume(&vm.VM{
 		ID:           "i-start-badtype-001",
 		Status:       vm.StateStopped,
 		InstanceType: "z99.nonexistent",
@@ -2772,7 +2851,13 @@ func TestHandleEC2StartStoppedInstance_InstanceTypeNotAvailable(t *testing.T) {
 			InstanceId:   aws.String("i-start-badtype-001"),
 			InstanceType: aws.String("z99.nonexistent"),
 		},
-	}
+	})
+	objects := objectstore.NewMemoryObjectStore()
+	seedBootVolume(t, objects, daemon.config.Predastore.Bucket, testAccountID, stoppedVM.ID)
+	daemon.instanceService = ec2instance.NewInstanceServiceImpl(
+		daemon.config, daemon.resourceMgr.instanceTypes, daemon.natsConn,
+		objects, daemon.vmMgr, daemon.resourceMgr, daemon.jsManager,
+	)
 	err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(stoppedVM.ID) })

spinifex/runtime/daemon/daemon_handlers_volume.go

@@ -11,7 +11,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
@@ -139,50 +138,6 @@ func (d *Daemon) handleDetachVolume(ctx context.Context, msg *nats.Msg, command
 	return outcomeSuccess
 }
 
-// handleDrainVolume flushes the volume's in-flight writes to S3 by dialing the
-// drain socket the local NBD plugin serves. ec2.CreateSnapshot is answered by
-// any node in the cluster, but only the node hosting the instance subscribes
-// ec2.cmd.{instanceID}, so routing the drain here lands it where the writes are.
-//
-// Runs on its own goroutine (see handleEC2Events) so a long flush cannot hold
-// the instance's command subscription.
-func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command ec2v1.EC2InstanceCommand, instance *vm.VM) string {
-	ctx, span := startOpSpan(ctx, "ec2.DrainVolume", command.ID)
-	var err error
-	defer func() { endOpSpan(span, err) }()
-
-	if command.DrainVolumeData == nil || command.DrainVolumeData.VolumeID == "" {
-		slog.ErrorContext(ctx, "DrainVolume: missing drain volume data", "instanceId", command.ID)
-		return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue)
-	}
-
-	volumeID := command.DrainVolumeData.VolumeID
-
-	// Only stable post-teardown states prove the volume was sealed. Transitional
-	// states still have a live or shutting-down writer, so they must drain or
-	// fail rather than acknowledge an older checkpoint as current.
-	status := d.vmMgr.Status(instance)
-	if status == vm.StateStopped || status == vm.StateTerminated {
-		slog.InfoContext(ctx, "DrainVolume: instance teardown is complete, nothing to drain",
-			"volumeId", volumeID, "instanceId", command.ID, "status", status)
-		respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusNotRunning})
-		return outcomeSuccess
-	}
-
-	slog.InfoContext(ctx, "Draining volume before snapshot", "volumeId", volumeID, "instanceId", command.ID)
-
-	// A missing or unresponsive socket under a running instance means the writes
-	// cannot be made current, so the caller must fail its snapshot rather than
-	// read a stale checkpoint.
-	if err = ec2snapshot.DrainVolumeSocket(d.config.DataDir, volumeID); err != nil {
-		slog.ErrorContext(ctx, "DrainVolume: drain failed", "volumeId", volumeID, "instanceId", command.ID, "err", err)
-		return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal)
-	}
-
-	respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	return outcomeSuccess
-}
-
 // attachDetachErrorCode maps a vm.Manager error returned by AttachVolume
 // or DetachVolume to the AWS API error code that the SDK expects.
 func attachDetachErrorCode(err error) string {

spinifex/runtime/daemon/daemon_handlers_volume_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
 	"testing"
 	"time"
@@ -12,17 +11,14 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
 )
 
 // TestAttachDetachErrorCode locks down the manager-error → AWS-API-code
@@ -370,257 +366,6 @@ func TestAttachVolume_IdempotentSameInstance_DeviceMismatch(t *testing.T) {
 	assert.Contains(t, string(resp.Data), awserrors.ErrorVolumeInUse)
 }
 
-// drainCommandFor builds the drain command a snapshot addresses to the node
-// hosting instanceID.
-func drainCommandFor(t *testing.T, instanceID, volumeID string) []byte {
-	t.Helper()
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-	data, err := json.Marshal(command)
-	require.NoError(t, err)
-	return data
-}
-
-// drainTestDaemon returns a daemon owning a running instance, with a data dir
-// short enough to hold the drain socket path.
-func drainTestDaemon(t *testing.T, instanceID string) *Daemon {
-	t.Helper()
-	return drainTestDaemonInState(t, instanceID, vm.StateRunning)
-}
-
-// drainTestDaemonInState is drainTestDaemon for an instance the node still
-// holds in some other state.
-func drainTestDaemonInState(t *testing.T, instanceID string, status vm.InstanceState) *Daemon {
-	t.Helper()
-	daemon := createTestDaemon(t, sharedNATSURL)
-	daemon.config.DataDir = testutil.SocketTempDir(t)
-	daemon.vmMgr.Insert(&vm.VM{
-		ID:           instanceID,
-		Status:       status,
-		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(t),
-		Instance:     &ec2.Instance{},
-	})
-	return daemon
-}
-
-// drainRequest issues a drain on the instance's command subject, returning the
-// raw reply.
-func drainRequest(t *testing.T, daemon *Daemon, instanceID, volumeID string) *nats.Msg {
-	t.Helper()
-	reply, err := sendDrainCommand(daemon, instanceID, drainCommandFor(t, instanceID, volumeID))
-	require.NoError(t, err)
-	return reply
-}
-
-// sendDrainCommand is drainRequest without the assertions, so a test can issue
-// one from a goroutine and assert on the result back on the test goroutine.
-func sendDrainCommand(daemon *Daemon, instanceID string, command []byte) (*nats.Msg, error) {
-	msg := nats.NewMsg("ec2.cmd." + instanceID)
-	msg.Data = command
-	msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
-	return daemon.natsConn.RequestMsg(msg, 5*time.Second)
-}
-
-// subscribeEC2Commands wires the daemon's real dispatcher onto the per-instance
-// command subject, so tests exercise the same serial delivery production has.
-func subscribeEC2Commands(t *testing.T, daemon *Daemon, instanceID string) {
-	t.Helper()
-	sub, err := daemon.natsConn.Subscribe("ec2.cmd."+instanceID, daemon.handleEC2Events)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-}
-
-// A drain routed to the node hosting the instance reaches the local socket and
-// acks, which is what lets a snapshot answered elsewhere read a current
-// checkpoint.
-func TestDrainVolume_HostNodeAcksLocalSocket(t *testing.T) {
-	const instanceID, volumeID = "i-drain-ok", "vol-drain-ok"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(resp.Data, &ack))
-	assert.Equal(t, volumeID, ack.VolumeID)
-	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-}
-
-// The owning node having no socket for the volume means the writes cannot be
-// flushed. That must reach the caller as an error, never as a silent success:
-// the caller would otherwise snapshot a stale checkpoint.
-func TestDrainVolume_HostNodeWithoutSocketFails(t *testing.T) {
-	const instanceID, volumeID = "i-drain-nosock", "vol-drain-nosock"
-	daemon := drainTestDaemon(t, instanceID)
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-}
-
-// A plugin that refuses the drain (ERR rather than OK) is a failure, not an ack.
-func TestDrainVolume_HostNodeRelaysSocketFailure(t *testing.T) {
-	const instanceID, volumeID = "i-drain-err", "vol-drain-err"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "ERR\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-}
-
-// A drain command with no volume is a caller error, not a server fault.
-func TestDrainVolume_MissingVolumeDataIsInvalidParameter(t *testing.T) {
-	const instanceID = "i-drain-novol"
-	daemon := drainTestDaemon(t, instanceID)
-
-	command := ec2v1.EC2InstanceCommand{
-		ID:         instanceID,
-		Attributes: ec2v1.EC2CommandAttributes{DrainVolume: true},
-	}
-	data, err := json.Marshal(command)
-	require.NoError(t, err)
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, data)
-
-	assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, resp.Data))
-}
-
-// Only the instance's owner may drain its volume: the command carries the same
-// ownership gate as every other per-instance command.
-func TestDrainVolume_ForeignAccountRejected(t *testing.T) {
-	const instanceID, volumeID = "i-drain-foreign", "vol-drain-foreign"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		"999988887777", drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, replyErrCode(t, resp.Data))
-}
-
-// Only stable post-teardown states prove the volume is sealed. These states
-// acknowledge not-running without touching the absent plugin socket.
-func TestDrainVolume_CompletedTeardownAcksNotRunning(t *testing.T) {
-	for _, status := range []vm.InstanceState{vm.StateStopped, vm.StateTerminated} {
-		t.Run(string(status), func(t *testing.T) {
-			instanceID := "i-drain-" + string(status)
-			volumeID := "vol-drain-" + string(status)
-			daemon := drainTestDaemonInState(t, instanceID, status)
-
-			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-				testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-			var ack ec2v1.DrainVolumeResponse
-			require.NoError(t, json.Unmarshal(resp.Data, &ack))
-			assert.Equal(t, volumeID, ack.VolumeID)
-			assert.Equal(t, ec2v1.DrainVolumeStatusNotRunning, ack.Status)
-		})
-	}
-}
-
-// Transitional and uncertain states are not proof of a sealed checkpoint. An
-// absent socket in any of them must fail rather than silently snapshot stale
-// data while launch or teardown may still own the volume.
-func TestDrainVolume_UnsealedStatesRequireDrain(t *testing.T) {
-	statuses := []vm.InstanceState{
-		vm.StateRunning,
-		vm.StateStopping,
-		vm.StateShuttingDown,
-		vm.StatePending,
-		vm.StateProvisioning,
-		vm.StateError,
-	}
-	for _, status := range statuses {
-		t.Run(string(status), func(t *testing.T) {
-			instanceID := "i-drain-" + string(status)
-			volumeID := "vol-drain-" + string(status)
-			daemon := drainTestDaemonInState(t, instanceID, status)
-
-			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-				testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-			assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-		})
-	}
-}
-
-// nats.go delivers a subscription's messages serially, so a drain run inline
-// would hold ec2.cmd.{instanceID} for the length of the flush — stalling stop,
-// terminate and hot-plug for that instance. A drain still flushing must not
-// keep the next command waiting.
-func TestDrainVolume_SlowDrainDoesNotBlockTheCommandSubject(t *testing.T) {
-	const instanceID = "i-drain-parallel"
-	daemon := drainTestDaemon(t, instanceID)
-	accepted, release := testutil.StartBlockingDrainSocket(t, daemon.config.DataDir, "vol-drain-slow", "OK\n")
-	testutil.StartDrainSocket(t, daemon.config.DataDir, "vol-drain-fast", "OK\n")
-	subscribeEC2Commands(t, daemon, instanceID)
-
-	type reply struct {
-		msg *nats.Msg
-		err error
-	}
-	slowCommand := drainCommandFor(t, instanceID, "vol-drain-slow")
-	slow := make(chan reply, 1)
-	go func() {
-		msg, err := sendDrainCommand(daemon, instanceID, slowCommand)
-		slow <- reply{msg, err}
-	}()
-
-	// Only assert once the slow drain is genuinely mid-flush.
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("the slow drain never reached its socket")
-	}
-
-	fast := drainRequest(t, daemon, instanceID, "vol-drain-fast")
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(fast.Data, &ack))
-	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-
-	release()
-	select {
-	case got := <-slow:
-		require.NoError(t, got.err)
-		require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
-		assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-	case <-time.After(5 * time.Second):
-		t.Fatal("the released drain never replied")
-	}
-}
-
-// Running the drain off the delivery goroutine must not leak it: the goroutine
-// has to be gone once the ack is on the wire, or a snapshot loop against a
-// wedged plugin accumulates them.
-func TestDrainVolume_DispatchedGoroutineDoesNotLeak(t *testing.T) {
-	const instanceID, volumeID = "i-drain-leak", "vol-drain-leak"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-	subscribeEC2Commands(t, daemon, instanceID)
-
-	// Take the baseline after one full round trip: the first request is what
-	// makes nats.go stand up its long-lived response-inbox goroutine, which
-	// would otherwise read as the leak.
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
-	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-	ignoreExisting := goleak.IgnoreCurrent()
-
-	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
-	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-
-	goleak.VerifyNone(t, ignoreExisting)
-}
-
 // seedProviderVolume allocates volumeID in the daemon's EBS provider so
 // operations that reach the provider (expand, delete) find it there.
 func seedProviderVolume(t *testing.T, daemon *Daemon, volumeID string, sizeGiB int64) {