refactor(uefi): read instance state for the store reaper and orphan report through daemon capabilities

The reaper and the orphan report named the raw instance-state records,
buckets and key prefixes, which ADR-0007:S6 forbids a new consumer to do.
They now ask TerminateDecided, OpenTeardowns, MarkTeardownDone and
KnownInstanceIDs, each stating its absent-record and failure behaviour.

contracts/ec2/v1/README.md

@@ -23,7 +23,7 @@ subject shape are wire compatibility. Callers and receivers must use this
 package rather than rebuild those details. `commands_test.go` preserves the
 version-1 JSON shape and subject form.
 
-This package owns shared command payloads. It
+This package owns shared command payloads and the drain-volume response. It
 does not define every action's reply body, authorize the request, store EC2
 state or implement NATS transport. Those remain responsibilities of the EC2
 domain and its runtime composition.
@@ -37,13 +37,6 @@ contract or an approved compatibility plan; it is not a package refactor.
 Only the subject is owned here today.
 The request (`account_id`) and reply (`vpc_id`, `error`) bodies are still declared as anonymous structs on each side; moving them into this package is a contract change that needs its own characterization first.
 
-## Volume held elsewhere
-
-A start refused because another node holds one of the instance's volumes is an `IncorrectState` error payload on the `ec2.start`, `ec2.start.<node>` and `ec2.cmd.<instance-id>` replies, and the AWS body cannot say why.
-The owning node adds the `X-Volume-Held-Elsewhere: true` header to that reply, the `ec2.start` forwarder relays it, and `StartInstances` wraps `ErrVolumeHeldElsewhere` around the error it rebuilds, so a caller matches it with `errors.Is` rather than reading the message.
-The owning node sets the header only from the EBS provider's `volume_held_elsewhere` code, which every provider returns for this refusal; no reader matches the message text.
-`VolumeHeldElsewhereText` is the wording viperblockd puts in that refusal's message, not a signal.
-
 ## Platform inventory
 
 `PlatformInventorySubject` (`ec2.v1.PlatformInventory`) is a request/reply route on which a daemon in the `spinifex-workers` queue group reports one account's key pairs, placement groups, launch templates and their `ec2-tags` entries from the EC2 owner stores.

contracts/ec2/v1/commands.go

@@ -29,6 +29,7 @@ type EC2InstanceCommand struct {
 	Attributes                EC2CommandAttributes       `json:"attributes"`
 	AttachVolumeData          *AttachVolumeData          `json:"attach_volume_data,omitempty"`
 	DetachVolumeData          *DetachVolumeData          `json:"detach_volume_data,omitempty"`
+	DrainVolumeData           *DrainVolumeData           `json:"drain_volume_data,omitempty"`
 	AttachENIData             *AttachENIData             `json:"attach_eni_data,omitempty"`
 	DetachENIData             *DetachENIData             `json:"detach_eni_data,omitempty"`
 	IamProfileAssociationData *IamProfileAssociationData `json:"iam_profile_association_data,omitempty"`
@@ -44,6 +45,7 @@ type EC2CommandAttributes struct {
 	StartInstance               bool `json:"start_instance"`
 	AttachVolume                bool `json:"attach_volume"`
 	DetachVolume                bool `json:"detach_volume"`
+	DrainVolume                 bool `json:"drain_volume,omitempty"`
 	RebootInstance              bool `json:"reboot_instance"`
 	AttachENI                   bool `json:"attach_eni"`
 	DetachENI                   bool `json:"detach_eni"`
@@ -67,6 +69,33 @@ type DetachVolumeData struct {
 	Force    bool   `json:"force,omitempty"`
 }
 
+// DrainVolumeData carries parameters for a drain-volume command, which flushes
+// the volume's in-flight writes to S3 before a snapshot reads them. The command
+// is addressed to the instance the volume is attached to because the drain
+// socket only exists on the node hosting it.
+type DrainVolumeData struct {
+	VolumeID string `json:"volume_id"`
+}
+
+const (
+	// DrainVolumeStatusDrained is the ack a node returns once the volume's
+	// in-flight writes have reached S3.
+	DrainVolumeStatusDrained = "drained"
+
+	// DrainVolumeStatusNotRunning is the ack a node returns when it still holds
+	// the instance but the VM is not running, so no process is writing to the
+	// volume and its sealed checkpoint is already current. An attachment record
+	// outlives the writer — stop deliberately leaves boot volumes attached — so
+	// the caller cannot tell this from the record alone.
+	DrainVolumeStatusNotRunning = "not-running"
+)
+
+// DrainVolumeResponse is the reply to a drain-volume command.
+type DrainVolumeResponse struct {
+	VolumeID string `json:"volume_id"`
+	Status   string `json:"status"`
+}
+
 // AttachENIData carries parameters for an attach-network-interface command.
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`

contracts/ec2/v1/commands_test.go

@@ -16,6 +16,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			StartInstance:               true,
 			AttachVolume:                true,
 			DetachVolume:                true,
+			DrainVolume:                 true,
 			RebootInstance:              true,
 			AttachENI:                   true,
 			DetachENI:                   true,
@@ -27,6 +28,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
+		DrainVolumeData:           &DrainVolumeData{VolumeID: "vol-3"},
 		AttachENIData:             &AttachENIData{NetworkInterfaceID: "eni-1", DeviceIndex: 2},
 		DetachENIData:             &DetachENIData{AttachmentID: "eni-attach-1", Force: true},
 		IamProfileAssociationData: &IamProfileAssociationData{InstanceProfileArn: "arn:aws:iam::123456789012:instance-profile/example"},
@@ -45,6 +47,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"start_instance": true,
 			"attach_volume": true,
 			"detach_volume": true,
+			"drain_volume": true,
 			"reboot_instance": true,
 			"attach_eni": true,
 			"detach_eni": true,
@@ -56,6 +59,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
+		"drain_volume_data": {"volume_id": "vol-3"},
 		"attach_eni_data": {"network_interface_id": "eni-1", "device_index": 2},
 		"detach_eni_data": {"attachment_id": "eni-attach-1", "force": true},
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
@@ -65,6 +69,16 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 	}`, string(data))
 }
 
+func TestDrainVolumeResponseJSONContract(t *testing.T) {
+	data, err := json.Marshal(DrainVolumeResponse{
+		VolumeID: "vol-1",
+		Status:   DrainVolumeStatusDrained,
+	})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"volume_id":"vol-1","status":"drained"}`, string(data))
+	require.Equal(t, "not-running", DrainVolumeStatusNotRunning)
+}
+
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/ec2/v1/volume_held.go

@@ -1,13 +1,7 @@
 package ec2v1
 
-import "errors"
-
-// VolumeHeldElsewhereText is the message viperblockd gives a lease refusal. It
-// is wording for the operator; callers classify the refusal by its code.
+// VolumeHeldElsewhereText is the text of a start refused because another node
+// holds one of the instance's volumes. It crosses NATS as message text, so
+// readers match on it; the provider that refuses and every reader share this
+// string so they cannot drift.
 const VolumeHeldElsewhereText = "volume is leased by another owner"
-
-// ErrVolumeHeldElsewhere marks a start refused because another node still
-// holds one of the instance's volumes. The owning node sets it from the EBS
-// provider's held-elsewhere code, and a start reply carries it across NATS as
-// the X-Volume-Held-Elsewhere header rather than as message text.
-var ErrVolumeHeldElsewhere = errors.New("a volume is leased by another node")

docs/COMMANDS.md

@@ -135,7 +135,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
+| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
 | `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
 

docs/DESIGN.md

@@ -112,14 +112,14 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
+3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
 6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
 
 ```go
-// Volume mounting through the EBS provider contract (runtime/daemon/vm_adapters.go)
-published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: req.Name, NodeID: a.node})
+// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
+reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
 
 // QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
 cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64

internal/testkit/drainsocket.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
-	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -29,11 +28,10 @@ func SocketTempDir(t *testing.T) string {
 // StartDrainSocket stands in for the NBD plugin's drain socket, at the path the
 // plugin serves it on the node hosting volumeID ({dataDir}/viperblock/{volumeID}
 // /snapshot.sock). Every connection is answered with ack ("OK\n" for a drain
-// that reached S3, "ERR\n" for one that did not) and closed. connections
-// reports how many drains have been asked for so far.
-func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) (connections func() int) {
+// that reached S3, "ERR\n" for one that did not) and closed.
+func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) {
 	t.Helper()
-	return serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
+	serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
 }
 
 // StartBlockingDrainSocket stands in for a plugin whose flush is still running:
@@ -45,7 +43,7 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 
 	acceptedCh := make(chan struct{})
 	releaseCh := make(chan struct{})
-	_ = serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
+	serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
 
 	var once sync.Once
 	t.Cleanup(func() { once.Do(func() { close(releaseCh) }) })
@@ -54,9 +52,8 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 }
 
 // serveDrainSocket answers every connection with ack, optionally closing
-// accepted on the first one and waiting for release before it replies. The
-// returned func counts the connections accepted so far.
-func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) func() int {
+// accepted on the first one and waiting for release before it replies.
+func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) {
 	t.Helper()
 
 	dir := filepath.Join(dataDir, "viperblock", volumeID)
@@ -66,7 +63,6 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = ln.Close() })
 
-	var count atomic.Int64
 	go func() {
 		var once sync.Once
 		for {
@@ -74,7 +70,6 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			if err != nil {
 				return // listener closed by the cleanup above
 			}
-			count.Add(1)
 			if accepted != nil {
 				once.Do(func() { close(accepted) })
 			}
@@ -85,5 +80,4 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			_ = conn.Close()
 		}
 	}()
-	return func() int { return int(count.Load()) }
 }

spinifex/domains/ec2/awsapi/instance/StartInstances.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
 	"time"
@@ -103,7 +102,7 @@ func startLiveInstance(ctx context.Context, natsConn *nats.Conn, instanceID, acc
 
 	if responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data); parseErr != nil {
 		slog.ErrorContext(ctx, "StartInstances: owner returned error", "instance_id", instanceID, "code", *responseError.Code)
-		return nil, false, startRefusalError(responseError, msg.Header)
+		return nil, false, startRefusalError(responseError)
 	}
 
 	slog.InfoContext(ctx, "StartInstances: restarted via owner node", "instance_id", instanceID, "response", string(msg.Data))
@@ -136,25 +135,18 @@ func startStoppedInstance(ctx context.Context, natsConn *nats.Conn, instanceID,
 
 	if responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data); parseErr != nil {
 		slog.ErrorContext(ctx, "StartInstances: Daemon returned error", "instance_id", instanceID, "code", *responseError.Code)
-		return nil, startRefusalError(responseError, msg.Header)
+		return nil, startRefusalError(responseError)
 	}
 
 	slog.InfoContext(ctx, "StartInstances: Command sent successfully", "instance_id", instanceID, "response", string(msg.Data))
 	return newStateChange(instanceID, 0, "pending", 80, "stopped"), nil
 }
 
-// startRefusalError rebuilds the daemon's error, keeping the message it sent,
-// which names the node holding the volume, and wrapping ErrVolumeHeldElsewhere
-// when the reply's header says that is why the start was refused.
-func startRefusalError(responseError ec2.ResponseError, header nats.Header) error {
-	err := startRefusalCodeError(responseError)
-	if header.Get(natsmsg.VolumeHeldElsewhereHeader) == "true" {
-		return fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere, err)
-	}
-	return err
-}
-
-func startRefusalCodeError(responseError ec2.ResponseError) error {
+// startRefusalError rebuilds the daemon's error, keeping the message it sent.
+// Reconstructing from the code alone renders the generic ErrorLookup text
+// instead, which replaces the name of the node holding the volume with
+// boilerplate about attaching volumes -- the one fact the operator needed.
+func startRefusalError(responseError ec2.ResponseError) error {
 	var code, message string
 	if responseError.Code != nil {
 		code = *responseError.Code

spinifex/domains/ec2/awsapi/instance/StartInstances_test.go

@@ -3,14 +3,11 @@ package instance
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -164,33 +161,3 @@ func TestStartInstances_MixedSuccessAndFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, "InvalidInstanceID.NotFound", err.Error())
 }
-
-// The AWS body says only IncorrectState, so the header is the one thing that
-// tells a held volume apart from any other refusal in that code.
-func TestStartInstances_VolumeHeldElsewhereHeader(t *testing.T) {
-	t.Parallel()
-	for name, held := range map[string]bool{"with header": true, "without header": false} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			_, nc := startTestNATSServer(t)
-			const id = "i-0abc0000000000002"
-			_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(id), func(msg *nats.Msg) {
-				reply := nats.NewMsg(msg.Reply)
-				reply.Data = awserrors.GenerateErrorPayloadWithMessage(awserrors.ErrorIncorrectState, "volume vol-1 is published to node-a")
-				if held {
-					reply.Header.Set(natsmsg.VolumeHeldElsewhereHeader, "true")
-				}
-				_ = msg.RespondMsg(reply)
-			})
-			require.NoError(t, err)
-
-			_, err = StartInstances(t.Context(), &ec2.StartInstancesInput{InstanceIds: []*string{aws.String(id)}}, nc, "123456789012")
-			require.Error(t, err)
-			assert.Equal(t, held, errors.Is(err, ec2v1.ErrVolumeHeldElsewhere))
-			code, message, ok := awserrors.ResolveErrorDetail(err)
-			require.True(t, ok)
-			assert.Equal(t, awserrors.ErrorIncorrectState, code)
-			assert.Equal(t, "volume vol-1 is published to node-a", message)
-		})
-	}
-}

spinifex/domains/ec2/image/drain_test.go

@@ -0,0 +1,140 @@
+package image
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const drainImageInstanceID = "i-image-drain-host"
+
+// setupDrainImageService mirrors domains/ec2/snapshot's setupDrainService: a
+// reachable (but rejecting) Predastore host so a drained volume falls through
+// into the real snapshot path and fails fast, a data dir with no local drain
+// socket, and a NATS connection to route the drain to the hosting node.
+func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
+	t.Helper()
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	t.Cleanup(srv.Close)
+
+	_, nc := testutil.StartTestNATS(t)
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{
+		DataDir: testutil.SocketTempDir(t),
+		Predastore: config.PredastoreConfig{
+			Host:   srv.URL,
+			Bucket: testBucket,
+		},
+	}
+	svc := NewImageServiceImplWithConfig(cfg, store, nc)
+	return svc, store, nc
+}
+
+func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
+	t.Helper()
+	require.NoError(t, volumestate.Write(context.Background(), store, testBucket, volumeID, volumestate.Record{
+		State:            state,
+		AttachedInstance: instanceID,
+	}))
+}
+
+func drainImageResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
+	t.Helper()
+
+	got := make(chan *nats.Msg, 4)
+	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		got <- msg
+		_ = msg.Respond(reply)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	return got
+}
+
+func drainedImageAck(t *testing.T, volumeID string) []byte {
+	t.Helper()
+	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
+	require.NoError(t, err)
+	return data
+}
+
+func awaitImageDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
+	t.Helper()
+	select {
+	case msg := <-got:
+		var command ec2v1.EC2InstanceCommand
+		require.NoError(t, json.Unmarshal(msg.Data, &command))
+		return command
+	case <-time.After(2 * time.Second):
+		t.Fatal("the node hosting the volume never received a drain command")
+		return ec2v1.EC2InstanceCommand{}
+	}
+}
+
+// CreateImageFromInstance's IsRunning path must drain the volume's hosting node
+// before it reads the live checkpoint — the same guarantee ec2.CreateSnapshot
+// has. Without this, a checkpoint predating a guest-triggered GPT rewrite
+// (e.g. growpart on first boot) can be captured silently.
+func TestSnapshotRunningVolume_DrainsAttachedVolume(t *testing.T) {
+	svc, store, nc := setupDrainImageService(t)
+	createTestVolumeConfig(t, store, "vol-image-drain", 10)
+	seedImageVolumeAttachment(t, store, "vol-image-drain", "in-use", drainImageInstanceID)
+	got := drainImageResponder(t, nc, drainImageInstanceID, drainedImageAck(t, "vol-image-drain"))
+
+	err := svc.snapshotRunningVolume("vol-image-drain", "snap-image-drain", testAccountID)
+	// The rejecting Predastore stub fails the snapshot itself; what matters
+	// here is that the drain reached the hosting node first.
+	require.Error(t, err)
+
+	command := awaitImageDrainCommand(t, got)
+	assert.True(t, command.Attributes.DrainVolume)
+	assert.Equal(t, drainImageInstanceID, command.ID)
+	require.NotNil(t, command.DrainVolumeData)
+	assert.Equal(t, "vol-image-drain", command.DrainVolumeData.VolumeID)
+}
+
+// An attached volume whose host reports the drain failed must fail
+// CreateImage rather than register an AMI built from a stale checkpoint.
+func TestSnapshotRunningVolume_UndrainableAttachedVolumeFails(t *testing.T) {
+	svc, store, nc := setupDrainImageService(t)
+	createTestVolumeConfig(t, store, "vol-image-nodrain", 10)
+	seedImageVolumeAttachment(t, store, "vol-image-nodrain", "in-use", drainImageInstanceID)
+	drainImageResponder(t, nc, drainImageInstanceID,
+		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
+
+	err := svc.snapshotRunningVolume("vol-image-nodrain", "snap-image-nodrain", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+// An unattached (stopped-instance) volume takes the no-drain path: the
+// checkpoint Close() left behind is already current.
+func TestSnapshotRunningVolume_AvailableSkipsDrain(t *testing.T) {
+	svc, store, _ := setupDrainImageService(t)
+	createTestVolumeConfig(t, store, "vol-image-available", 10)
+	seedImageVolumeAttachment(t, store, "vol-image-available", "available", "")
+	svc.natsConn = nil
+
+	// No responder exists and the socket is never dialled: proceeding straight
+	// to the (rejecting) snapshot path proves the drain was skipped, not stuck.
+	err := svc.snapshotRunningVolume("vol-image-available", "snap-image-available", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -2,20 +2,16 @@ package image
 
 import (
 	"context"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -100,15 +96,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
-// root-volume snapshot under a provider is created via provider.CreateSnapshot
+// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a
+// stopped-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -122,22 +118,22 @@ func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
+	err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
+	err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -226,98 +222,3 @@ func TestCreateImageFromInstance_SetsEnaSupportFromInstance(t *testing.T) {
 	require.NotNil(t, ami.EnaSupport)
 	assert.True(t, *ami.EnaSupport)
 }
-
-// createUnavailableProvider refuses every CreateSnapshot as retry-later.
-type createUnavailableProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
-}
-
-// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
-// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
-func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
-	svc, _, provider := setupProviderSnapshotImageService(t)
-	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
-	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
-
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
-}
-
-// createInUseProvider refuses every CreateSnapshot as volume in use.
-type createInUseProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
-}
-
-// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
-// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
-func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
-	svc, _, provider := setupProviderSnapshotImageService(t)
-	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
-	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
-
-	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
-
-	code, message, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok, "unregistered error: %v", err)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code)
-	assert.Contains(t, message, doc.VolumeID)
-	assert.Contains(t, message, "Detach")
-}
-
-// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
-type countingSnapshotProvider struct {
-	ebsprovider.EBSProvider
-
-	creates atomic.Int32
-}
-
-func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	p.creates.Add(1)
-	return p.EBSProvider.CreateSnapshot(ctx, req)
-}
-
-// An image of a running instance is snapshotted by the provider, which drains
-// a buffered volume itself, so CreateImage neither dials the drain socket nor
-// asks the instance's host to drain.
-func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *testing.T) {
-	const instanceID = "i-running-drain"
-	svc, _, provider := setupProviderSnapshotImageService(t)
-	_, nc := testutil.StartTestNATS(t)
-	svc.natsConn = nc
-	svc.config.DataDir = testutil.SocketTempDir(t)
-	counting := &countingSnapshotProvider{EBSProvider: provider}
-	svc.SetEBSProvider(counting)
-
-	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
-	doc.State, doc.AttachedInstance = "in-use", instanceID
-	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
-	drains := testutil.StartDrainSocket(t, svc.config.DataDir, doc.VolumeID, "OK\n")
-	var commands atomic.Int32
-	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(instanceID), func(msg *nats.Msg) {
-		commands.Add(1)
-		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	out, err := svc.CreateImageFromInstance(CreateImageParams{
-		Input:        &ec2.CreateImageInput{InstanceId: aws.String(instanceID), Name: aws.String("running-ami")},
-		RootVolumeID: doc.VolumeID,
-		IsRunning:    true,
-	}, testAccountID)
-	require.NoError(t, err)
-	require.NotNil(t, out.ImageId)
-
-	assert.Equal(t, int32(1), counting.creates.Load(), "the provider must take the snapshot")
-	assert.Zero(t, drains(), "CreateImage must not dial the drain socket")
-	assert.Zero(t, commands.Load(), "CreateImage must not ask the instance's host to drain")
-}

spinifex/domains/ec2/image/service_impl.go

@@ -38,7 +38,7 @@ type CreateImageParams struct {
 	Input         *ec2.CreateImageInput
 	RootVolumeID  string
 	SourceImageID string
-	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
+	IsRunning     bool // true = use live checkpoint (instance still running), false = use numbered checkpoint from Close
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -69,7 +69,9 @@ func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store {
 	return s.metadata
 }
 
-// NewImageServiceImpl creates a new daemon-side image service.
+// NewImageServiceImpl creates a new daemon-side image service. natsConn is used
+// to drain a running instance's volume (routed to the node hosting it) before
+// CreateImageFromInstance reads its live checkpoint.
 func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceImpl {
 	store := objectstore.NewS3ObjectStoreFromConfig(
 		cfg.Predastore.Host,
@@ -97,7 +99,8 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
 }
 
 // NewImageServiceImplWithConfig creates an image service with an explicit
-// config, object store and NATS connection (for testing).
+// config and NATS connection (for testing the running-volume drain path,
+// which needs config.DataDir/Predastore and a NATS connection to route to).
 func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn) *ImageServiceImpl {
 	return &ImageServiceImpl{
 		config:     cfg,
@@ -444,9 +447,15 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume
-	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
-		return nil, err
+	// Step 1: Snapshot root volume (live via NATS or offline from S3)
+	var snapshotErr error
+	if params.IsRunning {
+		snapshotErr = s.snapshotRunningVolume(params.RootVolumeID, snapshotID, accountID)
+	} else {
+		snapshotErr = s.snapshotStoppedVolume(params.RootVolumeID, snapshotID, accountID)
+	}
+	if snapshotErr != nil {
+		return nil, snapshotErr
 	}
 
 	// Step 2: Read source volume config for size
@@ -512,21 +521,51 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}, nil
 }
 
-// snapshotVolume snapshots the root volume through the EBS provider, whether
-// or not the instance is running. A provider that buffers writes drains the
-// volume inside its own CreateSnapshot, so no drain is issued here.
-func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
+// snapshotRunningVolume creates a snapshot of a running instance's volume by
+// draining it to the node hosting it (so any writes buffered there reach S3),
+// then reading the live checkpoint written by nbdkit on every NBD Flush. This
+// shares domains/ec2/snapshot's DrainVolume with ec2.CreateSnapshot so the two
+// live-snapshot paths cannot diverge on when a drain is required.
+func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID string) error {
 	if s.config == nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
 	if err != nil {
-		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
+	// Flush the writes still buffered by whichever node serves the volume, so
+	// the snapshot below is current rather than an arbitrarily stale one
+	// predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An
+	// attached volume that cannot be drained fails here rather than silently
+	// producing an image from a stale checkpoint.
+	if err := ec2snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil {
+		slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume")
+}
+
+// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the
+// volume is stopped/detached, so nothing is buffered on a serving node.
+// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening
+// a second one, so it handles the mounted case itself.
+func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID, accountID string) error {
+	if s.config == nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
+	if err != nil {
+		slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume")
 }
 
 // createSnapshot delegates snapshot creation for volMeta to the EBS provider.
@@ -541,7 +580,7 @@ func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetada
 	})
 	if err != nil || created == nil {
 		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
+		return errors.New(awserrors.ErrorServerInternal)
 	}
 	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
 	return nil

spinifex/domains/ec2/image/service_impl_test.go

@@ -2551,19 +2551,56 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
 
-func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
+func TestSnapshotRunningVolume_VolumeConfigMissing(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := &ImageServiceImpl{
 		config:     &config.Config{},
 		store:      store,
 		bucketName: testBucket,
 	}
-	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
+	err := svc.snapshotRunningVolume("vol-missing", "snap-r1", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotVolume_BackendInitFails(t *testing.T) {
+func TestSnapshotRunningVolume_BackendInitFails(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:   srv.URL,
+				Region: "us-east-1",
+				Bucket: testBucket,
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-running1", 10)
+
+	err := svc.snapshotRunningVolume("vol-running1", "snap-r2", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotStoppedVolume_VolumeConfigMissing(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config:     &config.Config{},
+		store:      store,
+		bucketName: testBucket,
+	}
+	err := svc.snapshotStoppedVolume("vol-missing", "snap-s1", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 	}))
@@ -2583,7 +2620,7 @@ func TestSnapshotVolume_BackendInitFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped1", 10)
 
-	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
+	err := svc.snapshotStoppedVolume("vol-stopped1", "snap-s2", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -2609,7 +2646,60 @@ func imageS3MockServer(t *testing.T) *httptest.Server {
 	}))
 }
 
-func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
+func TestSnapshotRunningVolume_EncryptionKeyLoadError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:   srv.URL,
+				Region: "us-east-1",
+				Bucket: testBucket,
+			},
+			Viperblock: config.ViperblockConfig{
+				EncryptionKeyFile: "/nonexistent-image-run-key.bin",
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-running-enckey", 10)
+
+	err := svc.snapshotRunningVolume("vol-running-enckey", "snap-run-enckey", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotRunningVolume_LoadStateFails(t *testing.T) {
+	srv := imageS3MockServer(t)
+	defer srv.Close()
+
+	store := objectstore.NewMemoryObjectStore()
+	svc := &ImageServiceImpl{
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:      srv.URL,
+				Region:    "us-east-1",
+				Bucket:    testBucket,
+				AccessKey: "test-key",
+				SecretKey: "test-secret",
+			},
+		},
+		store:      store,
+		bucketName: testBucket,
+	}
+	createTestVolumeConfig(t, store, "vol-running-lsf", 10)
+
+	err := svc.snapshotRunningVolume("vol-running-lsf", "snap-run-lsf", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -2632,12 +2722,12 @@ func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
 
-	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	err := svc.snapshotStoppedVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotVolume_LoadStateFails(t *testing.T) {
+func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
 	srv := imageS3MockServer(t)
 	defer srv.Close()
 
@@ -2657,7 +2747,7 @@ func TestSnapshotVolume_LoadStateFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
 
-	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	err := svc.snapshotStoppedVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }

spinifex/domains/ec2/instance/service.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 )
@@ -121,6 +122,15 @@ type StoppedInstanceStore interface {
 	ClaimStoppedInstance(instanceID string) (*vm.VM, error)
 }
 
+// UEFIStoreMarker records a terminated instance's UEFI variable store for
+// deletion. Implemented by uefistore.Store.
+type UEFIStoreMarker interface {
+	Exists(ctx context.Context, instanceID string) (bool, error)
+	MarkDeleting(ctx context.Context, instanceID string) (bool, error)
+}
+
+var _ UEFIStoreMarker = (*uefistore.Store)(nil)
+
 // InstanceTagWriter projects an instance record's full tag set into the
 // central tag store, and removes it on terminate. Implemented by
 // domains/ec2/tags.TagsServiceImpl.

spinifex/domains/ec2/instance/service_impl.go

@@ -234,6 +234,7 @@ type InstanceServiceImpl struct {
 	eniDeleter        ENIDeleter
 	ipReleaser        PublicIPReleaser
 	tagWriter         InstanceTagWriter
+	uefiStores        UEFIStoreMarker
 	gpuClaimer        GPUClaimer
 	amiLoader         AMIMetaLoader
 	ebsProvider       ebsprovider.EBSProvider
@@ -314,6 +315,12 @@ func (s *InstanceServiceImpl) SetTerminationDeps(vd VolumeDeleter, ed ENIDeleter
 	s.tagWriter = tw
 }
 
+// SetUEFIStores wires the store records a stopped instance's terminate marks
+// for deletion. nil leaves terminate recording no UEFI store obligation.
+func (s *InstanceServiceImpl) SetUEFIStores(m UEFIStoreMarker) {
+	s.uefiStores = m
+}
+
 // SetGPUClaimer wires the GPU passthrough claim/release dependency used by
 // StartStoppedInstance. nil disables GPU passthrough for the service.
 func (s *InstanceServiceImpl) SetGPUClaimer(g GPUClaimer) {
@@ -1237,12 +1244,7 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				"instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
 		default:
-			// A volume held elsewhere is the caller's 400, not a fault on this node.
-			if refusal := volumeHeldElsewhereError(err); refusal != nil {
-				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
-				return refusal
-			}
-			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
 	}
@@ -2843,15 +2845,40 @@ func (s *InstanceServiceImpl) RecoverInstance(ctx context.Context, instance *vm.
 	return s.launchClaimedInstance(ctx, instance, undo)
 }
 
-// volumeHeldElsewhereError converts a mount the EBS provider refused with its
-// held-elsewhere code into IncorrectState naming the holder, wrapping
-// ec2v1.ErrVolumeHeldElsewhere, and returns nil for anything else.
+// volumeExclusionMarkers are the refusals that mean another node is writing this
+// volume right now, or is still live and holds its unsealed writes. A previous
+// owner that has stopped heartbeating is opened over with a warning instead. The
+// text arrives having crossed NATS from viperblockd.
+var volumeExclusionMarkers = []string{
+	ec2v1.VolumeHeldElsewhereText,
+}
+
+// ErrVolumeHeldElsewhere reports a launch refused because another node still
+// holds one of the instance's volume leases.
+//
+// Recovery treats it differently from every other failure — it clears on the
+// lease's own clock rather than needing anything fixed — so it has to be
+// recognisable with errors.Is rather than inferred from text that has already
+// crossed NATS once.
+var ErrVolumeHeldElsewhere = errors.New("a volume is leased by another node")
+
+// volumeHeldElsewhereError converts a mount refused because another node holds
+// the volume into a client-facing error carrying that reason. Returns nil for
+// anything else. Without it the caller gets ServerInternal and is told to
+// retry, which cannot succeed while the holder is down, and the text naming the
+// holder never leaves the node's log.
 func volumeHeldElsewhereError(err error) error {
-	if !errors.Is(err, ebsprovider.ErrVolumeHeldElsewhere) {
+	if err == nil {
 		return nil
 	}
-	return fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere,
-		awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", err.Error()))
+	msg := err.Error()
+	for _, marker := range volumeExclusionMarkers {
+		if strings.Contains(msg, marker) {
+			return fmt.Errorf("%w: %w", ErrVolumeHeldElsewhere,
+				awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", msg))
+		}
+	}
+	return nil
 }
 
 // restoreClaimedStoppedInstance writes instance back to the shared
@@ -2907,6 +2934,7 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
 	s.deleteInstanceVolumes(ctx, instance, input.InstanceID)
 	s.releaseInstancePublicIP(ctx, instance, input.InstanceID)
 	s.deleteInstanceENI(ctx, instance, input.InstanceID)
+	hasUEFIStore := s.stampUEFIStoreTeardown(ctx, instance)
 
 	// Write to terminated KV FIRST so the instance is visible in DescribeInstances.
 	// If this fails the instance stays in the stopped bucket — safe to retry.
@@ -2916,6 +2944,16 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
+	// Marked only once the terminate is durable: a stopped instance whose
+	// terminate failed keeps its store. A failed mark leaves the teardown entry
+	// pending, from which the store reaper completes it.
+	if hasUEFIStore {
+		if _, err := s.uefiStores.MarkDeleting(ctx, input.InstanceID); err != nil {
+			slog.WarnContext(ctx, "TerminateStoppedInstance: failed to record UEFI variable store for deletion",
+				"instanceId", input.InstanceID, "err", err)
+		}
+	}
+
 	// Now safe to remove from stopped KV. Retry once on failure so the instance
 	// doesn't appear in both buckets.
 	if err := s.stoppedStore.DeleteStoppedInstance(input.InstanceID); err != nil {
@@ -2933,6 +2971,29 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
 	return &TerminateStoppedInstanceOutput{Status: "terminated", InstanceID: input.InstanceID}, nil
 }
 
+// stampUEFIStoreTeardown records the UEFI store teardown entry the terminated
+// record carries, and reports whether the instance has a store to mark. An
+// unreadable bucket is stamped failed, which the store reaper resolves.
+func (s *InstanceServiceImpl) stampUEFIStoreTeardown(ctx context.Context, instance *vm.VM) bool {
+	if s.uefiStores == nil {
+		return false
+	}
+	has, err := s.uefiStores.Exists(ctx, instance.ID)
+	state := vm.TeardownPending
+	switch {
+	case err != nil:
+		slog.WarnContext(ctx, "TerminateStoppedInstance: failed to read UEFI store record", "instanceId", instance.ID, "err", err)
+		state = vm.TeardownFailed
+	case !has:
+		return false
+	}
+	if instance.Teardown == nil {
+		instance.Teardown = make(map[string]string)
+	}
+	instance.Teardown[vm.TeardownUEFI] = string(state)
+	return has
+}
+
 func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instance *vm.VM, instanceID string) {
 	instance.EBSRequests.Mu.Lock()
 	defer instance.EBSRequests.Mu.Unlock()

spinifex/domains/ec2/instance/uefi_terminate_test.go

@@ -0,0 +1,129 @@
+package instance
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const uefiTerminateID = "i-0123456789abcdef0"
+
+// stoppedUEFIFixture is a stopped instance with a live store record and three
+// retained objects, the state a launch and a few stops leave.
+func stoppedUEFIFixture(t *testing.T) (*InstanceServiceImpl, *vmmock.StateStore, *uefistore.Store, *objectstore.MemoryObjectStore) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	stores := uefistore.NewStore(js)
+	require.NoError(t, stores.Open(t.Context()))
+	_, err := stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: uefiTerminateID, AccountID: "acc", State: uefistore.StateLive})
+	require.NoError(t, err)
+
+	objects := objectstore.NewMemoryObjectStore()
+	for _, k := range []string{"a", "b", "c"} {
+		_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
+			Bucket: aws.String("spinifex"), Key: aws.String(uefistore.KeyPrefix(uefiTerminateID) + k), Body: bytes.NewReader([]byte("vars")),
+		})
+		require.NoError(t, err)
+	}
+
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
+	}}
+	svc := &InstanceServiceImpl{stoppedStore: store, objectStore: objects}
+	svc.SetUEFIStores(stores)
+	return svc, store, stores, objects
+}
+
+func storeState(t *testing.T, stores *uefistore.Store) string {
+	t.Helper()
+	rec, _, err := stores.Get(t.Context(), uefiTerminateID)
+	require.NoError(t, err)
+	return rec.State
+}
+
+// TestS8_TerminateStoppedInstanceRecordsTheUEFIStoreObligationAndReturns
+// enforces ADR-0003:S8: the request deletes no store object, it records the
+// obligation on the store record and the terminated record, and returns.
+func TestS8_TerminateStoppedInstanceRecordsTheUEFIStoreObligationAndReturns(t *testing.T) {
+	svc, store, stores, objects := stoppedUEFIFixture(t)
+
+	_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
+	require.NoError(t, err)
+
+	keys, err := uefistore.ListObjects(t.Context(), objects, "spinifex", uefiTerminateID)
+	require.NoError(t, err)
+	assert.Len(t, keys, 3, "ADR-0003:S8: the request must leave the object deletion to the reaper")
+	assert.Equal(t, uefistore.StateDeleting, storeState(t, stores), "the store record carries the obligation")
+	require.NotNil(t, store.WroteTerminated[uefiTerminateID])
+	assert.Equal(t, string(vm.TeardownPending), store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI])
+}
+
+// TestS7_FailedStoppedTerminateKeepsTheUEFIStoreLive enforces ADR-0003:S7: a
+// terminate that fails before it is durable leaves the instance stopped, so
+// its store must stay live rather than be handed to the reaper.
+func TestS7_FailedStoppedTerminateKeepsTheUEFIStoreLive(t *testing.T) {
+	svc, store, stores, _ := stoppedUEFIFixture(t)
+	store.WriteTerminatedErr = errors.New("kv write boom")
+
+	_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
+	require.Error(t, err)
+
+	assert.Equal(t, uefistore.StateLive, storeState(t, stores),
+		"ADR-0003:S7: a stopped instance whose terminate failed must keep its store")
+}
+
+func TestTerminateStoppedInstance_NoUEFIStoreStampsNoEntry(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	stores := uefistore.NewStore(js)
+	require.NoError(t, stores.Open(t.Context()))
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
+	}}
+	svc := &InstanceServiceImpl{stoppedStore: store}
+	svc.SetUEFIStores(stores)
+
+	_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
+	require.NoError(t, err)
+
+	_, stamped := store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI]
+	assert.False(t, stamped)
+}
+
+type unreadableUEFIStores struct{ marked bool }
+
+func (u *unreadableUEFIStores) Exists(context.Context, string) (bool, error) {
+	return false, errors.New("kv unavailable")
+}
+
+func (u *unreadableUEFIStores) MarkDeleting(context.Context, string) (bool, error) {
+	u.marked = true
+	return false, nil
+}
+
+// An unreadable bucket does not fail the terminate; the entry is stamped
+// failed so the store reaper settles it from the terminated record.
+func TestTerminateStoppedInstance_UnreadableUEFIStoreIsStampedFailed(t *testing.T) {
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+		uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
+	}}
+	stores := &unreadableUEFIStores{}
+	svc := &InstanceServiceImpl{stoppedStore: store}
+	svc.SetUEFIStores(stores)
+
+	_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
+	require.NoError(t, err)
+
+	assert.Equal(t, string(vm.TeardownFailed), store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI])
+	assert.False(t, stores.marked)
+}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -0,0 +1,141 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// markAttempts bounds MarkDeleting's retries against concurrent writers. Each
+// lost race means another writer changed the record, and none of them can
+// return it to live, so a few rounds always settle.
+const markAttempts = 5
+
+// Exists reports whether instanceID has a store record, without decoding it,
+// so a record this node cannot read still counts as one.
+func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
+	return s.kv.Exists(ctx, instanceID)
+}
+
+// MarkDeleting records that instanceID's store is to be deleted, and drops any
+// write-back still owed. It reports false with no error when the instance has
+// no store record. Only a terminate or an operator's forget may call it.
+func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
+	for range markAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if errors.Is(err, kvstore.ErrNotFound) {
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		if rec.State == StateDeleting {
+			return true, nil
+		}
+		rec.State = StateDeleting
+		rec.WriteBack = nil
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return true, nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return false, err
+		}
+	}
+	return false, fmt.Errorf("uefistore: mark %s deleting: %w after %d attempts", instanceID, kvstore.ErrConflict, markAttempts)
+}
+
+// InstanceIDs lists the key of every store record, sorted. It decodes
+// nothing, so a caller reads each record through Get and its validation.
+func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
+	kv, err := s.kv.KV(ctx)
+	if err != nil {
+		return nil, err
+	}
+	lister, err := kv.ListKeys(ctx)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("uefistore: list %s: %w", BucketName, err)
+	}
+	defer func() { _ = lister.Stop() }()
+
+	var ids []string
+	for key := range lister.Keys() {
+		if key != kvutil.VersionKey {
+			ids = append(ids, key)
+		}
+	}
+	slices.Sort(ids)
+	return ids, nil
+}
+
+// validOwner reports whether ownerID is an instance or image ID that names
+// exactly one prefix, so a malformed ID can never widen a delete.
+func validOwner(ownerID string) bool {
+	if strings.ContainsAny(ownerID, "/*") {
+		return false
+	}
+	for _, p := range []string{"i-", "ami-"} {
+		if rest, ok := strings.CutPrefix(ownerID, p); ok && rest != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// ListObjects returns the key of every object under ownerID's prefix, across
+// every page of the listing.
+func ListObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID string) ([]string, error) {
+	if !validOwner(ownerID) {
+		return nil, fmt.Errorf("uefistore: %q is not an instance or image ID", ownerID)
+	}
+	contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(KeyPrefix(ownerID)),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
+	}
+	keys := make([]string, 0, len(contents))
+	for _, obj := range contents {
+		keys = append(keys, aws.StringValue(obj.Key))
+	}
+	return keys, nil
+}
+
+// DeleteObjects deletes every object under ownerID's prefix and reports how
+// many it removed. It succeeds only once a fresh listing finds the prefix
+// empty, so an object written during the delete is not left unaccounted for.
+func DeleteObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID string) (int, error) {
+	keys, err := ListObjects(ctx, objects, bucket, ownerID)
+	if err != nil {
+		return 0, err
+	}
+	deleted := 0
+	for _, key := range keys {
+		_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
+			return deleted, fmt.Errorf("uefistore: delete %s: %w", key, err)
+		}
+		deleted++
+	}
+	left, err := ListObjects(ctx, objects, bucket, ownerID)
+	if err != nil {
+		return deleted, err
+	}
+	if len(left) > 0 {
+		return deleted, fmt.Errorf("uefistore: %d object(s) remain under %s", len(left), KeyPrefix(ownerID))
+	}
+	return deleted, nil
+}

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -0,0 +1,213 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const bucket = "spinifex"
+
+func putObject(t *testing.T, objects objectstore.ObjectStore, key string) {
+	t.Helper()
+	_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader([]byte("vars")),
+	})
+	require.NoError(t, err)
+}
+
+func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []string {
+	t.Helper()
+	contents, _, err := objectstore.ListAll(t.Context(), objects, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(prefix)})
+	require.NoError(t, err)
+	var keys []string
+	for _, o := range contents {
+		keys = append(keys, aws.StringValue(o.Key))
+	}
+	return keys
+}
+
+// Deletion is final: a writer holding the revision of a deleting record, such
+// as a write-back retry, must not bring the store back to live.
+func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
+	s, _ := openStore(t)
+	rev, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
+	require.NoError(t, err)
+
+	revived := liveRecord()
+	revived.WriteBack = writeBack()
+	_, err = s.CompareAndSet(t.Context(), revived, rev)
+	require.ErrorIs(t, err, uefistore.ErrDeleting)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.StateDeleting, got.State)
+}
+
+// A write-back owed on a deleting store would put an object behind the delete.
+func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+	s, _ := openStore(t)
+	rev, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+	rec.WriteBack = writeBack()
+
+	_, err = s.CompareAndSet(t.Context(), rec, rev)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+}
+
+func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	has, err := s.MarkDeleting(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.True(t, has)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateDeleting, got.State)
+	assert.Nil(t, got.WriteBack)
+	assert.Equal(t, "111122223333", got.AccountID)
+
+	has, err = s.MarkDeleting(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.True(t, has, "a repeated terminate finds the obligation already recorded")
+	_, again, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, again, "a repeat writes nothing")
+}
+
+func TestMarkDeleting_NoRecordIsNoStore(t *testing.T) {
+	s, _ := openStore(t)
+	has, err := s.MarkDeleting(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.False(t, has)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound, "marking must not create a record")
+}
+
+// A record this node cannot trust is reported, never rewritten as deleting.
+func TestMarkDeleting_RefusesAnUntrustedRecord(t *testing.T) {
+	s, js := openStore(t)
+	rawPut(t, js, instanceID, `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`)
+
+	_, err := s.MarkDeleting(t.Context(), instanceID)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+}
+
+func TestExists_CountsAnUndecodableRecord(t *testing.T) {
+	s, js := openStore(t)
+	has, err := s.Exists(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.False(t, has)
+
+	rawPut(t, js, instanceID, `not json`)
+	has, err = s.Exists(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.True(t, has)
+}
+
+func TestInstanceIDs_ListsRecordsButNotTheVersionKey(t *testing.T) {
+	s, _ := openStore(t)
+	ids, err := s.InstanceIDs(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, ids)
+
+	for _, id := range []string{"i-0000000000000000b", "i-0000000000000000a"} {
+		rec := liveRecord()
+		rec.InstanceID = id
+		_, err := s.Create(t.Context(), rec)
+		require.NoError(t, err)
+	}
+	ids, err = s.InstanceIDs(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, []string{"i-0000000000000000a", "i-0000000000000000b"}, ids)
+}
+
+// The prefix carries its trailing slash, so an instance whose ID extends
+// another's keeps its objects.
+func TestDeleteObjects_RemovesOnlyTheOwnersPrefix(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	const other = instanceID + "1"
+	for _, key := range []string{"a", "b", "c"} {
+		putObject(t, objects, uefistore.KeyPrefix(instanceID)+key)
+	}
+	putObject(t, objects, uefistore.KeyPrefix(other)+"a")
+	putObject(t, objects, uefistore.KeyPrefix("ami-0123456789abcdef0")+"a")
+
+	deleted, err := uefistore.DeleteObjects(t.Context(), objects, bucket, instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, 3, deleted)
+	assert.Empty(t, keysUnder(t, objects, uefistore.KeyPrefix(instanceID)))
+	assert.Len(t, keysUnder(t, objects, uefistore.KeyPrefix(other)), 1)
+	assert.Len(t, keysUnder(t, objects, uefistore.KeyPrefix("ami-0123456789abcdef0")), 1)
+
+	deleted, err = uefistore.DeleteObjects(t.Context(), objects, bucket, instanceID)
+	require.NoError(t, err, "an empty prefix is already deleted")
+	assert.Zero(t, deleted)
+}
+
+func TestDeleteObjects_RefusesAnIDThatWidensTheDelete(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	putObject(t, objects, uefistore.KeyPrefix(instanceID)+"a")
+	for _, id := range []string{"", "i-", "ami-", "vol-0123", "i-0/../", "i-*", "x"} {
+		_, err := uefistore.DeleteObjects(t.Context(), objects, bucket, id)
+		require.Errorf(t, err, "owner %q", id)
+	}
+	assert.Len(t, keysUnder(t, objects, uefistore.ObjectRoot), 1)
+}
+
+// failingDeletes refuses every delete, as predastore does while it is down.
+type failingDeletes struct {
+	objectstore.ObjectStore
+}
+
+func (failingDeletes) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, errors.New("connection refused")
+}
+
+func TestDeleteObjects_ReportsAFailedDelete(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	putObject(t, mem, uefistore.KeyPrefix(instanceID)+"a")
+
+	_, err := uefistore.DeleteObjects(t.Context(), failingDeletes{mem}, bucket, instanceID)
+	require.Error(t, err)
+	assert.Len(t, keysUnder(t, mem, uefistore.KeyPrefix(instanceID)), 1)
+}
+
+// noopDeletes reports success without deleting, so the closing listing is
+// what notices the prefix is not empty.
+type noopDeletes struct {
+	objectstore.ObjectStore
+}
+
+func (noopDeletes) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return &s3.DeleteObjectOutput{}, nil
+}
+
+func TestDeleteObjects_FailsWhileAnObjectRemains(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	putObject(t, mem, uefistore.KeyPrefix(instanceID)+"a")
+
+	_, err := uefistore.DeleteObjects(t.Context(), noopDeletes{mem}, bucket, instanceID)
+	require.Error(t, err)
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,215 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	// ObjectRoot is the object-store prefix every store object is under.
+	ObjectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// ErrDeleting is returned for a change that would return a deleting store to
+// live. Deletion follows only a terminate or an operator's forget, and both are
+// final, so a writer that meets it drops whatever it was about to record.
+var ErrDeleting = errors.New("uefistore: store is being deleted")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return ObjectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	case rec.State == StateDeleting && rec.WriteBack != nil:
+		// A write-back retry puts an object, which would land behind the delete.
+		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// NewReader returns a Store over the existing bucket that neither creates it
+// nor migrates it, for an operator report that must not change the cluster.
+func NewReader(js jetstream.JetStream) *Store {
+	cfg := bucketConfig()
+	cfg.AttachOnly = true
+	cfg.OnOpen = nil
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, cfg)}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise. A
+// record at rev that is deleting cannot be set live again: that is ErrDeleting.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	if rec.State == StateLive {
+		// Read unvalidated: only the state at rev matters, and any other
+		// revision fails the guarded write below anyway.
+		cur, curRev, err := s.kv.Get(ctx, rec.InstanceID)
+		if err != nil && !errors.Is(err, kvstore.ErrNotFound) {
+			return 0, err
+		}
+		if err == nil && curRev == rev && cur.State == StateDeleting {
+			return 0, fmt.Errorf("%w: %s", ErrDeleting, rec.InstanceID)
+		}
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -4,19 +4,50 @@ package instance
 //would widen the handler API for a test rather than for a caller.
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"testing"
 
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// TestVolumeHeldElsewhere_NamesTheHolder covers both refusals that mean the
+// data is on another node. An operator who gets ServerInternal is told to retry
+// something that cannot succeed while that node is down, so the node name and
+// the reason have to survive the trip out of the daemon.
+func TestVolumeHeldElsewhere_NamesTheHolder(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "live holder, refused by the lease",
+			err: errors.New("failed to mount volume: volume is leased by another owner: " +
+				"held by node-b since 2026-09-02T17:04:39Z"),
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := volumeHeldElsewhereError(tc.err)
+			require.Error(t, got, "a volume held elsewhere must not fall through to ServerInternal")
+
+			code := awserrors.ValidErrorCodeFromError(got)
+			assert.NotEqual(t, awserrors.ErrorServerInternal, code,
+				"ServerInternal reads as a transient fault and invites a retry that cannot succeed")
+			assert.Contains(t, got.Error(), "node-b", "the caller has to learn where the data is")
+
+			assert.ErrorIs(t, got, ErrVolumeHeldElsewhere,
+				"recovery retries this on a shorter clock than anything else, so it has to be matchable")
+			resolved, _, ok := awserrors.ResolveErrorDetail(got)
+			require.True(t, ok, "the sentinel must not hide the code from the layer that returns it")
+			assert.Equal(t, awserrors.ErrorIncorrectState, resolved)
+		})
+	}
+}
+
 // TestVolumeHeldElsewhere_IgnoresUnrelatedFailures guards the other direction:
 // a genuine internal fault must keep reporting as one rather than being
 // relabelled as somebody else's volume.
@@ -29,76 +60,4 @@ func TestVolumeHeldElsewhere_IgnoresUnrelatedFailures(t *testing.T) {
 	// than refused, so this text is not an exclusion and must not become one.
 	assert.NoError(t, volumeHeldElsewhereError(errors.New(
 		"took over from node-b, which held unsealed writes since 2026-09-02T17:04:39Z")))
-
-	// Only the provider's code classifies a refusal, so its wording alone does not.
-	assert.NoError(t, volumeHeldElsewhereError(errors.New(
-		"failed to mount volume: "+ec2v1.VolumeHeldElsewhereText+": held by node-b")))
-}
-
-// A provider is free to word its refusal however it likes; the held-elsewhere
-// code is what says another node has the volume.
-func TestVolumeHeldElsewhere_MatchesTheProviderCodeWhateverTheWording(t *testing.T) {
-	refusal := fmt.Errorf("publish volume vol-1: %w", &ebsprovider.ProviderError{
-		Code:    ebsprovider.ErrorCodeVolumeHeldElsewhere,
-		Message: "volume vol-1 is published to node-a",
-	})
-
-	got := volumeHeldElsewhereError(refusal)
-	require.Error(t, got, "a held-elsewhere code must be classified even without viperblockd's text")
-	assert.ErrorIs(t, got, ec2v1.ErrVolumeHeldElsewhere)
-	assert.Contains(t, got.Error(), "node-a", "the caller has to learn where the data is")
-	code, _, ok := awserrors.ResolveErrorDetail(got)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code)
-}
-
-// volume_in_use is a read-only mismatch, not another node's claim, so it must
-// not be relabelled as a volume held elsewhere even though the sentinels nest.
-func TestVolumeHeldElsewhere_VolumeInUseAloneIsNotHeldElsewhere(t *testing.T) {
-	mismatch := fmt.Errorf("publish volume vol-1: %w", &ebsprovider.ProviderError{
-		Code:    ebsprovider.ErrorCodeVolumeInUse,
-		Message: "volume vol-1 is exported read-write, not read-only",
-	})
-	assert.NoError(t, volumeHeldElsewhereError(mismatch))
-}
-
-// heldElsewhereMounter refuses every mount the way the contract mounter
-// reports a provider's held-elsewhere code, in wording that is not viperblockd's.
-type heldElsewhereMounter struct{ raceVolumeMounter }
-
-func (heldElsewhereMounter) Mount(context.Context, *vm.VM) error {
-	return fmt.Errorf("publish volume vol-1: %w", &ebsprovider.ProviderError{
-		Code:    ebsprovider.ErrorCodeVolumeHeldElsewhere,
-		Message: "volume vol-1 is published to node-a",
-	})
-}
-
-// Both start paths a caller can reach must hand back the classified refusal:
-// the stopped-instance claim (ec2.start) and the live owner (ec2.cmd), which
-// the EKS reconciler tries first.
-func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	types, _ := defaultPrepareInstanceTypes()
-
-	t.Run("claimed relaunch", func(t *testing.T) {
-		mgr := vm.NewManager()
-		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: heldElsewhereMounter{}})
-		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
-
-		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-held-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
-		assert.ErrorIs(t, err, ec2v1.ErrVolumeHeldElsewhere)
-		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
-	})
-
-	t.Run("live owner", func(t *testing.T) {
-		const id = "i-held-live"
-		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
-		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: heldElsewhereMounter{}})
-		v, _ := mgr.Get(id)
-		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
-
-		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
-		assert.ErrorIs(t, err, ec2v1.ErrVolumeHeldElsewhere)
-		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
-	})
 }

spinifex/domains/ec2/snapshot/drain_test.go

@@ -2,8 +2,12 @@ package snapshot
 
 import (
 	"context"
-	"sync/atomic"
+	"encoding/json"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"net/http"
+	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -11,6 +15,8 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
@@ -18,78 +24,321 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const attachedInstanceID = "i-attached-host"
+const drainInstanceID = "i-drain-host"
 
-// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
-type countingSnapshotProvider struct {
-	ebsprovider.EBSProvider
-
-	creates atomic.Int32
-}
-
-func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	p.creates.Add(1)
-	return p.EBSProvider.CreateSnapshot(ctx, req)
-}
-
-// setupAttachedSnapshot is a viperblockd node answering CreateSnapshot for a
-// volume attached to a running instance, with a memory provider behind it.
-func setupAttachedSnapshot(t *testing.T, volumeID string) (*SnapshotServiceImpl, *countingSnapshotProvider, *nats.Conn) {
+// setupDrainService returns a snapshot service configured like a daemon that
+// answers ec2.CreateSnapshot: a reachable Predastore host (so drains are not
+// skipped as metadata-only), a data dir with no drain socket in it, and a NATS
+// connection. The Predastore stub rejects everything, so a snapshot that gets
+// past the drain fails fast rather than hanging on a dead address.
+func setupDrainService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
 	t.Helper()
 
-	_, nc := testutil.StartTestNATS(t)
-	cfg := &config.Config{
-		DataDir:    testutil.SocketTempDir(t),
-		EBS:        config.EBSConfig{Provider: config.EBSProviderViperblockd},
-		Predastore: config.PredastoreConfig{Host: "127.0.0.1:1", Bucket: "test-bucket"},
-	}
-	svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nc)
-	provider := &countingSnapshotProvider{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
-	svc.SetEBSProvider(provider)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	t.Cleanup(srv.Close)
 
+	_, nc := testutil.StartTestNATS(t)
+	store := objectstore.NewMemoryObjectStore()
+	cfg := &config.Config{
+		DataDir: testutil.SocketTempDir(t),
+		Predastore: config.PredastoreConfig{
+			Host:   srv.URL,
+			Bucket: "test-bucket",
+		},
+	}
+	return NewSnapshotServiceImplWithStore(cfg, store, nc), store, nc
+}
+
+// seedVolumeAttachment writes the control-plane state.json that decides whether
+// a snapshot must drain.
+func seedVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
+	t.Helper()
+	require.NoError(t, volumestate.Write(context.Background(), store, "test-bucket", volumeID, volumestate.Record{
+		State:            state,
+		AttachedInstance: instanceID,
+	}))
+}
+
+// drainResponder answers the per-instance command subject the way the node
+// hosting the instance does, and hands each request it received to the test.
+func drainResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
+	t.Helper()
+
+	got := make(chan *nats.Msg, 4)
+	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
+		got <- msg
+		_ = msg.Respond(reply)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	return got
+}
+
+// drainedAck is the reply of a node whose drain reached S3.
+func drainedAck(t *testing.T, volumeID string) []byte {
+	t.Helper()
+	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
+	require.NoError(t, err)
+	return data
+}
+
+// requireReturnsWithin fails the test if fn has not returned successfully by
+// limit, so a drain that blocks on a socket it should not have dialed reports
+// as a failure rather than as a slow test.
+func requireReturnsWithin(t *testing.T, limit time.Duration, fn func() error) {
+	t.Helper()
+
+	done := make(chan error, 1)
+	go func() { done <- fn() }()
+
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(limit):
+		t.Fatalf("drain did not return within %s", limit)
+	}
+}
+
+// awaitDrainCommand returns the command the hosting node received, failing the
+// test if none arrives.
+func awaitDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
+	t.Helper()
+	select {
+	case msg := <-got:
+		var command ec2v1.EC2InstanceCommand
+		require.NoError(t, json.Unmarshal(msg.Data, &command))
+		assert.Equal(t, testAccountID, msg.Header.Get(natsmsg.AccountIDHeader))
+		return command
+	case <-time.After(2 * time.Second):
+		t.Fatal("the node hosting the volume never received a drain command")
+		return ec2v1.EC2InstanceCommand{}
+	}
+}
+
+// An attached volume is drained on the node hosting its instance, which is the
+// only node with the socket — the node answering CreateSnapshot usually is not.
+func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID)
+	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached"))
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID))
+
+	command := awaitDrainCommand(t, got)
+	assert.True(t, command.Attributes.DrainVolume)
+	assert.Equal(t, drainInstanceID, command.ID)
+	require.NotNil(t, command.DrainVolumeData)
+	assert.Equal(t, "vol-attached", command.DrainVolumeData.VolumeID)
+}
+
+// No responder on the command subject means no node is running the instance,
+// which is what a stopped instance looks like: stop tears down both the plugin
+// and the subscription but deliberately leaves a boot volume attached. Failing
+// here would make every stopped instance's root volume unsnapshottable.
+func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) {
+	svc, store, _ := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID)
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID))
+}
+
+// A host that still holds the instance but reports it not running has nothing
+// to flush, so the sealed checkpoint stands. This is the host-drain stop, which
+// unmounts the volumes but keeps the subscription.
+func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-not-running", "in-use", drainInstanceID)
+	ack, err := json.Marshal(ec2v1.DrainVolumeResponse{
+		VolumeID: "vol-not-running", Status: ec2v1.DrainVolumeStatusNotRunning,
+	})
+	require.NoError(t, err)
+	got := drainResponder(t, nc, drainInstanceID, ack)
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID))
+	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
+}
+
+// A hosting node that reports the drain failed (wedged plugin, socket gone
+// mid-restart) fails the snapshot rather than falling through.
+func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-drain-err", "in-use", drainInstanceID)
+	drainResponder(t, nc, drainInstanceID,
+		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
+
+	err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "drain failed")
+}
+
+// An ack that is not the drained ack is not evidence the writes reached S3.
+func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID)
+	drainResponder(t, nc, drainInstanceID, []byte(`{}`))
+
+	err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unexpected ack")
+}
+
+// Nothing writes to an unattached volume, so the checkpoint its Close() left
+// behind is current and no drain is attempted — not even locally. Dialing the
+// socket is not a probe: it makes the plugin run a full flush, so an available
+// volume must be decided from the record without touching it.
+func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) {
+	svc, store, _ := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-available", "available", "")
+
+	// A socket that never answers: a dial would park here until the read
+	// deadline, so returning promptly is what proves none was made.
+	testutil.StartBlockingDrainSocket(t, svc.config.DataDir, "vol-available", "OK\n")
+
+	// A nil connection proves no command was issued: routing one would error.
+	svc.natsConn = nil
+
+	requireReturnsWithin(t, 2*time.Second, func() error {
+		return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID)
+	})
+}
+
+// A volume recorded as in-use but with no instance (drifted state) has no node
+// to address, so it takes the stopped path rather than failing every snapshot.
+func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) {
+	svc, store, _ := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "")
+	svc.natsConn = nil
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID))
+}
+
+// When the volume is served by this node the socket is local, so the drain
+// short-circuits without a hop. This is every single-node deployment.
+func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) {
+	svc, store, _ := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-local", "in-use", drainInstanceID)
+	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local", "OK\n")
+
+	// A nil connection proves the local ack was enough: no responder exists for
+	// drainInstanceID, so a routed command would fail.
+	svc.natsConn = nil
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID))
+}
+
+// A socket that answers without acking means this node does serve the volume
+// and the flush failed. Routing that onward would come back to this same node
+// and re-run the drain that has already failed, so it fails here instead.
+func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-local-err", "in-use", drainInstanceID)
+	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n")
+	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err"))
+
+	err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID)
+	require.Error(t, err)
+
+	assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node")
+}
+
+// state.json is the control-plane-owned attachment record; the copy inside
+// config.json is rewritten by the live NBD plugin from its stale in-memory
+// state, so a volume reading "available" there can still be attached.
+func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) {
+	svc, store, nc := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID)
+	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config"))
+
+	// "available" mirrors the stale State a live NBD plugin leaves in
+	// config.json; drainVolume must not trust it (see the test's own doc above).
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID))
+
+	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
+}
+
+// Volumes predating the state.json split have no state object, so the state
+// embedded in config.json is all there is to decide from.
+func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) {
+	svc, _, nc := setupDrainService(t)
+	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy"))
+
+	// The state/instance embedded in a legacy config.json, with no state.json to overlay it.
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID))
+
+	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
+}
+
+// Without Predastore the snapshot is metadata-only and never reads a live
+// checkpoint, so there is nothing for a drain to make current.
+func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) {
+	svc, store, _ := setupDrainService(t)
+	seedVolumeAttachment(t, store, "vol-meta-only", "in-use", drainInstanceID)
+	svc.config.Predastore.Host = ""
+	svc.natsConn = nil
+
+	require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID))
+}
+
+// putProviderVolume seeds both the control-plane document CreateSnapshot's
+// provider branch reads for attachment state, and the provider's own record,
+// mirroring what CreateVolume's provider branch leaves behind.
+func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) {
+	t.Helper()
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
+		State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a",
+		ProviderHandle: "memory://volume/" + volumeID,
+	}))
 	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
 		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024},
 		AvailabilityZone: "us-east-1a",
 	})
 	require.NoError(t, err)
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
-		State: "in-use", AttachedInstance: attachedInstanceID, AvailabilityZone: "us-east-1a",
-	}))
-	return svc, provider, nc
 }
 
-// The provider drains a buffered volume inside its own CreateSnapshot, so the
-// domain must not dial the drain socket itself: that would drain twice.
-func TestCreateSnapshot_AttachedVolumeOpensNoDrainSocket(t *testing.T) {
-	svc, provider, _ := setupAttachedSnapshot(t, "vol-attached-local")
-	drains := testutil.StartDrainSocket(t, svc.config.DataDir, "vol-attached-local", "OK\n")
+// The provider branch used to skip the drain entirely, silently snapshotting a
+// stale checkpoint of an attached, actively-written volume. It must drain the
+// same way the legacy branch does.
+func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) {
+	svc, _, nc := setupDrainService(t)
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	svc.SetEBSProvider(provider)
+	putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID)
+
+	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain"))
 
 	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-local")}, testAccountID)
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID)
 	require.NoError(t, err)
 
-	assert.Zero(t, drains(), "the domain must leave the drain to the provider")
-	assert.Equal(t, int32(1), provider.creates.Load())
+	command := awaitDrainCommand(t, got)
+	assert.True(t, command.Attributes.DrainVolume)
+	assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID)
 }
 
-// With no drain socket on this node, an attached volume is snapshotted by the
-// provider without asking the instance's host for a drain.
-func TestCreateSnapshot_AttachedVolumeSendsItsHostNoDrain(t *testing.T) {
-	svc, provider, nc := setupAttachedSnapshot(t, "vol-attached-remote")
-	var commands atomic.Int32
-	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(attachedInstanceID), func(msg *nats.Msg) {
-		commands.Add(1)
-		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
+// A provider-branch snapshot whose drain fails must fail closed rather than
+// return a snapshot of stale data: the provider must never be asked to
+// snapshot, and no snapshot metadata may be left behind.
+func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) {
+	svc, _, nc := setupDrainService(t)
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	svc.SetEBSProvider(provider)
+	putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID)
 
-	snap, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-remote")}, testAccountID)
+	got := drainResponder(t, nc, drainInstanceID,
+		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
+
+	_, err := svc.CreateSnapshot(context.Background(),
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
+
+	out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID)
 	require.NoError(t, err)
-	assert.Equal(t, "vol-attached-remote", aws.StringValue(snap.VolumeId))
-	assert.Equal(t, int32(1), provider.creates.Load())
-	assert.Zero(t, commands.Load(), "the instance's host must not be asked to drain")
+	assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind")
 }

spinifex/domains/ec2/snapshot/service_impl.go

@@ -5,7 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
+	"net"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -15,8 +18,10 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -204,12 +209,19 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
+	// Flush the writes buffered by whichever node serves the volume, so the
+	// provider reads a current checkpoint. An attached volume that cannot be
+	// drained fails here rather than silently snapshotting stale data.
+	if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
 	})
 	if err != nil || created == nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, ProviderCreateError(volumeID, err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	now := time.Now()
 	snapshotCfg := ebsmetadata.Snapshot{
@@ -247,6 +259,177 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
+const (
+	// drainDialTimeout bounds the connect to the local drain socket. The socket
+	// is either being served on this node or it is not, so a slow connect means
+	// a wedged plugin rather than a busy one.
+	drainDialTimeout = time.Second
+
+	// drainAckTimeout bounds DrainToBackend on the hosting node: it flushes the
+	// WAL and the live checkpoint to S3, so it scales with the dirty set rather
+	// than with a fixed unit of work.
+	drainAckTimeout = 30 * time.Second
+
+	// drainRequestTimeout bounds the ec2.cmd round-trip. It exceeds
+	// drainAckTimeout so a slow drain surfaces as the hosting node's error
+	// instead of an opaque NATS timeout here.
+	drainRequestTimeout = 35 * time.Second
+)
+
+// ErrNoDrainSocket reports that this node has no drain socket for the volume,
+// which is how a node learns it does not serve it. It is deliberately distinct
+// from a socket that answered badly: only the former is worth routing onward,
+// and conflating the two makes a node re-run a drain that has already failed.
+var ErrNoDrainSocket = errors.New("no drain socket on this node")
+
+// DrainVolumeSocket asks the NBD plugin serving volumeID on this node to flush
+// its WAL chunks and live checkpoint to S3, then waits for the ack. The plugin
+// creates the socket under {dataDir}/viperblock/{volumeID}/, so this only
+// succeeds on the node hosting the volume; the daemon calls it when a drain
+// command is routed to it.
+func DrainVolumeSocket(dataDir, volumeID string) error {
+	sockPath := filepath.Join(dataDir, "viperblock", volumeID, "snapshot.sock")
+	conn, err := net.DialTimeout("unix", sockPath, drainDialTimeout)
+	if err != nil {
+		return fmt.Errorf("%w: dial %s: %w", ErrNoDrainSocket, sockPath, err)
+	}
+	defer conn.Close()
+
+	_ = conn.SetDeadline(time.Now().Add(drainAckTimeout))
+	buf := make([]byte, 8)
+	n, err := conn.Read(buf)
+	if err != nil {
+		return fmt.Errorf("read drain ack for %s: %w", volumeID, err)
+	}
+	if !strings.HasPrefix(string(buf[:n]), "OK") {
+		return fmt.Errorf("drain of %s did not ack OK: %q", volumeID, string(buf[:n]))
+	}
+	return nil
+}
+
+// drainVolume flushes the volume's in-flight writes to S3 before the snapshot
+// reads the live checkpoint from there.
+func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
+	return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID)
+}
+
+// DrainVolume flushes the volume's in-flight writes to S3 before a live
+// checkpoint is read from there. Shared by ec2.CreateSnapshot and
+// ec2.CreateImage's running-instance path (domains/ec2/image) so the two
+// live-snapshot call sites cannot diverge on when a drain is required.
+//
+// The drain socket lives on the node serving the volume, but both callers are
+// queue-grouped across every node, so the node answering the request usually
+// is not that node. Whether a drain is required is therefore decided from the
+// volume's attachment record, never from whether the socket happens to be
+// local: an attached volume is being written to right now, and reading its
+// checkpoint without draining silently captures an older one.
+func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
+	// A metadata-only snapshot never reads the live checkpoint, so there is
+	// nothing for a drain to make current.
+	if cfg == nil || cfg.Predastore.Host == "" {
+		return nil
+	}
+
+	state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance)
+	if err != nil {
+		return err
+	}
+
+	// Nothing is writing to an unattached volume: the checkpoint its Close()
+	// left behind is the current one. Decide this before touching the socket —
+	// dialing it is not a probe, it makes the plugin run a full flush.
+	if state != "in-use" || instanceID == "" {
+		slog.InfoContext(ctx, "DrainVolume: volume not attached, snapshotting the checkpoint left by Close (stopped instance path)",
+			"volumeId", volumeID, "state", state, "attachedInstance", instanceID)
+		return nil
+	}
+
+	// Fast path: the volume is served by this node, so no hop is needed. This is
+	// every single-node deployment, and the node hosting the instance on a
+	// cluster.
+	localErr := DrainVolumeSocket(cfg.DataDir, volumeID)
+	if localErr == nil {
+		return nil
+	}
+
+	// A socket that answered but did not ack means this node does serve the
+	// volume and the flush itself failed. Routing that onward would land back on
+	// this same node and repeat the drain that has already failed.
+	if !errors.Is(localErr, ErrNoDrainSocket) {
+		return fmt.Errorf("drain volume %s on this node: %w", volumeID, localErr)
+	}
+
+	slog.InfoContext(ctx, "DrainVolume: volume attached, routing drain to the node hosting it",
+		"volumeId", volumeID, "instanceId", instanceID)
+	return drainOnHostNode(ctx, natsConn, volumeID, instanceID, accountID)
+}
+
+// volumeAttachment returns the volume's authoritative state and attached
+// instance. state.json is control-plane-owned and authoritative; the fallback
+// pair (passed in from the caller's own volume record) is rewritten by the
+// live NBD plugin from its stale in-memory state and is only used for volumes
+// predating the split.
+func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) {
+	rec, found, err := volumestate.Read(ctx, store, bucket, volumeID)
+	if err != nil {
+		return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err)
+	}
+	if found {
+		return rec.State, rec.AttachedInstance, nil
+	}
+	return fallbackState, fallbackInstance, nil
+}
+
+// drainOnHostNode issues the drain on the node hosting instanceID. Only that
+// node subscribes ec2.cmd.{instanceID}, so the command is self-routing and no
+// volume-to-node resolution is needed here.
+//
+// An attachment record is not proof of a live writer: stop deliberately leaves
+// a boot volume attached (runtime/daemon/vm_adapters.go's volumeMounterAdapter.Unmount)
+// while tearing down both the NBD plugin and this subscription. The two
+// not-running signals below are therefore the stopped-instance path, not a
+// failure — treating them as one would make every stopped instance's root
+// volume permanently unsnapshottable.
+func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanceID, accountID string) error {
+	command := ec2v1.EC2InstanceCommand{
+		ID:              instanceID,
+		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
+		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
+	}
+
+	resp, err := natsmsg.NATSRequest[ec2v1.DrainVolumeResponse](ctx, natsConn,
+		ec2v1.InstanceCommandSubject(instanceID), command, drainRequestTimeout, accountID)
+	if err != nil {
+		// No subscriber at all: the instance runs nowhere in the cluster, so it
+		// was stopped (stop migrates it to shared KV before unsubscribing) and
+		// its volumes were sealed by Close. Warn rather than Info because the
+		// same shape appears if the hosting node has lost NATS while its VM
+		// keeps writing, which yields a stale snapshot as it did before routing
+		// existed. A live host that answers and fails is still a hard error.
+		if errors.Is(err, nats.ErrNoResponders) {
+			slog.WarnContext(ctx, "drainVolume: no node hosts the instance, treating the volume as stopped and snapshotting its sealed checkpoint",
+				"volumeId", volumeID, "instanceId", instanceID)
+			return nil
+		}
+		return fmt.Errorf("drain volume %s on the node hosting %s: %w", volumeID, instanceID, err)
+	}
+
+	switch resp.Status {
+	case ec2v1.DrainVolumeStatusDrained:
+		return nil
+	// The host still holds the instance but it is not running: nothing is
+	// writing, so there is nothing to flush. This is a host-drain stop, which
+	// keeps the VM (and this subscription) in place after unmounting.
+	case ec2v1.DrainVolumeStatusNotRunning:
+		slog.InfoContext(ctx, "drainVolume: instance is not running on its host, snapshotting the volume's sealed checkpoint",
+			"volumeId", volumeID, "instanceId", instanceID)
+		return nil
+	default:
+		return fmt.Errorf("drain volume %s on the node hosting %s: unexpected ack %q", volumeID, instanceID, resp.Status)
+	}
+}
+
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
 var describeSnapshotsValidFilters = map[string]bool{
 	"snapshot-id": true,
@@ -557,21 +740,6 @@ func unreadableRecordError(ctx context.Context, snapshotID, accountID, kind stri
 		"Cannot confirm that snapshot '%s' is not in use: %d %s record(s) could not be read", snapshotID, len(unreadable), kind)
 }
 
-// ProviderCreateError maps a provider CreateSnapshot failure of volumeID to
-// the code a client can act on: retry-later for a provider that cannot answer
-// now, IncorrectState for one that needs the volume detached, else internal.
-func ProviderCreateError(volumeID string, err error) error {
-	switch {
-	case errors.Is(err, ebsprovider.ErrUnavailable):
-		return errors.New(awserrors.ErrorServiceUnavailable)
-	case errors.Is(err, ebsprovider.ErrVolumeInUse):
-		return awserrors.Errorf(awserrors.ErrorIncorrectState,
-			"The volume '%s' is in use. Detach it before creating a snapshot.", volumeID)
-	default:
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-}
-
 // providerDeleteError maps a provider DeleteSnapshot failure to the code a
 // client can act on. A transport failure is retryable: the delete is idempotent.
 func providerDeleteError(snapshotID string, err error) error {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1519,55 +1519,3 @@ func TestDropEmptySnapshotRefsDoesNotDropAConcurrentAdd(t *testing.T) {
 		assert.True(t, has, "iteration %d: the concurrent add was dropped by the empty-key delete", i)
 	}
 }
-
-// createFailingProvider fails only CreateSnapshot, with err.
-type createFailingProvider struct {
-	ebsprovider.EBSProvider
-
-	err error
-}
-
-func (p *createFailingProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	return nil, p.err
-}
-
-// A provider that cannot snapshot a volume while it is attached refuses it
-// as in use, which the client can act on by detaching the volume first.
-func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
-	svc, store := setupTestSnapshotService(t)
-	createTestVolume(t, svc, store, "vol-create-inuse", 10)
-	svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider,
-		err: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}})
-
-	_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-inuse")}, testAccountID)
-
-	code, message := clientErrorDetail(t, err)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code)
-	assert.Contains(t, message, "vol-create-inuse")
-	assert.Contains(t, message, "Detach")
-}
-
-// A provider that cannot answer now, such as one refusing a volume another
-// node holds, is retry-later for the client; any other failure is internal.
-func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
-	cases := []struct {
-		name     string
-		err      error
-		wantCode string
-	}{
-		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
-		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
-		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			svc, store := setupTestSnapshotService(t)
-			createTestVolume(t, svc, store, "vol-create-fail", 10)
-			svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider, err: tc.err})
-
-			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
-			require.Error(t, err)
-			assert.Equal(t, tc.wantCode, err.Error())
-		})
-	}
-}

spinifex/domains/ec2/volumestate/volumestate.go

@@ -0,0 +1,78 @@
+// Package volumestate reads and writes the control-plane-owned attachment
+// state for an EBS volume, persisted to a per-volume state.json object.
+//
+// It is kept out of config.json because config.json is rewritten by the live
+// nbdkit VB on every SaveState (clobbering any State the control plane wrote
+// there) and is a sealed object for encrypted volumes (a second writer reuses
+// the AES-GCM nonce). state.json is plaintext, viperblock never touches it, so
+// the control plane is its single writer. Readers outside the volume service
+// (the snapshot path) need the same record, which is why it lives here.
+package volumestate
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// Record is the control-plane-owned attachment state of a volume.
+type Record struct {
+	State            string    `json:"state"`
+	AttachedInstance string    `json:"attachedInstance"`
+	DeviceName       string    `json:"deviceName"`
+	AttachedAt       time.Time `json:"attachedAt"`
+}
+
+// Key is the S3 key for a volume's control-plane state object.
+func Key(volumeID string) string { return volumeID + "/state.json" }
+
+// Write persists the attachment state, replacing any previous record.
+func Write(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, rec Record) error {
+	data, err := json.Marshal(rec)
+	if err != nil {
+		return fmt.Errorf("marshal volume state: %w", err)
+	}
+	_, err = store.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(Key(volumeID)),
+		Body:   bytes.NewReader(data),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to write volume state to S3: %w", err)
+	}
+	return nil
+}
+
+// Read returns the attachment state. found=false with a nil error means the
+// object is absent (a volume predating the state.json split), in which case the
+// caller falls back to the State embedded in config.json.
+func Read(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string) (Record, bool, error) {
+	getResult, err := store.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(Key(volumeID)),
+	})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return Record{}, false, nil
+		}
+		return Record{}, false, fmt.Errorf("failed to get volume state: %w", err)
+	}
+	defer getResult.Body.Close()
+
+	body, err := io.ReadAll(getResult.Body)
+	if err != nil {
+		return Record{}, false, fmt.Errorf("failed to read volume state body: %w", err)
+	}
+	var rec Record
+	if err := json.Unmarshal(body, &rec); err != nil {
+		return Record{}, false, fmt.Errorf("failed to unmarshal volume state: %w", err)
+	}
+	return rec, true, nil
+}

spinifex/domains/eks/cluster/cluster_reconciler.go

@@ -795,7 +795,8 @@ func retryableStartRefusal(err error) bool {
 	if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorInsufficientInstanceCapacity {
 		return true
 	}
-	return errors.Is(err, ec2v1.ErrVolumeHeldElsewhere)
+	_, message, _ := awserrors.ResolveErrorDetail(err)
+	return strings.Contains(message, ec2v1.VolumeHeldElsewhereText) || strings.Contains(err.Error(), ec2v1.VolumeHeldElsewhereText)
 }
 
 // maybeReformEtcdQuorum escalates a wedged HA control plane that the in-place

spinifex/domains/eks/cluster/cluster_reconciler_start_refusal_test.go

@@ -3,11 +3,9 @@ package cluster
 import (
 	"context"
 	"errors"
-	"fmt"
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,12 +30,12 @@ func (c *refusingCPControl) StartInstance(context.Context, string) error {
 
 func (c *refusingCPControl) StopInstance(context.Context, string) error { return nil }
 
-// What the start path returns after crossing NATS: StartInstances wraps the
-// sentinel back on when the owner's reply carries the held-elsewhere header.
+// What the start path returns after crossing NATS: the code and the message the
+// owning node sent, with no sentinel left to match on.
 var (
 	errStartCapacity = errors.New(awserrors.ErrorInsufficientInstanceCapacity)
-	errStartHeld     = fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere,
-		awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", "volume vol-1 is published to node-a"))
+	errStartHeld     = awserrors.Errorf(awserrors.ErrorIncorrectState, "%s",
+		"volume is leased by another owner: held by node-a since 2026-10-30T01:00:00Z")
 )
 
 // A refusal that clears by itself must not use up the restart budget, or the
@@ -74,30 +72,18 @@ func TestMaybeRecoverCP_RetryableStartRefusalIsRetriedNotReplaced(t *testing.T)
 	}
 }
 
-// The cap still bounds the faults nothing says will clear. IncorrectState
-// without the held-elsewhere sentinel is one, whatever its message says.
+// The cap still bounds the faults nothing says will clear.
 func TestMaybeRecoverCP_UnknownStartFailureStillCountsAgainstTheBudget(t *testing.T) {
 	t.Parallel()
-	for name, refusal := range map[string]error{
-		"server internal": errors.New(awserrors.ErrorServerInternal),
-		"unrelated incorrect state": awserrors.Errorf(awserrors.ErrorIncorrectState, "%s",
-			"instance i-cp is not in a state from which it can be started"),
-		"lease text without the sentinel": awserrors.Errorf(awserrors.ErrorIncorrectState, "%s",
-			ec2v1.VolumeHeldElsewhereText+": held by node-a since 2026-10-30T01:00:00Z"),
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			cp := &refusingCPControl{err: refusal}
-			r := newRecoveryReconciler(t, cp)
-			r.degradedSince = time.Now().Add(-10 * time.Minute)
+	cp := &refusingCPControl{err: errors.New(awserrors.ErrorServerInternal)}
+	r := newRecoveryReconciler(t, cp)
+	r.degradedSince = time.Now().Add(-10 * time.Minute)
 
-			for range r.maxRestartAttempts + 2 {
-				r.lastRestartAt = time.Time{}
-				r.maybeRecoverControlPlane(t.Context(), metaWithCP("i-cp"), "stale")
-			}
-
-			require.Equal(t, r.maxRestartAttempts, cp.startCalls, "stops at the attempt cap")
-			assert.Equal(t, r.maxRestartAttempts, r.restartAttempts)
-		})
+	for range r.maxRestartAttempts + 2 {
+		r.lastRestartAt = time.Time{}
+		r.maybeRecoverControlPlane(t.Context(), metaWithCP("i-cp"), "stale")
 	}
+
+	require.Equal(t, r.maxRestartAttempts, cp.startCalls, "stops at the attempt cap")
+	assert.Equal(t, r.maxRestartAttempts, r.restartAttempts)
 }

spinifex/foundation/messaging/nats/nats.go

@@ -238,10 +238,6 @@ const AccountIDHeader = "X-Account-ID"
 // forwarder must relay it instead of starting the instance elsewhere.
 const UnsealedWritesHeader = "X-Unsealed-Writes"
 
-// VolumeHeldElsewhereHeader marks a start reply refused because another node
-// holds one of the instance's volumes, which the AWS error body cannot say.
-const VolumeHeldElsewhereHeader = "X-Volume-Held-Elsewhere"
-
 // PrincipalARNHeader carries the caller's resolved IAM principal ARN from gateway to daemon handlers.
 const PrincipalARNHeader = "X-Principal-ARN"
 

spinifex/foundation/messaging/nats/nats_trace.go

@@ -48,10 +48,10 @@ func ExtractTraceContext(ctx context.Context, msg *nats.Msg) context.Context {
 // front door to whichever service ends up doing the work.
 const AttrAccountID = "aws.account_id"
 
-// AccountAttrs returns the account attribute for accountID, or nothing. Work
+// accountAttrs returns the account attribute for accountID, or nothing. Work
 // with no caller — a sweep, a reconciler, a node-to-node probe — is left
 // unattributed rather than being credited to whoever happens to be admin.
-func AccountAttrs(accountID string) []attribute.KeyValue {
+func accountAttrs(accountID string) []attribute.KeyValue {
 	if accountID == "" {
 		return nil
 	}
@@ -66,7 +66,7 @@ func StartConsumerSpan(msg *nats.Msg) (context.Context, trace.Span) {
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", msg.Subject),
-	}, AccountAttrs(AccountIDFromMsg(msg))...)
+	}, accountAttrs(AccountIDFromMsg(msg))...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+msg.Subject,
 		trace.WithSpanKind(trace.SpanKindConsumer),
@@ -78,7 +78,7 @@ func startProducerSpan(ctx context.Context, subject, accountID string) (context.
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", subject),
-	}, AccountAttrs(accountID)...)
+	}, accountAttrs(accountID)...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+subject,
 		trace.WithSpanKind(trace.SpanKindClient),

spinifex/operator/admin/orphan_uefi_stores_test.go

@@ -0,0 +1,173 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type staticInstanceIndex struct {
+	ids map[string]bool
+	err error
+}
+
+func (s staticInstanceIndex) InstanceIDs(context.Context) (map[string]bool, error) {
+	return s.ids, s.err
+}
+
+type uefiOrphanFixture struct {
+	objects  *objectstore.MemoryObjectStore
+	metadata *ebsmetadata.Store
+	stores   *uefistore.Store
+	index    staticInstanceIndex
+}
+
+func newUEFIOrphanFixture(t *testing.T) *uefiOrphanFixture {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	stores := uefistore.NewStore(js)
+	require.NoError(t, stores.Open(t.Context()))
+	objects := objectstore.NewMemoryObjectStore()
+	return &uefiOrphanFixture{
+		objects:  objects,
+		metadata: ebsmetadata.NewStore(objects, orphanTestBucket),
+		stores:   stores,
+		index:    staticInstanceIndex{ids: map[string]bool{}},
+	}
+}
+
+func (f *uefiOrphanFixture) putStoreObject(t *testing.T, owner string) {
+	t.Helper()
+	_, err := f.objects.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(orphanTestBucket),
+		Key:    aws.String(uefistore.KeyPrefix(owner) + "0f1e2d3c"),
+		Body:   bytes.NewReader([]byte("vars")),
+	})
+	require.NoError(t, err)
+}
+
+func (f *uefiOrphanFixture) record(t *testing.T, id string, deleting bool) {
+	t.Helper()
+	_, err := f.stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: id, AccountID: "111122223333", State: uefistore.StateLive})
+	require.NoError(t, err)
+	if deleting {
+		_, err = f.stores.MarkDeleting(t.Context(), id)
+		require.NoError(t, err)
+	}
+}
+
+func (f *uefiOrphanFixture) scan(t *testing.T) (*OrphanUEFIStores, error) {
+	t.Helper()
+	return FindOrphanUEFIStores(t.Context(), f.objects, orphanTestBucket, f.metadata, f.stores, f.index)
+}
+
+// seedEveryCase plants one store of each shape the report must tell apart.
+func (f *uefiOrphanFixture) seedEveryCase(t *testing.T) {
+	t.Helper()
+	// Owned: an instance record, a store record, and a registered image.
+	f.index.ids["i-00000000000000live"] = true
+	f.record(t, "i-00000000000000live", false)
+	f.putStoreObject(t, "i-00000000000000live")
+	f.record(t, "i-000000000recordonly", false)
+	f.putStoreObject(t, "i-000000000recordonly")
+	f.record(t, "i-0000000000deleting", true)
+	f.putStoreObject(t, "i-0000000000deleting")
+	require.NoError(t, f.metadata.PutAMI(t.Context(), ebsmetadata.AMI{ImageID: "ami-0000000000000live", Name: "img", State: "available"}))
+	f.putStoreObject(t, "ami-0000000000000live")
+
+	// Unowned.
+	f.putStoreObject(t, "i-00000000000noowner")
+	f.putStoreObject(t, "ami-000000000noimage")
+	f.putStoreObject(t, "vol-00000000000other")
+}
+
+func TestFindOrphanUEFIStores_ListsEachUnownedShape(t *testing.T) {
+	f := newUEFIOrphanFixture(t)
+	f.seedEveryCase(t)
+
+	orphans, err := f.scan(t)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"uefi/i-00000000000noowner/"}, orphans.InstancePrefixes,
+		"only a prefix with neither an instance record nor a store record is unowned")
+	assert.Equal(t, []string{"uefi/ami-000000000noimage/"}, orphans.ImagePrefixes)
+	assert.Equal(t, []string{"uefi/vol-00000000000other/"}, orphans.OtherPrefixes)
+	assert.Equal(t, []OrphanUEFIStoreRecord{{InstanceID: "i-000000000recordonly", AccountID: "111122223333", State: uefistore.StateLive}},
+		orphans.Records, "a deleting record is already on its way out, and the reaper owns it")
+	assert.False(t, orphans.Empty())
+}
+
+// TestS7_UEFIOrphanReportDeletesNothing enforces ADR-0003:S7 for the report:
+// an absent instance record is not a terminate, so a scan that lists every
+// unowned store leaves every object and every record as it found them.
+func TestS7_UEFIOrphanReportDeletesNothing(t *testing.T) {
+	f := newUEFIOrphanFixture(t)
+	f.seedEveryCase(t)
+	objectsBefore := f.objects.Count()
+	ids, err := f.stores.InstanceIDs(t.Context())
+	require.NoError(t, err)
+	revs := map[string]uint64{}
+	for _, id := range ids {
+		_, rev, err := f.stores.Get(t.Context(), id)
+		require.NoError(t, err)
+		revs[id] = rev
+	}
+
+	_, err = f.scan(t)
+	require.NoError(t, err)
+
+	assert.Equal(t, objectsBefore, f.objects.Count(), "ADR-0003:S7: the orphan report must delete no object")
+	for id, rev := range revs {
+		rec, got, err := f.stores.Get(t.Context(), id)
+		require.NoError(t, err, "ADR-0003:S7: the orphan report removed the store record %s", id)
+		assert.Equalf(t, rev, got, "ADR-0003:S7: the orphan report changed the store record %s", id)
+		assert.NotEmpty(t, rec.State)
+	}
+}
+
+func TestFindOrphanUEFIStores_EmptyClusterReportsNothing(t *testing.T) {
+	f := newUEFIOrphanFixture(t)
+	orphans, err := f.scan(t)
+	require.NoError(t, err)
+	assert.True(t, orphans.Empty())
+}
+
+// Reading an unreadable record or a failed index as absent would point an
+// operator at live data, so either aborts the scan.
+func TestFindOrphanUEFIStores_UnreadableInputsAbortTheScan(t *testing.T) {
+	t.Run("store record", func(t *testing.T) {
+		_, _, js := testutil.StartTestJetStream(t)
+		stores := uefistore.NewStore(js)
+		require.NoError(t, stores.Open(t.Context()))
+		kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+		require.NoError(t, err)
+		_, err = kv.Put(t.Context(), "i-0123456789abcdef0", []byte(`{"schema_version":9,"instance_id":"i-0123456789abcdef0","state":"live"}`))
+		require.NoError(t, err)
+		objects := objectstore.NewMemoryObjectStore()
+
+		_, err = FindOrphanUEFIStores(t.Context(), objects, orphanTestBucket, ebsmetadata.NewStore(objects, orphanTestBucket), stores, staticInstanceIndex{ids: map[string]bool{}})
+		require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	})
+	t.Run("instance index", func(t *testing.T) {
+		f := newUEFIOrphanFixture(t)
+		f.index.err = errors.New("kv unavailable")
+		_, err := f.scan(t)
+		require.Error(t, err)
+	})
+	t.Run("AMI document", func(t *testing.T) {
+		f := newUEFIOrphanFixture(t)
+		putRawAMIDocument(t, f.objects, "ami-corrupted000", []byte("{not json"))
+		_, err := f.scan(t)
+		require.Error(t, err)
+	})
+}

spinifex/operator/admin/orphan_volumes.go

@@ -2,12 +2,18 @@ package admin
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 )
 
 // efiVolumeSuffix marks a volume derived from another one. An EFI variable
@@ -115,3 +121,119 @@ func listProviderVolumes(ctx context.Context, provider ebsprovider.EBSProvider)
 		token = resp.NextToken
 	}
 }
+
+// UEFIStoreRecords is the read side of the UEFI store-record bucket.
+type UEFIStoreRecords interface {
+	InstanceIDs(ctx context.Context) ([]string, error)
+	Get(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// InstanceIndex reports every instance ID the control plane holds a record
+// for, live or terminated. A record it cannot decode still counts.
+type InstanceIndex interface {
+	InstanceIDs(ctx context.Context) (map[string]bool, error)
+}
+
+// OrphanUEFIStoreRecord is a store record that is not deleting and whose
+// instance has no record, so nothing will ever delete its objects.
+type OrphanUEFIStoreRecord struct {
+	InstanceID string
+	AccountID  string
+	State      string
+}
+
+// OrphanUEFIStores is what the object store and the store records hold for
+// UEFI variable stores that nothing owns.
+type OrphanUEFIStores struct {
+	// InstancePrefixes are uefi/i-*/ prefixes with no instance record and no
+	// store record.
+	InstancePrefixes []string
+	// ImagePrefixes are uefi/ami-*/ prefixes with no AMI document.
+	ImagePrefixes []string
+	// OtherPrefixes are prefixes named for neither an instance nor an image,
+	// which no owner can ever claim.
+	OtherPrefixes []string
+	Records       []OrphanUEFIStoreRecord
+}
+
+// Empty reports whether the scan found nothing.
+func (o *OrphanUEFIStores) Empty() bool {
+	return len(o.InstancePrefixes) == 0 && len(o.ImagePrefixes) == 0 && len(o.OtherPrefixes) == 0 && len(o.Records) == 0
+}
+
+// FindOrphanUEFIStores reports UEFI variable stores that nothing owns. Like
+// FindOrphanVolumes it never deletes: an absent instance record is not proof
+// of a terminate, so settling each one is an operator's decision.
+//
+// Every read is strict. A record or document this cannot read aborts the scan
+// rather than reading as absent, which would point an operator at live data.
+func FindOrphanUEFIStores(ctx context.Context, objects objectstore.ObjectStore, bucket string,
+	metadata *ebsmetadata.Store, stores UEFIStoreRecords, instances InstanceIndex) (*OrphanUEFIStores, error) {
+	if objects == nil || bucket == "" || metadata == nil || stores == nil || instances == nil {
+		return nil, errors.New("uefi orphan scan: object store, bucket, metadata, store records and instance index are all required")
+	}
+
+	known, err := instances.InstanceIDs(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("uefi orphan scan: list instance records: %w", err)
+	}
+	images, err := metadata.ListAMIsStrict(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("uefi orphan scan: list control-plane images: %w", err)
+	}
+	imageIDs := make(map[string]bool, len(images))
+	for _, image := range images {
+		imageIDs[image.ImageID] = true
+	}
+
+	out := &OrphanUEFIStores{}
+	ids, err := stores.InstanceIDs(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("uefi orphan scan: list store records: %w", err)
+	}
+	recorded := make(map[string]bool, len(ids))
+	for _, id := range ids {
+		recorded[id] = true
+		rec, _, err := stores.Get(ctx, id)
+		if errors.Is(err, kvstore.ErrNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("uefi orphan scan: read store record %s: %w", id, err)
+		}
+		if rec.State != uefistore.StateDeleting && !known[id] {
+			out.Records = append(out.Records, OrphanUEFIStoreRecord{InstanceID: id, AccountID: rec.AccountID, State: rec.State})
+		}
+	}
+
+	_, prefixes, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket:    aws.String(bucket),
+		Prefix:    aws.String(uefistore.ObjectRoot),
+		Delimiter: aws.String("/"),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("uefi orphan scan: list store objects: %w", err)
+	}
+	for _, p := range prefixes {
+		prefix := aws.StringValue(p.Prefix)
+		owner := strings.TrimSuffix(strings.TrimPrefix(prefix, uefistore.ObjectRoot), "/")
+		switch {
+		case strings.HasPrefix(owner, "i-"):
+			if !known[owner] && !recorded[owner] {
+				out.InstancePrefixes = append(out.InstancePrefixes, prefix)
+			}
+		case strings.HasPrefix(owner, "ami-"):
+			if !imageIDs[owner] {
+				out.ImagePrefixes = append(out.ImagePrefixes, prefix)
+			}
+		default:
+			out.OtherPrefixes = append(out.OtherPrefixes, prefix)
+		}
+	}
+	sort.Strings(out.InstancePrefixes)
+	sort.Strings(out.ImagePrefixes)
+	sort.Strings(out.OtherPrefixes)
+	return out, nil
+}

spinifex/operator/cli/admin.go

@@ -32,6 +32,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
@@ -45,6 +46,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/viperblock/dirty"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
 	"github.com/mulgadc/spinifex/spinifex/runtime/formation"
 	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
 	"github.com/mulgadc/spinifex/spinifex/runtime/host/dns"
@@ -952,6 +954,13 @@ func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
 		fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
 		os.Exit(1)
 	}
+	// The variable stores are reported first, so the volume report's early
+	// return on a clean scan cannot hide them.
+	if err := reportUEFIStoreOrphans(context.Background(), nc, store, node.Predastore.Bucket); err != nil {
+		fmt.Fprintln(os.Stderr, "UEFI store orphan scan failed:", err)
+		os.Exit(1)
+	}
+	fmt.Println()
 	if len(orphans) == 0 {
 		fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
 		return
@@ -970,6 +979,53 @@ func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
 	fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
 }
 
+// reportUEFIStoreOrphans prints the UEFI variable stores that nothing owns.
+// It reads only, and attaches to the buckets rather than creating them.
+func reportUEFIStoreOrphans(ctx context.Context, nc *nats.Conn, store objectstore.ObjectStore, bucket string) error {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return fmt.Errorf("jetstream: %w", err)
+	}
+	orphans, err := admin.FindOrphanUEFIStores(ctx, store, bucket, ebsmetadata.NewStore(store, bucket),
+		uefistore.NewReader(js), kvInstanceIndex{js: js})
+	if err != nil {
+		return err
+	}
+	if orphans.Empty() {
+		fmt.Println("No orphaned UEFI variable stores.")
+		return nil
+	}
+	fmt.Println("Orphaned UEFI variable stores:")
+	for _, p := range orphans.InstancePrefixes {
+		fmt.Printf("  %s  (no instance record and no store record)\n", p)
+	}
+	for _, p := range orphans.ImagePrefixes {
+		fmt.Printf("  %s  (no AMI document)\n", p)
+	}
+	for _, p := range orphans.OtherPrefixes {
+		fmt.Printf("  %s  (names neither an instance nor an image)\n", p)
+	}
+	for _, r := range orphans.Records {
+		fmt.Printf("  store record %s  (account %s, %s, no instance record)\n", r.InstanceID, r.AccountID, r.State)
+	}
+	fmt.Println()
+	fmt.Println("Nothing was deleted. An absent instance record is not proof of a terminate,")
+	fmt.Println("so settling each of these is an operator decision.")
+	return nil
+}
+
+// kvInstanceIndex is the instance domain's index of every instance ID it holds
+// a record for.
+type kvInstanceIndex struct {
+	js jetstream.JetStream
+}
+
+var _ admin.InstanceIndex = kvInstanceIndex{}
+
+func (k kvInstanceIndex) InstanceIDs(ctx context.Context) (map[string]bool, error) {
+	return daemon.KnownInstanceIDs(ctx, k.js)
+}
+
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 	imageID, _ := cmd.Flags().GetString("image-id")
 	force, _ := cmd.Flags().GetBool("force")

spinifex/operator/cli/admin_uefi_orphans_test.go

@@ -0,0 +1,90 @@
+package cli
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
+)
+
+func putKeys(t *testing.T, js jetstream.JetStream, bucket string, keys ...string) {
+	t.Helper()
+	kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket})
+	require.NoError(t, err)
+	for _, key := range keys {
+		_, err := kv.Put(t.Context(), key, []byte("not decoded"))
+		require.NoError(t, err)
+	}
+}
+
+// Every key space an instance record has lived in counts, and the node
+// blobs and presence markers name nodes rather than instances.
+func TestKVInstanceIndex_CountsEveryInstanceKeySpace(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	putKeys(t, js, daemon.InstanceStateBucket, "i.i-running", "instance.i-frozenstopped", "node.node-1", "nodepresence.node-1")
+	putKeys(t, js, daemon.TerminatedInstanceBucket, "i.i-terminated", "terminated.i-frozenterminated")
+
+	ids, err := kvInstanceIndex{js: js}.InstanceIDs(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, map[string]bool{
+		"i-running": true, "i-frozenstopped": true, "i-terminated": true, "i-frozenterminated": true,
+	}, ids)
+}
+
+// The report attaches to the buckets rather than creating them, so a cluster
+// missing one fails the scan instead of reporting every store as unowned.
+func TestReportUEFIStoreOrphans_FailsWhenABucketIsMissing(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	objects := objectstore.NewMemoryObjectStore()
+
+	require.Error(t, reportUEFIStoreOrphans(t.Context(), nc, objects, "spinifex"))
+}
+
+func TestReportUEFIStoreOrphans_PrintsAndDeletesNothing(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	putKeys(t, js, daemon.InstanceStateBucket)
+	putKeys(t, js, daemon.TerminatedInstanceBucket)
+	stores := uefistore.NewStore(js)
+	require.NoError(t, stores.Open(t.Context()))
+	_, err := stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: "i-0000000000recorded", AccountID: "111122223333", State: uefistore.StateLive})
+	require.NoError(t, err)
+	objects := objectstore.NewMemoryObjectStore()
+	for _, owner := range []string{"i-00000000000noowner", "ami-000000000noimage", "vol-00000000000other"} {
+		_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
+			Bucket: aws.String("spinifex"), Key: aws.String(uefistore.KeyPrefix(owner) + "a"), Body: bytes.NewReader([]byte("vars")),
+		})
+		require.NoError(t, err)
+	}
+
+	out := captureStdout(t, func() {
+		require.NoError(t, reportUEFIStoreOrphans(t.Context(), nc, objects, "spinifex"))
+	})
+
+	assert.Contains(t, out, "uefi/i-00000000000noowner/")
+	assert.Contains(t, out, "uefi/ami-000000000noimage/")
+	assert.Contains(t, out, "uefi/vol-00000000000other/")
+	assert.Contains(t, out, "store record i-0000000000recorded")
+	assert.Contains(t, out, "Nothing was deleted")
+	assert.Equal(t, 3, objects.Count())
+}
+
+func TestReportUEFIStoreOrphans_CleanCluster(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	putKeys(t, js, daemon.InstanceStateBucket)
+	putKeys(t, js, daemon.TerminatedInstanceBucket)
+	require.NoError(t, uefistore.NewStore(js).Open(t.Context()))
+
+	out := captureStdout(t, func() {
+		require.NoError(t, reportUEFIStoreOrphans(t.Context(), nc, objectstore.NewMemoryObjectStore(), "spinifex"))
+	})
+	assert.Contains(t, out, "No orphaned UEFI variable stores.")
+}

spinifex/operator/upgrade/VERSIONS.md

@@ -23,6 +23,7 @@ Single source of truth for the current schema version of every config file Spini
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
 | `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
+| `spinifex-uefi-stores` | `1` | `uefistore.BucketVersion` |
 
 ## Registered migrations
 

spinifex/providers/ebs/conformance/conformance.go

@@ -145,11 +145,6 @@ type SuiteConfig struct {
 	// the provider, standing in for a crash. Nil skips the dead-export case,
 	// for a provider whose export is not a separate process.
 	KillExport func(t *testing.T, published *ebsprovider.PublishedVolume)
-
-	// QueueOnOtherNode returns a client answered only on the queue subjects, by
-	// a node other than NodeID: what a request meets when the volume's owner
-	// does not answer. Nil skips the cases that need it.
-	QueueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
 }
 
 func (c SuiteConfig) node() string {
@@ -215,9 +210,9 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 	t.Run("ContractVersions", func(t *testing.T) {
 		t.Run("listed versions are ascending and include the legacy version", func(t *testing.T) {
-			require.NotEmpty(t, contractVersions, "a provider must list the contract versions it serves")
+			require.NotEmpty(t, contractVersions, "a caller cannot choose a version for a provider that lists none")
 			assert.Contains(t, contractVersions, ebsprovider.LegacySchemaVersion,
-				"callers send version-neutral requests at the legacy version, so a provider must serve it")
+				"dropping the legacy version strands every caller still in an N/N-1 rollout")
 			for i := 1; i < len(contractVersions); i++ {
 				assert.Lessf(t, contractVersions[i-1], contractVersions[i], "contract versions %v are not strictly ascending", contractVersions)
 			}
@@ -635,39 +630,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			_, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
-
-		// Only the owner can drain what it buffers, so a node answering in its
-		// place must refuse rather than snapshot a checkpoint the owner has not
-		// flushed. The owner's own snapshot then succeeds.
-		t.Run("refused on the queue subject while another node holds the volume", func(t *testing.T) {
-			if !capabilities.OwnerRouting || !capabilities.Exclusion.SingleWriter() {
-				t.Skip("no owner routing behind cluster exclusion, so no node answers in place of an owner")
-			}
-			if cfg.QueueOnOtherNode == nil {
-				t.Skip("no queue-only client on another node configured")
-			}
-			provider := newProvider(t)
-			queue := newTrackingProvider(t, cfg.QueueOnOtherNode(t))
-			ctx := context.Background()
-			volumeID := cfg.id("vol-snap-held")
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
-			require.NoError(t, err)
-
-			for _, version := range contractVersions {
-				snapshotID := cfg.id(fmt.Sprintf("snap-held-queue-v%d", version))
-				_, err = queue.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, SnapshotID: snapshotID, VolumeID: volumeID})
-				require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "v%d: a node not holding the volume must refuse as retry-later", version)
-				if capabilities.SnapshotEnumeration {
-					assert.NotContains(t, drainSnapshotIDs(t, provider, 0), snapshotID)
-				}
-			}
-
-			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-held-owner"), VolumeID: volumeID})
-			require.NoError(t, err, "the holder answers for its own volume")
-			assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
-		})
 	})
 
 	t.Run("DeleteSnapshot", func(t *testing.T) {

spinifex/providers/ebs/nats.go

@@ -35,9 +35,8 @@ const (
 const defaultRequestTimeout = 30 * time.Second
 
 // dataMovementTimeout bounds the subjects whose duration is a function of the
-// volume rather than of the control plane. It must exceed viperblockd's 120s
-// kill grace plus a seal allowance, or an unpublish that uses the whole grace
-// could never be seen to succeed.
+// volume rather than of the control plane. Matches unmountSealTimeout, this
+// codebase's existing budget for an EBS operation the caller cannot bound.
 const dataMovementTimeout = 180 * time.Second
 
 // dataMovementVerbs names every operation that references or moves real data,
@@ -89,10 +88,11 @@ func SnapshotCompletionSubject(snapshotID string) (string, error) {
 
 // Last tokens of the per-node subjects: ebs.provider.v1.<node>.<verb>.
 const (
-	publishVerb     = "mount"
-	unpublishVerb   = "unmount"
-	unsealedVerb    = "unsealed"
-	fenceNoticeVerb = "fenced"
+	publishVerb          = "mount"
+	unpublishVerb        = "unmount"
+	unsealedVerb         = "unsealed"
+	nodeCapabilitiesVerb = "capabilities"
+	fenceNoticeVerb      = "fenced"
 )
 
 // nodeSubjectWildcard is the subject matching verb on every node, for a server
@@ -101,9 +101,10 @@ func nodeSubjectWildcard(verb string) string { return subjectPrefix + "*." + ver
 
 // Wildcards over every node for each per-node subject.
 var (
-	PublishSubjectWildcard        = nodeSubjectWildcard(publishVerb)
-	UnpublishSubjectWildcard      = nodeSubjectWildcard(unpublishVerb)
-	UnsealedWritesSubjectWildcard = nodeSubjectWildcard(unsealedVerb)
+	PublishSubjectWildcard          = nodeSubjectWildcard(publishVerb)
+	UnpublishSubjectWildcard        = nodeSubjectWildcard(unpublishVerb)
+	UnsealedWritesSubjectWildcard   = nodeSubjectWildcard(unsealedVerb)
+	NodeCapabilitiesSubjectWildcard = nodeSubjectWildcard(nodeCapabilitiesVerb)
 )
 
 func nodeSubject(nodeID, verb string) (string, error) {
@@ -132,6 +133,13 @@ func UnsealedWritesSubject(nodeID string) (string, error) {
 	return nodeSubject(nodeID, unsealedVerb)
 }
 
+// NodeCapabilitiesSubject returns nodeID's per-node GetCapabilities subject
+// (ebs.provider.v1.<node>.capabilities). Unlike CapabilitiesSubject, which
+// any node's provider answers, only nodeID's provider answers it.
+func NodeCapabilitiesSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, nodeCapabilitiesVerb)
+}
+
 // FenceNoticeSubject returns nodeID's fence-notice subject
 // (ebs.provider.v1.<node>.fenced), on which a provider that revoked a live
 // export there sends a FenceNotice for that node's daemon to acknowledge.
@@ -210,38 +218,6 @@ func validateSubjectToken(value string) error {
 	return nil
 }
 
-// A publish answered without an error but without a usable export. The
-// provider may have started the export anyway, so a caller must not assume
-// nothing is left to tear down.
-var (
-	ErrNoPublishedVolume = errors.New("returned no published volume")
-	ErrInvalidExport     = errors.New("returned an invalid export")
-)
-
-// ErrUnreadableReply is a reply that arrived but could not be decoded, or
-// carried a schema version this build does not serve. The provider did
-// receive the request, so whatever it asked for may have happened.
-var ErrUnreadableReply = errors.New("provider reply could not be read")
-
-// AccountIDHeader carries the account a contract request acts for. It is the
-// header the control plane's other NATS hops use, so one name attributes a
-// request from the gateway through to the provider.
-const AccountIDHeader = "X-Account-ID"
-
-type accountIDKey struct{}
-
-// WithAccountID returns ctx carrying the account the contract requests made
-// under it act for. An empty accountID leaves them unattributed.
-func WithAccountID(ctx context.Context, accountID string) context.Context {
-	return context.WithValue(ctx, accountIDKey{}, accountID)
-}
-
-// AccountIDFromContext returns the account WithAccountID put on ctx, or "".
-func AccountIDFromContext(ctx context.Context) string {
-	accountID, _ := ctx.Value(accountIDKey{}).(string)
-	return accountID
-}
-
 // NATSProvider drives a provider daemon through the versioned ebs.* contract.
 type NATSProvider struct {
 	conn           *nats.Conn
@@ -263,11 +239,26 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	return p.getCapabilities(ctx, CapabilitiesSubject, req)
+}
+
+// GetNodeCapabilities asks nodeID's own provider what it supports, over
+// NodeCapabilitiesSubject. A provider that predates that subject gets no
+// responder, which a caller must count as no evidence.
+func (p *NATSProvider) GetNodeCapabilities(ctx context.Context, nodeID string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	subject, err := NodeCapabilitiesSubject(nodeID)
+	if err != nil {
+		return nil, err
+	}
+	return p.getCapabilities(ctx, subject, req)
+}
+
+func (p *NATSProvider) getCapabilities(ctx context.Context, subject string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
-	if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil {
+	if err := p.request(ctx, subject, req, &response); err != nil {
 		return nil, err
 	}
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
@@ -382,7 +373,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	if err != nil {
 		return nil, err
 	}
-	if !p.connected() {
+	if p.conn == nil || !p.conn.IsConnected() {
 		return nil, nats.ErrConnectionClosed
 	}
 
@@ -422,7 +413,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	RecordResponseError(waitSpan, msg.Data)
 	var completed CreateSnapshotResponse
 	if err := json.Unmarshal(msg.Data, &completed); err != nil {
-		return nil, fmt.Errorf("decode snapshot completion: %w: %w", ErrUnreadableReply, err)
+		return nil, fmt.Errorf("decode snapshot completion: %w", err)
 	}
 	if err := responseError(completed.SchemaVersion, completed.Error); err != nil {
 		return nil, err
@@ -505,11 +496,11 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 			return nil, err
 		}
 		if response.Published == nil {
-			return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
+			return nil, fmt.Errorf("%s returned no published volume", subject)
 		}
 		published := PublishedVolumeFromV1(response.Published)
 		if err := published.Export.Validate(); err != nil {
-			return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
+			return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
 		}
 		return published, nil
 	}
@@ -521,10 +512,10 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 		return nil, err
 	}
 	if response.Published == nil {
-		return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
+		return nil, fmt.Errorf("%s returned no published volume", subject)
 	}
 	if err := response.Published.Export.Validate(); err != nil {
-		return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
+		return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
 	}
 	return response.Published, nil
 }
@@ -566,15 +557,8 @@ func (p *NATSProvider) HoldsUnsealedWrites(ctx context.Context, nodeID string, v
 	return response.Held, nil
 }
 
-// connected reports whether a request can be sent now. A reconnecting
-// connection would buffer it and send it on reconnect, after its caller may
-// have given up, so the provider would act with nobody waiting on the result.
-func (p *NATSProvider) connected() bool {
-	return p.conn != nil && p.conn.IsConnected()
-}
-
 func (p *NATSProvider) request(ctx context.Context, subject string, input, output any) error {
-	if !p.connected() {
+	if p.conn == nil || !p.conn.IsConnected() {
 		return nats.ErrConnectionClosed
 	}
 	payload, err := json.Marshal(input)
@@ -583,11 +567,7 @@ func (p *NATSProvider) request(ctx context.Context, subject string, input, outpu
 	}
 
 	request := &nats.Msg{Subject: subject, Data: payload, Header: nats.Header{}}
-	accountID := AccountIDFromContext(ctx)
-	if accountID != "" {
-		request.Header.Set(AccountIDHeader, accountID)
-	}
-	ctx, span := startClientSpan(ctx, subject, accountID, request.Header)
+	ctx, span := startClientSpan(ctx, subject, request.Header)
 	defer span.End()
 
 	requestCtx, cancel := context.WithTimeout(ctx, p.timeoutFor(subject))
@@ -599,7 +579,7 @@ func (p *NATSProvider) request(ctx context.Context, subject string, input, outpu
 	}
 	if err := json.Unmarshal(msg.Data, output); err != nil {
 		RecordSpanError(span, err)
-		return fmt.Errorf("decode %s response: %w: %w", subject, ErrUnreadableReply, err)
+		return fmt.Errorf("decode %s response: %w", subject, err)
 	}
 	RecordResponseError(span, msg.Data)
 	return nil
@@ -618,7 +598,7 @@ func (p *NATSProvider) requestOwnerFirst(ctx context.Context, ownerSubject, queu
 
 func responseError(version uint16, providerErr *ProviderError) error {
 	if err := checkVersion(version); err != nil {
-		return fmt.Errorf("%w: %w", ErrUnreadableReply, err)
+		return err
 	}
 	if providerErr != nil {
 		return providerErr

spinifex/providers/ebs/nats_reconnect_test.go

@@ -1,58 +0,0 @@
-package ebsprovider
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A request on a connection that is down fails at once rather than buffering:
-// one flushed on reconnect would run after its caller gave up on it.
-func TestNATSProviderRequestRefusesWhileDisconnected(t *testing.T) {
-	connections := map[string]func(t *testing.T) *nats.Conn{
-		"Reconnecting": func(t *testing.T) *nats.Conn {
-			ns := startNATS(t)
-			nc, err := nats.Connect(ns.ClientURL(), nats.MaxReconnects(-1), nats.ReconnectWait(time.Hour))
-			require.NoError(t, err)
-			t.Cleanup(nc.Close)
-			ns.Shutdown()
-			require.Eventually(t, func() bool { return !nc.IsConnected() }, 5*time.Second, 5*time.Millisecond)
-			require.False(t, nc.IsClosed(), "the connection must still be trying to reconnect")
-			return nc
-		},
-		"Closed": func(t *testing.T) *nats.Conn {
-			nc, err := nats.Connect(startNATS(t).ClientURL())
-			require.NoError(t, err)
-			nc.Close()
-			return nc
-		},
-	}
-	for name, connect := range connections {
-		t.Run(name, func(t *testing.T) {
-			provider := NewNATSProvider(connect(t), 5*time.Second)
-			ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
-			defer cancel()
-
-			_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
-				Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: "node-down",
-			})
-			require.ErrorIs(t, err, nats.ErrConnectionClosed)
-			assert.NotErrorIs(t, err, context.DeadlineExceeded, "the request must be refused, not left waiting")
-		})
-	}
-}
-
-func startNATS(t *testing.T) *server.Server {
-	t.Helper()
-	ns, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true})
-	require.NoError(t, err)
-	go ns.Start()
-	require.True(t, ns.ReadyForConnections(5*time.Second))
-	t.Cleanup(ns.Shutdown)
-	return ns
-}

spinifex/providers/ebs/nats_test.go

@@ -475,7 +475,6 @@ func TestNATSProviderRejectsEmptySuccessResponse(t *testing.T) {
 		respondWith(t, conn, subject, PublishVolumeResponse{Versioned: empty})
 		_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-empty", NodeID: "node-empty"})
 		require.EqualError(t, err, subject+" returned no published volume")
-		require.ErrorIs(t, err, ErrNoPublishedVolume)
 	})
 
 	t.Run("CreateSnapshot", func(t *testing.T) {
@@ -513,7 +512,6 @@ func TestNATSProviderRejectsInvalidExport(t *testing.T) {
 			})
 			published, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: tt.node})
 			require.EqualError(t, err, subject+" returned an invalid export: "+tt.wantErr)
-			require.ErrorIs(t, err, ErrInvalidExport)
 			assert.Nil(t, published)
 		})
 	}
@@ -720,6 +718,8 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	// A delete costs one round trip per object, so its duration is a function
 	// of the volume. On prod a 100 GiB volume's delete took 52 s against a 30 s
@@ -740,7 +740,7 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
-		publish, unsealed,
+		publish, unsealed, nodeCapabilities,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -126,6 +126,8 @@ func wireVerbs(t *testing.T) []wireVerb {
 	require.NoError(t, err)
 	unsealedSubject, err := ebsprovider.UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	return []wireVerb{
 		{name: "capabilities", subject: ebsprovider.CapabilitiesSubject,
@@ -154,6 +156,8 @@ func wireVerbs(t *testing.T) []wireVerb {
 			request: ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
 		{name: "unsealed", subject: unsealedSubject,
 			request: ebsprovider.HoldsUnsealedWritesRequest{Versioned: ebsprovider.NewVersioned(), NodeID: "node-1", VolumeIDs: []string{"vol-1"}}},
+		{name: "node.capabilities", subject: nodeCapabilitiesSubject,
+			request: ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}},
 	}
 }
 
@@ -733,6 +737,22 @@ func TestServeNodeIDScopesMountSubjects(t *testing.T) {
 	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's mount subject must go unanswered")
 }
 
+// TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode is why the per-node subject
+// exists: the shared one is answered by whichever node's provider is fastest,
+// which says nothing about the provider on the node a caller will mount on.
+func TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode(t *testing.T) {
+	conn := serveProvider(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node-1"})
+	client := ebsprovider.NewNATSProvider(conn, requestTimeout)
+	request := ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}
+
+	resp, err := client.GetNodeCapabilities(t.Context(), "node-1", request)
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+
+	_, err = client.GetNodeCapabilities(t.Context(), "node-2", request)
+	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's capabilities subject must go unanswered")
+}
+
 // TestServeRejectsUnroutableNodeID covers the subject-build failure path: an
 // unsubscribable node ID must fail Serve outright rather than leave a server
 // running with the shared subjects subscribed and no mount subject at all.

spinifex/providers/ebs/natsserve/server.go

@@ -36,9 +36,9 @@ type Options struct {
 	// across workers does not apply.
 	NoQueueGroup bool
 
-	// NodeID scopes the per-node subjects (publish, unpublish and unsealed
-	// writes) to this node, as a production daemon serves them. Left empty,
-	// Serve subscribes their wildcards and answers for any node.
+	// NodeID scopes the per-node subjects (publish, unpublish, unsealed writes
+	// and per-node capabilities) to this node, as a production daemon serves
+	// them. Left empty, Serve subscribes their wildcards and answers for any node.
 	NodeID string
 }
 
@@ -135,6 +135,7 @@ func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider,
 		{ebsprovider.PublishSubjectWildcard, ebsprovider.PublishSubject, handlePublishVolume(provider)},
 		{ebsprovider.UnpublishSubjectWildcard, ebsprovider.UnpublishSubject, handleUnpublishVolume(provider)},
 		{ebsprovider.UnsealedWritesSubjectWildcard, ebsprovider.UnsealedWritesSubject, handleHoldsUnsealedWrites(provider)},
+		{ebsprovider.NodeCapabilitiesSubjectWildcard, ebsprovider.NodeCapabilitiesSubject, handleCapabilities(provider)},
 	}
 	for _, step := range nodeSteps {
 		subject := step.wildcard

spinifex/providers/ebs/natsserve/server_test.go

@@ -81,34 +81,3 @@ func TestServerSpan_SurvivesAnUntracedCaller(t *testing.T) {
 	assert.Equal(t, "ebs.volume.list", serverSpan.Name())
 	assert.False(t, serverSpan.Parent().IsValid(), "an untraced caller must not leave a dangling parent")
 }
-
-// TestServerSpan_CarriesTheCallersAccount attributes provider work to the
-// account the request acts for, on both halves of the hop, and leaves an
-// unattributed request unattributed.
-func TestServerSpan_CarriesTheCallersAccount(t *testing.T) {
-	recorder := testutil.RecordSpans(t)
-	client := serveMemoryProvider(t)
-
-	ctx := ebsprovider.WithAccountID(t.Context(), "000000000042")
-	_, err := client.ListVolumes(ctx, ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
-	require.NoError(t, err)
-	clientSpan, serverSpan := spansByKind(t, recorder)
-	assert.Equal(t, "000000000042", spanAccount(clientSpan))
-	assert.Equal(t, "000000000042", spanAccount(serverSpan))
-
-	recorder = testutil.RecordSpans(t)
-	_, err = client.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
-	require.NoError(t, err)
-	clientSpan, serverSpan = spansByKind(t, recorder)
-	assert.Empty(t, spanAccount(clientSpan))
-	assert.Empty(t, spanAccount(serverSpan))
-}
-
-func spanAccount(span sdktrace.ReadOnlySpan) string {
-	for _, attr := range span.Attributes() {
-		if attr.Key == "aws.account_id" {
-			return attr.Value.AsString()
-		}
-	}
-	return ""
-}

spinifex/providers/ebs/provider.go

@@ -40,13 +40,14 @@ type Versioned struct {
 	SchemaVersion uint16 `json:"schema_version"`
 }
 
-// NewVersioned returns LegacySchemaVersion, the oldest representation a
-// provider serves, for a request whose shape is the same at every version.
+// NewVersioned returns the oldest representation guaranteed to interoperate
+// during an N/N-1 rollout. Callers may use NewLatestVersioned only after a
+// capability exchange proves the target understands SchemaVersion.
 func NewVersioned() Versioned { return Versioned{SchemaVersion: LegacySchemaVersion} }
 
-// NewLatestVersioned returns SchemaVersion. A cluster runs one contract
-// version, which every provider in it serves, so a request that needs a field
-// only the newest version carries is sent at it without asking first.
+// NewLatestVersioned returns the newest representation this build understands.
+// It must not be used merely because this binary is new: a separately released
+// peer may still be serving LegacySchemaVersion.
 func NewLatestVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} }
 
 // SupportsSchemaVersion reports whether this build can safely decode version.

spinifex/providers/ebs/telemetry.go

@@ -30,9 +30,6 @@ const (
 	attrNodeID      = "ebs.provider.node_id"
 	attrOwnerRouted = "ebs.provider.owner_routed"
 	attrErrorCode   = "ebs.provider.error_code"
-	// attrAccountID matches the key the gateway and the control plane's own
-	// NATS spans use, so one field follows a request to the provider.
-	attrAccountID = "aws.account_id"
 )
 
 // VerbUnknown is the verb reported for a subject that is not part of the
@@ -118,6 +115,8 @@ func parseSubject(subject string) subjectTarget {
 		return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID}
 	case unsealedVerb:
 		return subjectTarget{verb: verbUnsealedWrites, nodeID: nodeID}
+	case nodeCapabilitiesVerb:
+		return subjectTarget{verb: verbCapabilities, nodeID: nodeID}
 	case fenceNoticeVerb:
 		return subjectTarget{verb: verbFenceNotice, nodeID: nodeID}
 	default:
@@ -125,16 +124,11 @@ func parseSubject(subject string) subjectTarget {
 	}
 }
 
-// attributes describes the target; an empty accountID is left off, as work
-// with no caller must not be credited to anyone.
-func (t subjectTarget) attributes(subject, accountID string) []attribute.KeyValue {
+func (t subjectTarget) attributes(subject string) []attribute.KeyValue {
 	attrs := []attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String(attrSubject, subject),
 	}
-	if accountID != "" {
-		attrs = append(attrs, attribute.String(attrAccountID, accountID))
-	}
 	if t.volumeID != "" {
 		attrs = append(attrs, attribute.String(attrVolumeID, t.volumeID))
 	}
@@ -162,11 +156,11 @@ func (c headerCarrier) Keys() []string {
 
 // startClientSpan opens a span for an outbound request and injects the trace
 // context into hdr so the serving provider joins the same trace.
-func startClientSpan(ctx context.Context, subject, accountID string, hdr nats.Header) (context.Context, trace.Span) {
+func startClientSpan(ctx context.Context, subject string, hdr nats.Header) (context.Context, trace.Span) {
 	target := parseSubject(subject)
 	ctx, span := otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindClient),
-		trace.WithAttributes(target.attributes(subject, accountID)...))
+		trace.WithAttributes(target.attributes(subject)...))
 	otel.GetTextMapPropagator().Inject(ctx, headerCarrier(hdr))
 	return ctx, span
 }
@@ -184,18 +178,16 @@ func startCompletionSpan(ctx context.Context, verb, completionSubject string) (c
 }
 
 // StartServerSpan joins the trace carried in msg's headers and opens a span
-// for the verb its subject names, attributed to the account the request
-// carries. Callers must End the span and pass the context into the handler.
+// for the verb its subject names. Callers must End the returned span and
+// should pass the returned context into the work the handler does.
 func StartServerSpan(ctx context.Context, msg *nats.Msg) (context.Context, trace.Span) {
 	target := parseSubject(msg.Subject)
-	var accountID string
 	if msg.Header != nil {
 		ctx = otel.GetTextMapPropagator().Extract(ctx, headerCarrier(msg.Header))
-		accountID = msg.Header.Get(AccountIDHeader)
 	}
 	return otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindServer),
-		trace.WithAttributes(target.attributes(msg.Subject, accountID)...))
+		trace.WithAttributes(target.attributes(msg.Subject)...))
 }
 
 // RecordSpanError marks span failed for a transport or handler error.

spinifex/providers/ebs/telemetry_test.go

@@ -32,6 +32,8 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject(nodeID)
 	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject(nodeID)
+	require.NoError(t, err)
 	fenced, err := FenceNoticeSubject(nodeID)
 	require.NoError(t, err)
 
@@ -52,6 +54,7 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 		publish:               verbVolumePublish,
 		unpublish:             verbVolumeUnpublish,
 		unsealed:              verbUnsealedWrites,
+		nodeCapabilities:      verbCapabilities,
 		fenced:                verbFenceNotice,
 	}
 

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
-	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
 )
@@ -54,46 +53,24 @@ func requireExportTools(t *testing.T) string {
 	return pluginPath
 }
 
-// queueOnlyNode stands a node up whose provider subjects are served on a NATS
-// of its own, with the cluster's lease, dirty and liveness state, so its
-// client reaches it on the queue subjects alone, as when an owner is silent.
-func queueOnlyNode(t *testing.T, node *Config, clusterNC *nats.Conn) ebsprovider.EBSProvider {
-	t.Helper()
-	dirty, err := newVolumeDirty(t.Context(), clusterNC, node.leaseOwner())
-	require.NoError(t, err)
-	node.dirty = dirty
-	node.liveness = newHeartbeatLiveness(clusterNC)
-
-	_, privateURL := setupEmbeddedNATS(t)
-	return ebsprovider.NewNATSProvider(startProviderSubjects(t, node, privateURL), 60*time.Second)
-}
-
-// providerHarness is what inProcessProvider stands up: a client for the
-// cluster, both node names, and the hooks the conformance suite takes.
-type providerHarness struct {
-	newProvider      func(t *testing.T) ebsprovider.EBSProvider
-	nodeID           string
-	otherNodeID      string
-	killExport       func(*testing.T, *ebsprovider.PublishedVolume)
-	queueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
-}
-
 // inProcessProvider stands two nodes of the provider handlers up over one
-// embedded NATS and one predastore. Everything is torn down with the test.
+// embedded NATS and one predastore, and returns a client for them plus both
+// node names. Everything is torn down with the test.
 //
 // Two nodes rather than one because the exclusion this provider advertises is
 // cluster-scoped, and a single node cannot exercise it: the second publish
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
-func inProcessProvider(t *testing.T) providerHarness {
+// The last return kills a published export's nbdkit behind the handlers' back.
+func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string, func(*testing.T, *ebsprovider.PublishedVolume)) {
 	t.Helper()
 	pluginPath := requireExportTools(t)
 
 	fixture := testpredastore.Start(t)
 	_, natsURL := setupEmbeddedNATS(t)
 
-	newNode := func(t *testing.T, name string) *Config {
+	newNode := func(name string) *Config {
 		cfg := setupTestConfig(t, natsURL)
 		cfg.S3Host = "https://" + fixture.Host
 		cfg.Bucket = testpredastore.DefaultBucket
@@ -111,8 +88,8 @@ func inProcessProvider(t *testing.T) providerHarness {
 		return cfg
 	}
 
-	first := newNode(t, "node-1")
-	second := newNode(t, "node-2")
+	first := newNode("node-1")
+	second := newNode("node-2")
 	nc := startProviderSubjects(t, first, natsURL)
 	startProviderSubjects(t, second, natsURL)
 
@@ -123,29 +100,21 @@ func inProcessProvider(t *testing.T) providerHarness {
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
-	return providerHarness{
-		newProvider: func(t *testing.T) ebsprovider.EBSProvider {
-			t.Helper()
-			return ebsprovider.NewNATSProvider(nc, 60*time.Second)
-		},
-		nodeID:      first.NodeName,
-		otherNodeID: second.NodeName,
-		// Kills a published export's nbdkit behind the handlers' back.
-		killExport: func(t *testing.T, published *ebsprovider.PublishedVolume) {
-			t.Helper()
-			for _, node := range []*Config{first, second} {
-				if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
-					require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
-					return
-				}
+	killExport := func(t *testing.T, published *ebsprovider.PublishedVolume) {
+		t.Helper()
+		for _, node := range []*Config{first, second} {
+			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
+				require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+				return
 			}
-			t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
-		},
-		queueOnOtherNode: func(t *testing.T) ebsprovider.EBSProvider {
-			t.Helper()
-			return queueOnlyNode(t, newNode(t, "node-3"), nc)
-		},
+		}
+		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
 	}
+
+	return func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		return ebsprovider.NewNATSProvider(nc, 60*time.Second)
+	}, first.NodeName, second.NodeName, killExport
 }
 
 // runPrefix names one run's volumes so it cannot meet an earlier run's
@@ -157,13 +126,12 @@ func runPrefix() string {
 // TestViperblockdConformance judges the provider handlers by the same suite
 // MemoryProvider and qemunbdd answer to.
 func TestViperblockdConformance(t *testing.T) {
-	h := inProcessProvider(t)
-	conformance.RunSuiteWithConfig(t, h.newProvider, conformance.SuiteConfig{
-		NamePrefix:       runPrefix(),
-		NodeID:           h.nodeID,
-		OtherNodeID:      h.otherNodeID,
-		KillExport:       h.killExport,
-		QueueOnOtherNode: h.queueOnOtherNode,
+	newProvider, nodeID, otherNodeID, killExport := inProcessProvider(t)
+	conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{
+		NamePrefix:  runPrefix(),
+		NodeID:      nodeID,
+		OtherNodeID: otherNodeID,
+		KillExport:  killExport,
 	})
 }
 
@@ -171,9 +139,9 @@ func TestViperblockdConformance(t *testing.T) {
 // cannot: whether the nbdkit export viperblockd publishes is usable by libnbd,
 // which knows nothing about this codebase.
 func TestViperblockdNBDClient(t *testing.T) {
-	h := inProcessProvider(t)
-	conformance.RunNBDClientSuiteWithConfig(t, h.newProvider, conformance.NBDClientConfig{
-		NodeID:       h.nodeID,
+	newProvider, nodeID, _, _ := inProcessProvider(t)
+	conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{
+		NodeID:       nodeID,
 		VolumePrefix: "vol-" + runPrefix() + "nbd",
 	})
 }

spinifex/providers/ebs/viperblock/provider_contract_test.go

@@ -12,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
@@ -177,6 +176,23 @@ func TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion(t *test
 	}
 }
 
+// TestProviderHandlers_NodeCapabilitiesListsServedVersions pins the per-node
+// subject to the same contract versions this build serves.
+func TestProviderHandlers_NodeCapabilitiesListsServedVersions(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	require.NoError(t, err)
+	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}))
+
+	var resp ebsprovider.GetCapabilitiesResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+}
+
 // TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused covers a caller
 // newer than this build: a no_seal it cannot vouch for is refused, and the
 // export is neither torn down nor sealed.
@@ -236,9 +252,6 @@ func TestPublishError_Codes(t *testing.T) {
 			assert.Equal(t, tc.want, got.Code)
 		})
 	}
-	// Callers still recognise held-elsewhere by its text, so the message keeps it.
-	heldElsewhere := publishError(ebsprovider.NoSealSchemaVersion, "vol-publisherror01", leaseHeld)
-	assert.Contains(t, heldElsewhere.Message, ec2v1.VolumeHeldElsewhereText)
 	assert.True(t, ebsprovider.ErrorCodeUnavailable.Retryable(), "a backing store that is not ready must read as retry-later")
 }
 

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -149,6 +149,16 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", unsealedSubject, err)
 	}
 
+	// The queue subject is answered by any node's provider; this one only by
+	// this node's, so a daemon can learn what its own provider serves.
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	if err != nil {
+		return fmt.Errorf("build node capabilities subject: %w", err)
+	}
+	if _, err := nc.Subscribe(nodeCapabilitiesSubject, tracedProviderHandler(handleProviderCapabilities)); err != nil {
+		return fmt.Errorf("subscribe to %s: %w", nodeCapabilitiesSubject, err)
+	}
+
 	return nil
 }
 
@@ -1131,9 +1141,13 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 		},
 	})
 
-	// The drain runs in the background with the snapshot, so only the accept
-	// reply is freed of it; a caller waiting on completion still waits for it.
-	// WithoutCancel keeps the trace context but drops the request's deadline.
+	// The caller is responsible for draining the volume (flushing guest
+	// writes to the live checkpoint) before requesting a snapshot; that
+	// coordination goes through the guest over ec2.cmd.<instanceID> and
+	// stays a control-plane concern this provider boundary does not reach.
+	// WithoutCancel keeps the trace context while dropping the deadline: the
+	// work has to outlive the request it was accepted by, and it still has to
+	// be attributable to it.
 	go completeCreateSnapshot(context.WithoutCancel(ctx), cfg, nc, req, operationID, completionSubject)
 }
 
@@ -1174,7 +1188,7 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 		return ebsprovider.ErrorCodeNotFound
 	case errors.Is(err, errSnapshotExistsElsewhere):
 		return ebsprovider.ErrorCodeAlreadyExists
-	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore), errors.Is(err, errExportSettling):
+	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore):
 		// Exclusive access could not be established. The holder gives the
 		// volume up on detach, so this is worth repeating rather than the
 		// permanent failure an internal error reads as.
@@ -1202,36 +1216,26 @@ func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) error {
 }
 
 // snapshotVolumeEngine creates snapshotID off volumeID's live checkpoint,
-// preferring an already-mounted VB, drained first so the checkpoint is
-// current, over opening a second engine on the same volume.
+// preferring an already-mounted VB over opening a second engine on the same
+// volume. Draining the volume so the checkpoint is current is the caller's
+// responsibility (see handleCreateSnapshot).
+//
+// The unmounted branch opens through openVolumeVB, so it holds the volume's
+// cluster-wide lease for as long as the engine is open: a snapshot taken
+// beside a live writer on another node is two engines on one volume.
 func snapshotVolumeEngine(ctx context.Context, cfg *Config, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) {
 	if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil {
-		if err := drainMountedExport(ctx, cfg, volumeID); err != nil {
-			return nil, err
-		}
 		return createSnapshotWithVB(mv.VB, volumeID, snapshotID)
 	}
 
-	// The lease refuses a volume another node holds. It is reentrant here, so
-	// an export this node runs but has not registered is excluded or drained
-	// separately, and the engine opens only once that is settled.
-	lease, err := cfg.acquireVolumeLease(ctx, volumeID)
+	vb, lease, err := openVolumeVB(ctx, cfg, volumeID)
 	if err != nil {
 		return nil, err
 	}
-	defer cfg.releaseVolumeLease(ctx, lease)
-
-	lock, err := excludeLocalWriter(ctx, cfg, volumeID)
-	if err != nil {
-		return nil, err
-	}
-	defer releaseWriterLock(ctx, lock, volumeID)
-
-	vb, err := openLeasedVolumeVB(ctx, cfg, volumeID, lease)
-	if err != nil {
-		return nil, err
-	}
-	defer vb.Detach()
+	defer func() {
+		vb.Detach()
+		cfg.releaseVolumeLease(ctx, lease)
+	}()
 	return createSnapshotWithVB(vb, volumeID, snapshotID)
 }
 
@@ -1925,7 +1929,7 @@ func tearDownPublication(ctx context.Context, cfg *Config, req ebsprovider.Unpub
 	if err != nil {
 		// Nothing mounted here is already the state unpublish asks for, so a
 		// retry after a request that completed server-side converges rather
-		// than erroring.
+		// than erroring, matching the legacy path's unmountResponseError.
 		if unmountResp.NotFound {
 			return unmountResp.Reaped, nil
 		}

spinifex/providers/ebs/viperblock/provider_handlers_test.go

@@ -154,6 +154,7 @@ func providerSubjectCases() []providerSubjectCase {
 			"node_id":    "test-node",
 			"volume_ids": []string{"vol-testunsealed01"},
 		}},
+		{"node_capabilities", "ebs.provider.v1.test-node.capabilities", map[string]any{}},
 	}
 }
 
@@ -1006,8 +1007,9 @@ func TestProviderHandlers_UnpublishVolume_RemovesMountedVolume(t *testing.T) {
 }
 
 // TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent covers a volume this
-// node never mounted, which counts as a successful seal: a retry after a
-// request that timed out client-side but completed server-side must converge.
+// node never mounted. The legacy path treats that as a successful seal (see
+// unmountResponseError), and so must this one: a retry after a request that
+// timed out client-side but completed server-side has to converge.
 func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
 	cfg := setupTestConfig(t, natsURL)
@@ -1041,8 +1043,8 @@ func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
 	cfgNamed := setupTestConfig(t, natsURL2)
 	ncNamed := startProviderSubjects(t, cfgNamed, natsURL2)
 
-	assert.Equal(t, ncNamed.NumSubscriptions()-3, ncNoNode.NumSubscriptions(),
-		"an empty NodeName must skip exactly the publish, unpublish and unsealed-writes subscriptions")
+	assert.Equal(t, ncNamed.NumSubscriptions()-4, ncNoNode.NumSubscriptions(),
+		"an empty NodeName must skip exactly the publish, unpublish, unsealed-writes and node capabilities subscriptions")
 
 	// The other provider subjects must still work without a node name.
 	body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion})

spinifex/providers/ebs/viperblock/snapshot_drain.go

@@ -1,129 +0,0 @@
-package viperblock
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log/slog"
-	"net"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/mulgadc/viperblock/viperblock"
-)
-
-const (
-	// exportDrainDialTimeout bounds the connect to the plugin's drain socket.
-	// The socket is local, so a slow connect is a wedged plugin.
-	exportDrainDialTimeout = time.Second
-
-	// exportDrainAckTimeout bounds the plugin's flush of its WAL and live
-	// checkpoint to the backend, which scales with the dirty set.
-	exportDrainAckTimeout = 30 * time.Second
-
-	// maxUnixSocketPath is sun_path's size. The plugin cannot listen on a
-	// longer path, so a drain socket there never exists.
-	maxUnixSocketPath = 108
-)
-
-// errExportDrainFailed is a drain the plugin was asked for and did not
-// confirm, so the live checkpoint cannot be shown to be current.
-var errExportDrainFailed = errors.New("export drain failed")
-
-// errExportSettling is an export whose client is connecting or disconnecting:
-// it holds the volume lock without serving the drain socket, so its writes are
-// neither drainable nor yet sealed. It clears once the client settles.
-var errExportSettling = errors.New("export is between connections")
-
-// drainMountedExport asks the nbdkit plugin serving volumeID on this node to
-// flush its WAL and live checkpoint to the backend before a snapshot reads
-// them. The plugin serves the socket only while a client is connected.
-func drainMountedExport(ctx context.Context, cfg *Config, volumeID string) error {
-	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
-	if len(sockPath) >= maxUnixSocketPath {
-		return requireNoConnectedClient(ctx, cfg, volumeID, "")
-	}
-	start := time.Now()
-	conn, err := net.DialTimeout("unix", sockPath, exportDrainDialTimeout)
-	if errors.Is(err, fs.ErrNotExist) {
-		return requireNoConnectedClient(ctx, cfg, volumeID, sockPath)
-	}
-	if err != nil {
-		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
-	}
-	defer conn.Close()
-
-	_ = conn.SetDeadline(time.Now().Add(exportDrainAckTimeout))
-	buf := make([]byte, 8)
-	n, err := conn.Read(buf)
-	if err != nil {
-		return fmt.Errorf("%w: %s: read ack: %w", errExportDrainFailed, volumeID, err)
-	}
-	if !strings.HasPrefix(string(buf[:n]), "OK") {
-		return fmt.Errorf("%w: %s: ack %q", errExportDrainFailed, volumeID, string(buf[:n]))
-	}
-	slog.InfoContext(ctx, "ebs.provider.snapshot.create: drained export", "volume", volumeID, "drain_ms", otelsetup.Millis(time.Since(start)))
-	return nil
-}
-
-// requireNoConnectedClient proves an export serving no drain socket has nothing
-// buffered. sockPath is empty when the socket path is too long for the plugin
-// to listen on, so a held lock there never clears.
-func requireNoConnectedClient(ctx context.Context, cfg *Config, volumeID, sockPath string) error {
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	if errors.Is(err, viperblock.ErrVolumeLocked) {
-		if sockPath == "" {
-			return fmt.Errorf("%w: %s: client connected with no drain socket", errExportDrainFailed, volumeID)
-		}
-		slog.WarnContext(ctx, "ebs.provider.snapshot.create: volume locked with no drain socket, refusing as retry-later",
-			"volume", volumeID, "socket", sockPath)
-		return fmt.Errorf("%w: %s", errExportSettling, volumeID)
-	}
-	if err != nil {
-		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
-	}
-	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
-		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
-	}
-	return requireLastClientSealed(cfg, volumeID)
-}
-
-// requireLastClientSealed refuses an export whose last client left WAL behind
-// without a seal receipt newer than it. The plugin releases the lock after a
-// failed seal too, so a free lock alone does not show the writes reached the backend.
-func requireLastClientSealed(cfg *Config, volumeID string) error {
-	if isAuxVolume(volumeID) || !volumeHoldsLocalWAL(cfg, volumeID) || sealReceiptCoversWAL(cfg, volumeID) {
-		return nil
-	}
-	return fmt.Errorf("%w: %s: last client left unsealed WAL", errExportDrainFailed, volumeID)
-}
-
-// excludeLocalWriter proves no export on this node holds writes the backend
-// lacks before a snapshot opens its own engine. The returned lock keeps a new
-// writer out until the snapshot is read; it is nil when a live export was drained.
-func excludeLocalWriter(ctx context.Context, cfg *Config, volumeID string) (*os.File, error) {
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	if errors.Is(err, viperblock.ErrVolumeLocked) {
-		return nil, drainMountedExport(ctx, cfg, volumeID)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
-	}
-	if err := requireLastClientSealed(cfg, volumeID); err != nil {
-		releaseWriterLock(ctx, lock, volumeID)
-		return nil, err
-	}
-	return lock, nil
-}
-
-// releaseWriterLock gives up a lock excludeLocalWriter took. A failed release
-// leaves only the kernel to drop it at exit, so it is logged, not returned.
-func releaseWriterLock(ctx context.Context, lock *os.File, volumeID string) {
-	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
-		slog.WarnContext(ctx, "ebs.provider.snapshot.create: failed to release volume lock", "volume", volumeID, "err", err)
-	}
-}

spinifex/providers/ebs/viperblock/snapshot_drain_test.go

@@ -1,450 +0,0 @@
-package viperblock
-
-import (
-	"encoding/json"
-	"io/fs"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// mountedSnapshotFixture is a node with volumeID mounted over a real engine,
-// whose base dir is where testutil's drain socket stand-in listens.
-func mountedSnapshotFixture(t *testing.T, volumeID string) (cfg *Config, dataDir string, vb *viperblock.VB) {
-	t.Helper()
-	dataDir = testutil.SocketTempDir(t)
-	cfg, vb = mountedAt(t, volumeID, filepath.Join(dataDir, "viperblock"))
-	return cfg, dataDir, vb
-}
-
-// mountedAt is a node with volumeID mounted over a real engine under baseDir.
-func mountedAt(t *testing.T, volumeID, baseDir string) (*Config, *viperblock.VB) {
-	t.Helper()
-	vb := createTestVBWithState(t, volumeID)
-	t.Cleanup(func() {
-		vb.StopChunkUploader()
-		vb.StopWALSyncer()
-	})
-	cfg := &Config{
-		NodeName:       "node-a",
-		BaseDir:        baseDir,
-		MountedVolumes: []MountedVolume{{Name: volumeID, VB: vb}},
-	}
-	require.NoError(t, os.MkdirAll(cfg.BaseDir, 0o750))
-	return cfg, vb
-}
-
-// leaveUnsealedWAL stands in for a client whose disconnect seal failed: the
-// plugin keeps the volume's local WAL and writes no receipt.
-func leaveUnsealedWAL(t *testing.T, cfg *Config, volumeID string) {
-	t.Helper()
-	require.NoError(t, os.MkdirAll(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks"), 0o750))
-}
-
-func requireNoSnapshot(t *testing.T, vb *viperblock.VB, snapshotID string) {
-	t.Helper()
-	exists, err := snapshotConfigExists(vb, snapshotID)
-	require.NoError(t, err)
-	assert.False(t, exists, "a snapshot that could not be shown current must not be written")
-}
-
-// The snapshot reads the checkpoint the plugin flushes, so it must not start
-// until the plugin has acknowledged the drain.
-func TestSnapshotVolumeEngine_MountedVolumeIsDrainedBeforeTheSnapshot(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainfirst", "snap-drainfirst00001"
-	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
-	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
-
-	type result struct {
-		snapshot *ebsprovider.Snapshot
-		err      error
-	}
-	done := make(chan result, 1)
-	go func() {
-		snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-		done <- result{snapshot, err}
-	}()
-
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("a mounted volume's snapshot never asked the plugin to drain")
-	}
-	requireNoSnapshot(t, vb, snapshotID)
-	release()
-
-	select {
-	case got := <-done:
-		require.NoError(t, got.err)
-		assert.Equal(t, ebsprovider.SnapshotStateCompleted, got.snapshot.State)
-	case <-time.After(10 * time.Second):
-		t.Fatal("snapshot did not complete after the drain acked")
-	}
-}
-
-// The control plane issues no drain of its own, so this one is the only flush
-// an attached volume's snapshot gets, and it must happen exactly once.
-func TestSnapshotVolumeEngine_MountedVolumeIsDrainedOnce(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainonce", "snap-drainonce000001"
-	cfg, dataDir, _ := mountedSnapshotFixture(t, volumeID)
-	drains := testutil.StartDrainSocket(t, dataDir, volumeID, "OK\n")
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
-	assert.Equal(t, 1, drains())
-}
-
-// A plugin that answers without confirming the flush leaves the checkpoint
-// unproven, so the snapshot fails rather than capturing older data.
-func TestSnapshotVolumeEngine_RefusedDrainFailsTheSnapshot(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainerr", "snap-drainerr0000001"
-	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
-	testutil.StartDrainSocket(t, dataDir, volumeID, "ERR\n")
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// A socket file nobody listens on is a plugin that died mid-connection, whose
-// writes were never drained.
-func TestSnapshotVolumeEngine_StaleDrainSocketFailsTheSnapshot(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainstale", "snap-drainstale00001"
-	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
-	sockDir := filepath.Join(cfg.BaseDir, volumeID)
-	require.NoError(t, os.MkdirAll(sockDir, 0o750))
-	ln, err := net.Listen("unix", filepath.Join(sockDir, "snapshot.sock"))
-	require.NoError(t, err)
-	ln.(*net.UnixListener).SetUnlinkOnClose(false)
-	require.NoError(t, ln.Close())
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// The plugin serves the socket only while a client is connected, and drains
-// on disconnect. An export with no client has nothing buffered to flush.
-func TestSnapshotVolumeEngine_ExportWithoutAClientNeedsNoDrain(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainidle", "snap-drainidle000001"
-	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
-}
-
-// A client between connections holds the volume lock without serving the
-// socket: its writes are neither drainable nor sealed yet, which clears soon.
-func TestSnapshotVolumeEngine_ExportBetweenConnectionsIsRetryLater(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainsettle", "snap-drainsettle0001"
-	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
-	logs := captureLogs(t)
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportSettling)
-	assert.Nil(t, snapshot)
-	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
-	requireNoSnapshot(t, vb, snapshotID)
-	// A plugin that never serves its socket refuses every retry, so each
-	// refusal is logged with the socket an operator would look for.
-	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
-	assert.True(t, slices.ContainsFunc(strings.Split(logs.String(), "\n"), func(line string) bool {
-		return strings.Contains(line, "level=WARN") && strings.Contains(line, "volume="+volumeID) &&
-			strings.Contains(line, "socket="+sockPath)
-	}), "the refusal must be logged at WARN with the volume and socket: %s", logs.String())
-}
-
-// The plugin releases the lock even when its disconnect seal fails, so a free
-// lock with WAL left behind and no receipt is writes that never reached the
-// backend, and the checkpoint a snapshot would read predates them.
-func TestSnapshotVolumeEngine_UnsealedLastClientFailsTheSnapshot(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainunsealed", "snap-drainunsealed01"
-	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
-	leaveUnsealedWAL(t, cfg, volumeID)
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// A seal receipt is the plugin's proof the last client's writes reached the
-// backend, so local WAL beside it does not block the snapshot.
-func TestSnapshotVolumeEngine_SealedLastClientProceeds(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainsealed", "snap-drainsealed0001"
-	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
-	leaveUnsealedWAL(t, cfg, volumeID)
-	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: time.Now()})
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
-	assert.FileExists(t, sealReceiptPath(cfg.BaseDir, volumeID), "the receipt belongs to the unmount it attests to")
-}
-
-// A socket path the plugin could not have listened on never appears, so a
-// connected client there can never be drained: that is permanent, not retry.
-func TestSnapshotVolumeEngine_ConnectedClientWithUnservableSocketFails(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainlongpath", "snap-drainlongpath01"
-	baseDir := filepath.Join(t.TempDir(), strings.Repeat("d", maxUnixSocketPath))
-	cfg, vb := mountedAt(t, volumeID, baseDir)
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.NotErrorIs(t, err, errExportSettling)
-	assert.Nil(t, snapshot)
-	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// A plugin that closes the connection without an ack has not confirmed the
-// flush, so the snapshot fails.
-func TestSnapshotVolumeEngine_DrainClosedWithoutAckFailsTheSnapshot(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainnoack", "snap-drainnoack00001"
-	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
-	testutil.StartDrainSocket(t, dataDir, volumeID, "")
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// lockProbingS3Host is a backend that refuses every request, recording for
-// each whether a new writer could have taken volumeID's lock at that moment.
-type lockProbingS3Host struct {
-	url      string
-	requests atomic.Int32
-	lockable atomic.Int32
-}
-
-func startLockProbingS3Host(t *testing.T, baseDir, volumeID string) *lockProbingS3Host {
-	t.Helper()
-	probe := &lockProbingS3Host{}
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		probe.requests.Add(1)
-		if lock, err := viperblock.AcquireVolumeLock(baseDir, volumeID); err == nil {
-			probe.lockable.Add(1)
-			_ = viperblock.ReleaseVolumeLock(lock)
-		}
-		w.WriteHeader(http.StatusBadRequest)
-	}))
-	t.Cleanup(srv.Close)
-	probe.url = srv.URL
-	return probe
-}
-
-// unmountedSnapshotFixture is the volume's own host with nothing registered
-// for it, so a snapshot takes the open-an-engine branch.
-func unmountedSnapshotFixture(t *testing.T, volumeID string) (*Config, string, *lockProbingS3Host) {
-	t.Helper()
-	_, natsURL := setupEmbeddedNATS(t)
-	dataDir := testutil.SocketTempDir(t)
-	baseDir := filepath.Join(dataDir, "viperblock")
-	require.NoError(t, os.MkdirAll(baseDir, 0o750))
-	backend := startLockProbingS3Host(t, baseDir, volumeID)
-	cfg := &Config{
-		NodeName:               "node-a",
-		BaseDir:                baseDir,
-		S3Host:                 backend.url,
-		Bucket:                 "test-bucket",
-		Region:                 "us-east-1",
-		AccessKey:              "test-access-key",
-		SecretKey:              "test-secret-key",
-		loadStateRetryAttempts: 1,
-	}
-	installTestVolumeLeases(t, cfg, natsURL)
-	return cfg, dataDir, backend
-}
-
-// holdLockAsExport stands in for an nbdkit this node runs but never
-// registered, as a failed rebuild in recovery leaves behind.
-func holdLockAsExport(t *testing.T, cfg *Config, volumeID string) {
-	t.Helper()
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
-}
-
-// An unregistered export still takes guest writes, so the snapshot must have
-// it drain before an engine reads the backend's checkpoint.
-func TestSnapshotVolumeEngine_UnregisteredExportIsDrainedBeforeTheOpen(t *testing.T) {
-	const volumeID, snapshotID = "vol-strayserving", "snap-strayserving001"
-	cfg, dataDir, backend := unmountedSnapshotFixture(t, volumeID)
-	holdLockAsExport(t, cfg, volumeID)
-	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
-
-	done := make(chan error, 1)
-	go func() {
-		_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-		done <- err
-	}()
-
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("an unregistered export was never asked to drain")
-	}
-	assert.Zero(t, backend.requests.Load(), "the engine must not read the backend before the drain acks")
-	release()
-
-	select {
-	case err := <-done:
-		require.Error(t, err, "the refusing backend fails the open")
-		assert.NotErrorIs(t, err, errExportDrainFailed)
-	case <-time.After(10 * time.Second):
-		t.Fatal("snapshot did not proceed after the drain acked")
-	}
-	assert.Positive(t, backend.requests.Load(), "a drained export lets the engine open")
-}
-
-// An unregistered export with no socket cannot be drained, so the snapshot is
-// refused as retry-later before any engine reads the backend.
-func TestSnapshotVolumeEngine_UnregisteredExportWithoutSocketIsRetryLater(t *testing.T) {
-	const volumeID, snapshotID = "vol-straysettle", "snap-straysettle0001"
-	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
-	holdLockAsExport(t, cfg, volumeID)
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportSettling)
-	assert.Nil(t, snapshot)
-	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
-	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
-}
-
-// A departed export that left unsealed WAL holds writes the backend lacks.
-func TestSnapshotVolumeEngine_UnmountedUnsealedWALFailsBeforeTheOpen(t *testing.T) {
-	const volumeID, snapshotID = "vol-strayunsealed", "snap-strayunsealed01"
-	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
-	leaveUnsealedWAL(t, cfg, volumeID)
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
-}
-
-// With no export to drain, the volume lock is held across the open so a
-// writer cannot start between the check and the read.
-func TestSnapshotVolumeEngine_UnmountedOpenHoldsTheVolumeLock(t *testing.T) {
-	const volumeID, snapshotID = "vol-strayidle", "snap-strayidle000001"
-	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
-
-	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.Error(t, err, "the refusing backend fails the open")
-	assert.NotErrorIs(t, err, errExportDrainFailed)
-	assert.NotErrorIs(t, err, errExportSettling)
-	require.Positive(t, backend.requests.Load(), "a node with no export and no WAL opens the engine")
-	assert.Zero(t, backend.lockable.Load(), "a writer could take the volume lock while the engine was reading")
-	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
-	require.NoError(t, err, "the snapshot must give the lock up when it is done")
-	require.NoError(t, viperblock.ReleaseVolumeLock(lock))
-}
-
-// writeSealReceiptAt leaves the receipt the plugin writes after a seal at sealedAt.
-func writeSealReceiptAt(t *testing.T, cfg *Config, volumeID string, sealedAt time.Time) {
-	t.Helper()
-	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: sealedAt})
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
-}
-
-// touchWAL dates every entry of the volume's local WAL to at.
-func touchWAL(t *testing.T, cfg *Config, volumeID string, at time.Time) {
-	t.Helper()
-	dir, err := localWALDir(cfg, volumeID)
-	require.NoError(t, err)
-	require.NoError(t, filepath.WalkDir(dir, func(path string, _ fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		return os.Chtimes(path, at, at)
-	}))
-}
-
-// A receipt is proof only for the WAL that existed when it was written. WAL a
-// later client wrote and failed to seal is newer than it, so it is refused.
-func TestSnapshotVolumeEngine_SealReceiptOlderThanTheWALFails(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainstaleseal", "snap-drainstaleseal1"
-	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
-	sealedAt := time.Now().Add(-time.Minute)
-	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
-	leaveUnsealedWAL(t, cfg, volumeID)
-	require.NoError(t, os.WriteFile(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks", "wal.00000002.bin"), []byte("w"), 0o600))
-	touchWAL(t, cfg, volumeID, sealedAt.Add(time.Second))
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.ErrorIs(t, err, errExportDrainFailed)
-	assert.Nil(t, snapshot)
-	requireNoSnapshot(t, vb, snapshotID)
-}
-
-// A receipt written after every WAL change attests to all of it.
-func TestSnapshotVolumeEngine_SealReceiptNewerThanTheWALProceeds(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainfreshseal", "snap-drainfreshseal1"
-	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
-	leaveUnsealedWAL(t, cfg, volumeID)
-	walChanged := time.Now().Add(-time.Minute)
-	touchWAL(t, cfg, volumeID, walChanged)
-	writeSealReceiptAt(t, cfg, volumeID, walChanged.Add(time.Second))
-
-	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
-}
-
-// The plugin dates its receipt to the millisecond, after the seal's own WAL
-// changes, so a WAL change inside the receipt's millisecond is not newer.
-func TestSnapshotVolumeEngine_SealReceiptInTheWALsMillisecondProceeds(t *testing.T) {
-	const volumeID, snapshotID = "vol-drainsamemilli", "snap-drainsamemilli1"
-	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
-	leaveUnsealedWAL(t, cfg, volumeID)
-	sealedAt := time.Now().Add(-time.Minute).Truncate(time.Millisecond)
-	touchWAL(t, cfg, volumeID, sealedAt.Add(400*time.Microsecond))
-	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
-
-	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
-
-	require.NoError(t, err)
-}

spinifex/providers/ebs/viperblock/viperblock.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
-	"io/fs"
 	"log/slog"
 	"net/http"
 	"os"
@@ -460,22 +459,12 @@ func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperbl
 	if err != nil {
 		return nil, nil, err
 	}
-	vb, err := openLeasedVolumeVB(ctx, cfg, volumeName, lease)
-	if err != nil {
-		cfg.releaseVolumeLease(ctx, lease)
-		return nil, nil, err
-	}
-	return vb, lease, nil
-}
 
-// openLeasedVolumeVB is openVolumeVB under a lease the caller already holds,
-// for a caller with a check to make between the lease and the open. The lease
-// stays the caller's to release, on failure too.
-func openLeasedVolumeVB(ctx context.Context, cfg *Config, volumeName string, lease *volumeLease) (*viperblock.VB, error) {
 	vbconfig, s3cfg := volumeVBConfig(cfg, volumeName)
 	vb, err := viperblock.New(&vbconfig, "s3", s3cfg)
 	if err != nil {
-		return nil, fmt.Errorf("new viperblock: %w", err)
+		cfg.releaseVolumeLease(ctx, lease)
+		return nil, nil, fmt.Errorf("new viperblock: %w", err)
 	}
 
 	// New starts the chunk uploader and WAL syncer, so a failure below must
@@ -484,21 +473,22 @@ func openLeasedVolumeVB(ctx context.Context, cfg *Config, volumeName string, lea
 	defer func() {
 		if !opened {
 			vb.Detach()
+			cfg.releaseVolumeLease(ctx, lease)
 		}
 	}()
 
 	if err := vb.Backend.InitCtx(ctx); err != nil {
-		return nil, fmt.Errorf("backend init: %w", err)
+		return nil, nil, fmt.Errorf("backend init: %w", err)
 	}
 	if err := loadStateWithRetry(ctx, cfg, vb, volumeName); err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
 			cfg.clearVolumeDirty(ctx, volumeName, lease.generation)
 		}
-		return nil, fmt.Errorf("load state: %w", err)
+		return nil, nil, fmt.Errorf("load state: %w", err)
 	}
 
 	opened = true
-	return vb, nil
+	return vb, lease, nil
 }
 
 // isAuxVolume reports whether a volume is an -efi auxiliary volume. Auxiliary
@@ -556,55 +546,6 @@ func volumeNeedsSeal(volumeName, baseDir string) bool {
 	return err == nil
 }
 
-// volumeHoldsLocalWAL reports whether an nbdkit plugin has opened the
-// volume's WAL here and not removed it. A successful seal removes it; an
-// export no client has connected to yet never created it.
-func volumeHoldsLocalWAL(cfg *Config, volumeName string) bool {
-	dir, err := localWALDir(cfg, volumeName)
-	if err != nil {
-		return false
-	}
-	_, err = os.Stat(dir)
-	return err == nil
-}
-
-// localWALDir is where the nbdkit plugin keeps volumeName's WAL on this node.
-func localWALDir(cfg *Config, volumeName string) (string, error) {
-	walBase := cfg.WALBaseDir
-	if walBase == "" {
-		walBase = cfg.BaseDir
-	}
-	dir, err := localVolumeDir(walBase, volumeName)
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(dir, "wal"), nil
-}
-
-// latestWALChange is the newest modification time under the volume's local
-// WAL, the directory itself included, so a file created or removed counts too.
-func latestWALChange(cfg *Config, volumeName string) (time.Time, error) {
-	dir, err := localWALDir(cfg, volumeName)
-	if err != nil {
-		return time.Time{}, err
-	}
-	var latest time.Time
-	err = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		info, err := d.Info()
-		if err != nil {
-			return err
-		}
-		if info.ModTime().After(latest) {
-			latest = info.ModTime()
-		}
-		return nil
-	})
-	return latest, err
-}
-
 // sealReceiptSuffix names the file the nbdkit plugin leaves at
 // baseDir/<volume>.sealed after a successful seal.
 const sealReceiptSuffix = ".sealed"
@@ -643,51 +584,17 @@ func consumeSealReceipt(baseDir, volume string) bool {
 	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
 		slog.Warn("failed to remove seal receipt", "volume", volume, "path", path, "err", err)
 	}
-	_, ok := validSealReceipt(data, volume, path)
-	return ok
-}
 
-// sealReceiptCoversWAL reports whether the volume's seal receipt postdates
-// every change to its local WAL, leaving the receipt for the unmount it
-// attests to. An older receipt is an earlier client's, not the last one's.
-func sealReceiptCoversWAL(cfg *Config, volume string) bool {
-	path := sealReceiptPath(cfg.BaseDir, volume)
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return false
-	}
-	receipt, ok := validSealReceipt(data, volume, path)
-	if !ok {
-		return false
-	}
-	latest, err := latestWALChange(cfg, volume)
-	if err != nil {
-		slog.Warn("could not read local WAL to date its seal receipt", "volume", volume, "err", err)
-		return false
-	}
-	// The plugin writes SealedAt to the millisecond, after the seal's own WAL
-	// changes, so a WAL change in the receipt's millisecond is the seal's.
-	if receipt.SealedAt.Before(latest.Truncate(time.Millisecond)) {
-		slog.Warn("seal receipt predates the local WAL, treating the WAL as unsealed",
-			"volume", volume, "sealed_at", receipt.SealedAt, "wal_changed_at", latest)
-		return false
-	}
-	return true
-}
-
-// validSealReceipt decodes data, reporting whether it is a receipt the plugin
-// wrote for volume.
-func validSealReceipt(data []byte, volume, path string) (sealReceipt, bool) {
 	var receipt sealReceipt
 	if err := json.Unmarshal(data, &receipt); err != nil {
 		slog.Warn("malformed seal receipt", "volume", volume, "path", path, "err", err)
-		return sealReceipt{}, false
+		return false
 	}
 	if receipt.Volume != volume || receipt.SealedAt.IsZero() {
 		slog.Warn("seal receipt missing required fields", "volume", volume, "path", path, "receipt", receipt)
-		return sealReceipt{}, false
+		return false
 	}
-	return receipt, true
+	return true
 }
 
 // clearStaleSealReceipt removes any seal receipt left by a previous mount of

spinifex/runtime/compute/vm/desired_state_test.go

@@ -107,7 +107,7 @@ func TestVMMarshal_LegacyHostfwdNotWrittenBack(t *testing.T) {
 	assert.Contains(t, raw, "hostfwd_port_map")
 }
 
-// The other eleven command booleans had nowhere to go because nothing read them
+// The other twelve command booleans had nowhere to go because nothing read them
 // back. A record must carry none of them, so they are gone rather than unread.
 func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	legacy, err := json.Marshal(map[string]any{
@@ -115,6 +115,7 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 		"attributes": ec2v1.EC2CommandAttributes{
 			AttachVolume:                true,
 			DetachVolume:                true,
+			DrainVolume:                 true,
 			StartInstance:               true,
 			RebootInstance:              true,
 			AttachENI:                   true,
@@ -136,7 +137,7 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, field := range []string{
-		"attach_volume", "detach_volume", "start_instance",
+		"attach_volume", "detach_volume", "drain_volume", "start_instance",
 		"reboot_instance", "attach_eni", "detach_eni",
 		"associate_iam_instance_profile", "set_spot_lineage", "set_instance_tags",
 		"remove_instance_tags", "set_instance_monitoring",

spinifex/runtime/compute/vm/errors.go

@@ -32,10 +32,11 @@ var ErrVolumeNotDetachable = errors.New("volume cannot be detached")
 // supplies a device name that does not match the recorded attachment.
 var ErrVolumeDeviceMismatch = errors.New("volume device name mismatch")
 
-// ErrMountAmbiguous is returned by VolumeMounter.Mount and MountOne when a sent
-// publish got no usable export back (bad reply, timeout, cancelled caller), so
-// the provider may have started one; it is unpublished so it is not orphaned.
-var ErrMountAmbiguous = errors.New("publish reply carried no usable export")
+// ErrMountAmbiguous is returned by VolumeMounter.MountOne when the
+// backend acknowledged the mount but returned an empty URI. The mount may
+// or may not have started serving NBD; AttachVolume must invoke UnmountOne
+// defensively so a half-started backend mount is not orphaned.
+var ErrMountAmbiguous = errors.New("ebs.mount succeeded with empty URI")
 
 // ErrENINotAttached is returned by HotUnplugENI when the supplied ENI id
 // is not present in the instance's ENIRequests.AttachedByENIID map.

spinifex/runtime/compute/vm/manager.go

@@ -36,6 +36,7 @@ type Deps struct {
 	Resources          ResourceController
 	VolumeStateUpdater VolumeStateUpdater
 	InstanceCleaner    InstanceCleaner
+	UEFIStores         UEFIStoreTerminator
 	Hooks              ManagerHooks
 
 	// ShutdownSignal returns true once the daemon has begun coordinated

spinifex/runtime/compute/vm/record.go

@@ -106,6 +106,10 @@ type InstanceStatus struct {
 	EBSRequests       []EBSRequest   `json:"ebs_requests,omitempty"`
 	ENIAvailableSlots []int          `json:"eni_available_slots,omitempty"`
 	ENIAttachedByID   map[string]int `json:"eni_attached_by_id,omitempty"`
+
+	// UEFIVarStore is optional, so records written without it still decode
+	// and the bucket schema does not change.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
 }
 
 // Record projects a VM onto the persisted record. QMPClient and the request
@@ -166,6 +170,7 @@ func (v *VM) Record() *InstanceRecord {
 			EBSRequests:                     v.EBSRequests.Requests,
 			ENIAvailableSlots:               v.ENIRequests.AvailableSlots,
 			ENIAttachedByID:                 v.ENIRequests.AttachedByENIID,
+			UEFIVarStore:                    v.UEFIVarStore,
 		},
 	}
 }
@@ -222,6 +227,7 @@ func VMFromRecord(r *InstanceRecord) *VM {
 		TerminatedAt:                    r.Status.TerminatedAt,
 		ShuttingDownAt:                  r.Status.ShuttingDownAt,
 		FirstLaunch:                     r.Status.FirstLaunch,
+		UEFIVarStore:                    r.Status.UEFIVarStore,
 	}
 	v.EBSRequests.Requests = r.Status.EBSRequests
 	v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots

spinifex/runtime/compute/vm/record_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -114,6 +115,37 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
 	assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
 }
 
+// The pointer is optional: a record written before it existed decodes with
+// none, and a record without one writes no key, so the bucket schema holds.
+func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
+	var decoded vm.InstanceRecord
+	require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
+	assert.Nil(t, decoded.Status.UEFIVarStore)
+
+	encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
+	require.NoError(t, err)
+	assert.NotContains(t, string(encoded), "uefi_var_store")
+}
+
+// The pointer's wire names are what the record carries across releases, so
+// they are pinned rather than left to follow the Go field names.
+func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
+	encoded, err := json.Marshal(populatedVM().Record())
+	require.NoError(t, err)
+
+	var wire struct {
+		Status struct {
+			UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
+		} `json:"status"`
+	}
+	require.NoError(t, json.Unmarshal(encoded, &wire))
+	ref := wire.Status.UEFIVarStore
+	assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
+	assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
+	assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
+	assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
+}
+
 // The mutexes are dropped on the way out, so the rebuilt instance must arrive
 // with usable zero-value locks rather than ones somebody else left held.
 func TestVMFromRecord_RequestWrappersAreUsable(t *testing.T) {
@@ -206,6 +238,12 @@ func populatedVM() *vm.VM {
 		CapacityReservationId: "cr-1",
 		InstanceLifecycle:     "spot",
 		SpotInstanceRequestId: "sir-1",
+		UEFIVarStore: &vm.UEFIVarStoreRef{
+			Key:        "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
+			Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
+			WriterNode: "node-3",
+			WrittenAt:  time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
+		},
 
 		Status:                          vm.StateRunning,
 		Instance:                        &ec2.Instance{InstanceId: aws.String("i-round-trip")},

spinifex/runtime/compute/vm/shutdown.go

@@ -433,6 +433,7 @@ func (m *Manager) finalizeTerminated(instance *VM) error {
 // already spent and terminated is terminal.
 func (m *Manager) reconcileVanishedQEMU(instance *VM) error {
 	m.markTeardown(instance, TeardownQEMU, TeardownDone)
+	m.markUEFIStoreForDeletion(instance)
 	m.stampOutstandingTeardownFailed(instance)
 	return m.finalizeTerminated(instance)
 }
@@ -460,6 +461,7 @@ func (m *Manager) forceFinalizeStuckTerminate(instance *VM) error {
 	if m.deps.InstanceCleaner != nil {
 		m.markTeardownResult(instance, TeardownVolumes, m.deps.InstanceCleaner.DeleteVolumes(instance))
 	}
+	m.markUEFIStoreForDeletion(instance)
 	m.stampOutstandingTeardownFailed(instance)
 	return m.finalizeTerminated(instance)
 }
@@ -548,8 +550,8 @@ func (m *Manager) releaseAutoAssignedPublicIP(instance *VM) {
 }
 
 // terminateCleanup is stopCleanup plus the AWS-resource cleanup that
-// only applies on terminate: volume deletion, public IP release, ENI
-// deletion, placement-group removal.
+// only applies on terminate: volume deletion, the UEFI store's deletion
+// record, public IP release, ENI deletion, placement-group removal.
 func (m *Manager) terminateCleanup(instance *VM) {
 	// Terminate deletes the volumes next, so an unsealed block map loses
 	// nothing a caller can still ask for. Tolerated to keep terminate
@@ -560,6 +562,7 @@ func (m *Manager) terminateCleanup(instance *VM) {
 	if m.deps.InstanceCleaner != nil {
 		m.markTeardownResult(instance, TeardownVolumes, m.deps.InstanceCleaner.DeleteVolumes(instance))
 	}
+	m.markUEFIStoreForDeletion(instance)
 
 	m.cleanupTapDevices(instance)
 	m.markTeardown(instance, TeardownTap, TeardownDone)

spinifex/runtime/compute/vm/teardown.go

@@ -23,6 +23,9 @@ const (
 	TeardownENI       = "eni"
 	TeardownOVN       = "ovn"
 	TeardownPlacement = "placement"
+	// TeardownUEFI is pending once the variable store is recorded for
+	// deletion; the cluster-wide store reaper marks it done.
+	TeardownUEFI = "uefi"
 )
 
 // markTeardown records dep → state on the VM under the manager lock. Inspect

spinifex/runtime/compute/vm/uefi_terminate.go

@@ -0,0 +1,35 @@
+package vm
+
+import "log/slog"
+
+// UEFIStoreTerminator settles a terminated instance's UEFI variable store. The
+// objects are deleted by a cluster-wide reaper, so terminate only records the
+// obligation and removes this node's local copy.
+type UEFIStoreTerminator interface {
+	// MarkUEFIStoreDeleting moves the instance's store record to deleting. It
+	// reports false with no error when the instance has no store.
+	MarkUEFIStoreDeleting(instanceID string) (bool, error)
+	// RemoveLocalUEFIStore removes this node's copy of the store. Idempotent.
+	RemoveLocalUEFIStore(instanceID string) error
+}
+
+// markUEFIStoreForDeletion records a terminating instance's store for the
+// reaper. A failed mark is stamped failed, which the reaper completes from the
+// terminated record; an instance with no store gets no teardown entry.
+func (m *Manager) markUEFIStoreForDeletion(instance *VM) {
+	if m.deps.UEFIStores == nil {
+		return
+	}
+	has, err := m.deps.UEFIStores.MarkUEFIStoreDeleting(instance.ID)
+	switch {
+	case err != nil:
+		slog.Warn("Failed to record UEFI variable store for deletion; the store reaper completes it",
+			"instanceId", instance.ID, "err", err)
+		m.markTeardown(instance, TeardownUEFI, TeardownFailed)
+	case has:
+		m.markTeardown(instance, TeardownUEFI, TeardownPending)
+	}
+	if err := m.deps.UEFIStores.RemoveLocalUEFIStore(instance.ID); err != nil {
+		slog.Warn("Failed to remove local UEFI variable store", "instanceId", instance.ID, "err", err)
+	}
+}

spinifex/runtime/compute/vm/uefi_terminate_test.go

@@ -0,0 +1,153 @@
+package vm
+
+import (
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeUEFIStores records each terminator call; has and markErr shape the mark.
+type fakeUEFIStores struct {
+	mu      sync.Mutex
+	has     bool
+	markErr error
+	marked  []string
+	removed []string
+}
+
+var _ UEFIStoreTerminator = (*fakeUEFIStores)(nil)
+
+func (f *fakeUEFIStores) MarkUEFIStoreDeleting(id string) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.marked = append(f.marked, id)
+	return f.has && f.markErr == nil, f.markErr
+}
+
+func (f *fakeUEFIStores) RemoveLocalUEFIStore(id string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.removed = append(f.removed, id)
+	return nil
+}
+
+func (f *fakeUEFIStores) calls() (marked, removed []string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.marked...), append([]string(nil), f.removed...)
+}
+
+func withUEFIStores(m *Manager, f *fakeUEFIStores) {
+	deps := m.deps
+	deps.UEFIStores = f
+	m.SetDeps(deps)
+}
+
+func runningUEFIInstance(id string) *VM {
+	return &VM{ID: id, Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+}
+
+// TestS8_TerminateRecordsTheUEFIStoreObligationAndReturns enforces ADR-0003:S8
+// for the store: terminate deletes no object, it records the obligation and
+// returns, and the terminated record carries it for the reaper to converge.
+func TestS8_TerminateRecordsTheUEFIStoreObligationAndReturns(t *testing.T) {
+	store := newFakeStateStore()
+	m, _, _, _, _ := terminateTestManager(t, store)
+	stores := &fakeUEFIStores{has: true}
+	withUEFIStores(m, stores)
+	v := runningUEFIInstance("i-uefi-terminate")
+	m.Insert(v)
+
+	require.NoError(t, m.Terminate(v.ID))
+
+	marked, removed := stores.calls()
+	assert.Equal(t, []string{v.ID}, marked, "terminate must record the store for deletion")
+	assert.Equal(t, []string{v.ID}, removed, "the node that ran the instance removes its local copy")
+	terminated := store.terminated[v.ID]
+	require.NotNil(t, terminated, "terminate must write the terminated record")
+	assert.Equal(t, string(TeardownPending), terminated.Teardown[TeardownUEFI],
+		"ADR-0003:S8: the terminated record must carry the store obligation while the reaper converges it")
+	assert.False(t, terminated.TeardownComplete(), "a pending store keeps the record from early purge")
+}
+
+func TestTerminate_InstanceWithNoUEFIStoreStampsNoEntry(t *testing.T) {
+	store := newFakeStateStore()
+	m, _, _, _, _ := terminateTestManager(t, store)
+	withUEFIStores(m, &fakeUEFIStores{has: false})
+	v := runningUEFIInstance("i-bios")
+	m.Insert(v)
+
+	require.NoError(t, m.Terminate(v.ID))
+
+	_, stamped := store.terminated[v.ID].Teardown[TeardownUEFI]
+	assert.False(t, stamped, "a BIOS instance has no store, so nothing is owed")
+}
+
+// A mark that fails is stamped failed rather than dropped, so the reaper can
+// complete it from the terminated record.
+func TestTerminate_FailedUEFIMarkIsStampedFailed(t *testing.T) {
+	store := newFakeStateStore()
+	m, _, _, _, _ := terminateTestManager(t, store)
+	withUEFIStores(m, &fakeUEFIStores{has: true, markErr: errors.New("kv unavailable")})
+	v := runningUEFIInstance("i-uefi-mark-fails")
+	m.Insert(v)
+
+	require.NoError(t, m.Terminate(v.ID), "a store mark never fails the terminate")
+
+	assert.Equal(t, string(TeardownFailed), store.terminated[v.ID].Teardown[TeardownUEFI])
+}
+
+// A terminate that wedged before its cleanup still records the store when the
+// backstop finalises it.
+func TestReconcileVanishedQEMU_RecordsTheUEFIStoreForDeletion(t *testing.T) {
+	store := newFakeStateStore()
+	m, _, _, _, _ := terminateTestManager(t, store)
+	stores := &fakeUEFIStores{has: true}
+	withUEFIStores(m, stores)
+	v := runningUEFIInstance("i-uefi-wedged")
+	v.Status = StateShuttingDown
+	m.Insert(v)
+
+	require.NoError(t, m.reconcileVanishedQEMU(v))
+
+	marked, _ := stores.calls()
+	assert.Equal(t, []string{v.ID}, marked)
+	assert.Equal(t, string(TeardownPending), store.terminated[v.ID].Teardown[TeardownUEFI])
+}
+
+// TestS7_OnlyTerminateRecordsTheUEFIStoreForDeletion enforces ADR-0003:S7 at
+// the node: a stop, a node-wide stop and a failed recovery of an instance
+// that has run never move its store toward deletion or remove its local copy.
+func TestS7_OnlyTerminateRecordsTheUEFIStoreForDeletion(t *testing.T) {
+	m, _, _, _, rt := shutdownTestManager(t)
+	stores := &fakeUEFIStores{has: true}
+	withUEFIStores(m, stores)
+
+	stopped := runningUEFIInstance("i-uefi-stop")
+	m.Insert(stopped)
+	require.NoError(t, m.Stop(stopped.ID))
+
+	drained := runningUEFIInstance("i-uefi-drain")
+	m.Insert(drained)
+	require.NoError(t, m.StopAll())
+
+	failed := &VM{ID: "i-uefi-has-run", Status: StatePending, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+	m.Insert(failed)
+	errored := rt.waitFor(failed.ID, StateError)
+	m.MarkFailed(t.Context(), failed, "relaunch_timeout")
+	select {
+	case <-errored:
+	case <-time.After(markFailedDeadline):
+		t.Fatalf("MarkFailed on an instance that has run did not reach error within %s", markFailedDeadline)
+	}
+	m.goroutineWg.Wait()
+
+	marked, removed := stores.calls()
+	assert.Empty(t, marked, "ADR-0003:S7: only a decided terminate may record a store for deletion")
+	assert.Empty(t, removed, "ADR-0003:S7: a stop or failure keeps the node's copy of the store")
+}

spinifex/runtime/compute/vm/vm.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
@@ -308,6 +309,20 @@ type VM struct {
 	// tolerated by migrate.
 	InstanceLifecycle     string `json:"instance_lifecycle,omitempty"`
 	SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
+
+	// UEFIVarStore points at the object holding this instance's UEFI variable
+	// store. Status, so written only by the node; nil for a BIOS instance.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
+}
+
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string                     `json:"key"`
+	Firmware   uefistore.FirmwareIdentity `json:"firmware"`
+	WriterNode string                     `json:"writer_node"`
+	WrittenAt  time.Time                  `json:"written_at"`
 }
 
 // UnmarshalJSON folds the fields of legacy records that mixed a request with an

spinifex/runtime/compute/vm/volume_mounter.go

@@ -5,8 +5,8 @@ import (
 )
 
 // VolumeMounter mounts and unmounts the EBS volumes attached to a VM. The
-// real implementation publishes and unpublishes them through the EBS provider
-// over NATS; the abstraction keeps NATS out of the manager.
+// real implementation routes ebs.mount / ebs.unmount NATS requests; the
+// abstraction keeps NATS out of the manager.
 //
 // Every method takes the caller's context so the ebs.* span joins the trace
 // that asked for the work rather than rooting one of its own. Shutdown and
@@ -14,9 +14,9 @@ import (
 // honest answer rather than a missing one.
 type VolumeMounter interface {
 	// Mount mounts every attached volume in v.EBSRequests.Requests, recording
-	// the resolved export back onto each request entry.
+	// the resolved NBDURI back onto each request entry.
 	Mount(ctx context.Context, v *VM) error
-	// Unmount unmounts each attached volume, which drives the
+	// Unmount sends ebs.unmount for each attached volume, which drives the
 	// synchronous block-map seal. Every volume is attempted; the seal
 	// failures are aggregated and returned, and mean the data is not durable.
 	Unmount(ctx context.Context, v *VM) error
@@ -31,12 +31,12 @@ type VolumeMounter interface {
 	// anywhere.
 	Abandon(ctx context.Context, v *VM, reason string) error
 
-	// MountOne mounts a single request and writes the resolved export back
-	// into req on success. Used by hot-attach. accountID
+	// MountOne sends ebs.mount for a single request and writes the resolved
+	// NBDURI back into req.NBDURI on success. Used by hot-attach. accountID
 	// names the owner, which a lone request carries no instance to supply.
 	MountOne(ctx context.Context, accountID string, req *EBSRequest) error
-	// UnmountOne unmounts a single request and returns any error. The
-	// unmount drives the synchronous block-map seal to the backend, so hot
+	// UnmountOne sends ebs.unmount for a single request and returns any error.
+	// ebs.unmount drives the synchronous block-map seal to predastore, so hot
 	// detach gates the volume's available transition on the returned error.
 	UnmountOne(ctx context.Context, accountID string, req EBSRequest) error
 }

spinifex/runtime/compute/vm/volumes.go

@@ -25,7 +25,7 @@ const blockdevDelMaxAttempts = 20
 // object-del steps), so a wedged-but-responsive QEMU cannot stack per-step
 // timeouts into an effectively unbounded hang. It comfortably covers those
 // steps' own budgets (qmpCommandTimeout + the bounded blockdev-del retry) plus
-// margin; the volume seal is not bound by it and keeps its own budget.
+// margin; the ebs.unmount seal keeps its separate unmountSealTimeout.
 const detachAggregateTimeout = 3 * time.Minute
 
 // ErrHotAttachUnsupported is returned by AttachVolume for a direct-boot microvm,
@@ -119,9 +119,9 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 		return "", fmt.Errorf("VolumeMounter not wired")
 	}
 	if err := m.deps.VolumeMounter.MountOne(ctx, instance.AccountID, &ebsRequest); err != nil {
-		slog.ErrorContext(ctx, "AttachVolume: publish failed", "volumeId", volumeID, "err", err)
-		// A publish with no usable reply may have started an export, so it is
-		// unpublished defensively rather than orphaned.
+		slog.ErrorContext(ctx, "AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err)
+		// Empty-URI response leaves backend NBD state ambiguous; unmount
+		// defensively to avoid orphaning a half-started mount.
 		if errors.Is(err, ErrMountAmbiguous) {
 			_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
 		}
@@ -300,7 +300,7 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 }
 
 // DetachVolume hot-unplugs a volume via the QMP pipeline (device_del →
-// blockdev-del → object-del → unpublish). Returns the recorded AWS-API device
+// blockdev-del → object-del → ebs.unmount). Returns the recorded AWS-API device
 // name. force=true tolerates device_del failure; blockdev-del is always required.
 func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string, force bool) (string, error) {
 	instance, ok := m.Get(id)
@@ -344,8 +344,8 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// Bound the ctx-driven portion of the hot-unplug chain so a wedged-but-
 	// responsive QEMU cannot stack per-step QMP timeouts into an unbounded hang
-	// that pins attachMu forever. The volume seal below is not bound by it and
-	// keeps its own budget. This deadline cannot preempt a plain-mutex QMP call,
+	// that pins attachMu forever. The ebs.unmount seal below keeps its own
+	// unmountSealTimeout. This deadline cannot preempt a plain-mutex QMP call,
 	// which is why a fully dead QEMU is short-circuited outright below.
 	ctx, cancel := context.WithTimeout(ctx, detachAggregateTimeout)
 	defer cancel()
@@ -359,7 +359,7 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// A confirmed-dead QEMU took every block node and guest device down with it,
 	// so the QMP unplug steps are moot — and worse, issuing them only risks
-	// hanging on the wedged process. Skip straight to the unpublish seal and
+	// hanging on the wedged process. Skip straight to the ebs.unmount seal and
 	// the attachment clear. This is the detach-wedges-when-QEMU-dies fix: the
 	// context deadline above cannot interrupt a plain-mutex QMP path, so a
 	// provably-gone process must bypass it rather than wait on it. A missing PID
@@ -422,15 +422,15 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 			"volumeId", volumeID, "instanceId", instance.ID)
 	}
 
-	// Unpublish drives the synchronous block-map seal to the backend. On
+	// ebs.unmount drives the synchronous block-map seal to predastore. On
 	// failure the volume's local WAL is retained, so keep the volume attached
 	// and return the error; an AWS-CLI retry re-drives the seal and same-node
 	// reattach still works meanwhile.
 	if m.deps.VolumeMounter != nil {
 		if err := m.deps.VolumeMounter.UnmountOne(ctx, instance.AccountID, ebsReq); err != nil {
-			slog.ErrorContext(ctx, "DetachVolume: unpublish seal failed, leaving volume attached",
+			slog.ErrorContext(ctx, "DetachVolume: ebs.unmount seal failed, leaving volume attached",
 				"volumeId", volumeID, "err", err)
-			return "", fmt.Errorf("unpublish seal: %w", err)
+			return "", fmt.Errorf("ebs.unmount seal: %w", err)
 		}
 	}
 
@@ -554,7 +554,7 @@ func (m *Manager) tryBlockdevDel(ctx context.Context, instance *VM, nodeName str
 			return nil
 		}
 		// A prior detach already removed the block node (e.g. an AWS-CLI retry
-		// after the unpublish seal failed): treat as success so the retry
+		// after the ebs.unmount seal failed): treat as success so the retry
 		// resumes through object-del to the seal instead of wedging on a
 		// now-absent node.
 		if isQMPNodeNotFound(err) {

spinifex/runtime/compute/vm/volumes_test.go

@@ -543,7 +543,7 @@ func TestDetachVolume_DeviceGuards(t *testing.T) {
 }
 
 // TestDetachVolume_SealFailureGatesAvailable covers the durability gate in the
-// detach-seal fix: when unpublish (the synchronous block-map seal) fails,
+// detach-seal fix: when ebs.unmount (the synchronous block-map seal) fails,
 // DetachVolume must propagate the error, must NOT flip the volume to "available",
 // and must leave it in EBSRequests so the volume stays attached/retryable and its
 // local WAL is retained. A regression back to fire-and-forget unmount would let a
@@ -661,7 +661,7 @@ func TestAttachVolume_MountAmbiguous_TriggersUnmountOne(t *testing.T) {
 // MountOne failures (NATS timeout, backend explicit Error, marshal errors)
 // do NOT trigger UnmountOne — only the empty-URI ambiguous case rolls back.
 func TestAttachVolume_GenericMountError_DoesNotUnmount(t *testing.T) {
-	mounter := &fakeVolumeMounter{mountOneErr: errors.New("publish request: timeout")}
+	mounter := &fakeVolumeMounter{mountOneErr: errors.New("ebs.mount NATS request: timeout")}
 	m, _ := attachVolumeRunningInstance(t, nil, mounter)
 
 	_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
@@ -1387,7 +1387,7 @@ func TestDetachVolume_DeviceDeletedTimeout_FallsBackToRetry(t *testing.T) {
 // detach-wedges-when-QEMU-dies fix: when QEMU is provably gone (PID file
 // present, process dead), DetachVolume issues no QMP command — those steps
 // would only hang on the wedged process — and proceeds straight to the
-// unpublish seal and the attachment clear.
+// ebs.unmount seal and the attachment clear.
 func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
@@ -1418,7 +1418,7 @@ func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "/dev/sdf", device)
 	assert.False(t, qmpCalled.Load(), "a confirmed-dead QEMU must receive no QMP command")
-	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the unpublish seal must still run")
+	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the ebs.unmount seal must still run")
 	calls := updater.snapshot()
 	require.Len(t, calls, 1, "the stale attachment must be cleared exactly once")
 	assert.Equal(t, "available", calls[0].State, "the volume must be returned to available")

spinifex/runtime/daemon/daemon.go

@@ -2211,6 +2211,9 @@ func (d *Daemon) startCluster() error {
 	}
 
 	d.instanceService.SetTerminationDeps(d.volumeService, d.vpcService, ipReleaser, d.tagsService)
+	if d.jsManager != nil && d.jsManager.UEFIStores() != nil {
+		d.instanceService.SetUEFIStores(d.jsManager.UEFIStores())
+	}
 	d.instanceService.SetVolumeCreator(d.volumeService)
 	d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, ipAllocator)
 
@@ -2581,6 +2584,9 @@ func (d *Daemon) startCluster() error {
 		if gpuRec := d.newGPUPoolReconciler(); gpuRec != nil {
 			reapers = append(reapers, gpuRec)
 		}
+		if uefiStores := d.newUEFIStoreReaper(); uefiStores != nil {
+			reapers = append(reapers, uefiStores)
+		}
 		if d.volumeService != nil {
 			reapers = append(reapers, d.volumeService.NewVolumeLeakReaper(d.leakedVolumeInstances))
 			reapers = append(reapers, d.volumeService.NewVolumeReclaimReaper())
@@ -2622,9 +2628,9 @@ const ebsProviderRequestTimeout = 30 * time.Second
 // a missing responder must be reported promptly, and the check is advisory.
 const ebsProviderProbeTimeout = 2 * time.Second
 
-// configureEBSProvider wires a NATSProvider into the EBS-adjacent services and
-// the volume mounter. The control plane never constructs a provider
-// implementation, so this runs whichever daemon answers on the wire.
+// configureEBSProvider wires a single NATSProvider into every EBS-adjacent
+// service. The control plane never constructs a provider implementation
+// itself, so this always runs regardless of which daemon answers on the wire.
 func (d *Daemon) configureEBSProvider() error {
 	provider := ebsprovider.NewNATSProvider(d.natsConn, ebsProviderRequestTimeout)
 	d.ebsProvider = provider
@@ -2754,6 +2760,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitClusterConfigBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitUEFIStoreBucket()
+		}
+
 		if err == nil {
 			d.jsManager.SetSyncObserver(d)
 			// Replicas cannot be undeclared here: bucket creation refuses

spinifex/runtime/daemon/daemon_handlers.go

@@ -85,13 +85,9 @@ func respondWithServiceError(nodeID string, msg *nats.Msg, err error) {
 	payload := awserrors.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), message)
 	// The AWS body stays as it is; the header is how ec2.start tells its
 	// forwarder this refusal must not be answered by starting elsewhere.
-	extra := nats.Header{}
+	var extra nats.Header
 	if _, ok := errors.AsType[*ec2instance.UnsealedWritesError](err); ok {
-		extra.Set(natsmsg.UnsealedWritesHeader, "true")
-	}
-	// A start caller cannot tell a held volume from any other IncorrectState.
-	if errors.Is(err, ec2v1.ErrVolumeHeldElsewhere) {
-		extra.Set(natsmsg.VolumeHeldElsewhereHeader, "true")
+		extra = nats.Header{natsmsg.UnsealedWritesHeader: []string{"true"}}
 	}
 	respondNATSMsgWithHeader(nodeID, msg, payload, extra)
 }
@@ -195,6 +191,9 @@ func handleNATSRequestWithPrincipal[I any, O any](nodeID string, serviceFn func(
 func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 	start := time.Now()
 	action, outcome := d.dispatchEC2Command(msg)
+	if outcome == outcomeDeferred {
+		return
+	}
 	otelsetup.RecordRequest(context.Background(), ec2CmdAction(action), outcome, time.Since(start))
 }
 
@@ -234,6 +233,21 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) {
 		return name, d.handleAttachVolume(ctx, msg, command, instance)
 	case command.Attributes.DetachVolume:
 		return name, d.handleDetachVolume(ctx, msg, command, instance)
+	case command.Attributes.DrainVolume:
+		// A drain flushes the volume's whole dirty set to S3, so it is bounded by
+		// the dirty set rather than by a unit of work. nats.go delivers a
+		// subscription's messages serially, so running it inline would stall
+		// every other command for this instance — stop, terminate, hot-plug —
+		// for the duration. It touches no VM state, so it is safe off-thread.
+		//
+		// It therefore records its own point: timing the dispatch would call
+		// every drain an instant success.
+		started := time.Now()
+		go func() {
+			outcome := d.handleDrainVolume(ctx, msg, command, instance)
+			otelsetup.RecordRequest(context.Background(), ec2CmdAction(name), outcome, time.Since(started))
+		}()
+		return name, outcomeDeferred
 	case command.Attributes.AttachENI:
 		return name, d.handleAttachNetworkInterface(ctx, msg, command, instance)
 	case command.Attributes.DetachENI:
@@ -300,6 +314,8 @@ func ec2CommandName(command ec2v1.EC2InstanceCommand) string {
 		return "AttachVolume"
 	case command.Attributes.DetachVolume:
 		return "DetachVolume"
+	case command.Attributes.DrainVolume:
+		return "DrainVolume"
 	case command.Attributes.AttachENI:
 		return "AttachNetworkInterface"
 	case command.Attributes.DetachENI:

spinifex/runtime/daemon/daemon_handlers_instance.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
-	"maps"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -315,7 +314,7 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) string {
 				// checkpoint and lose what this node has not sealed.
 				slog.Warn("ec2.start: owner at capacity holds unsealed writes; not starting elsewhere",
 					"instanceId", peek.InstanceID, "lastNode", lastNode)
-				if relayErr := relayStartReply(msg, resp); relayErr != nil {
+				if relayErr := msg.Respond(resp.Data); relayErr != nil {
 					slog.Error("ec2.start: failed to relay response from original node",
 						"instanceId", peek.InstanceID, "lastNode", lastNode, "err", relayErr)
 					return outcomeError
@@ -324,7 +323,7 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) string {
 			}
 			if !isCapacity {
 				// Relay success or any non-capacity error as-is.
-				if relayErr := relayStartReply(msg, resp); relayErr != nil {
+				if relayErr := msg.Respond(resp.Data); relayErr != nil {
 					slog.Error("ec2.start: failed to relay response from original node",
 						"instanceId", peek.InstanceID, "lastNode", lastNode, "err", relayErr)
 					return outcomeError
@@ -351,12 +350,3 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) string {
 
 	return handleNATSRequest(d.node, d.instanceService.StartStoppedInstance)(msg)
 }
-
-// relayStartReply answers msg with the owner's reply, body and every header,
-// since the headers carry what the AWS body cannot, such as why it refused.
-func relayStartReply(msg, resp *nats.Msg) error {
-	reply := nats.NewMsg("")
-	reply.Data = resp.Data
-	maps.Copy(reply.Header, resp.Header)
-	return msg.RespondMsg(reply)
-}

spinifex/runtime/daemon/daemon_handlers_instance_test.go

@@ -1,20 +1,15 @@
 package daemon
 
 import (
-	"context"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
@@ -591,17 +586,10 @@ func TestHandleEC2StartStoppedInstance_Forwarding(t *testing.T) {
 
 	t.Run("owner success is relayed", func(t *testing.T) {
 		d, _, body := newFixture(t, "i-p1-fwd-ok", "node-p1-ok")
-		owner(t, d, "node-p1-ok", func(m *nats.Msg) {
-			resp := nats.NewMsg(m.Reply)
-			resp.Data = []byte(`{"started":true}`)
-			resp.Header.Set(natsmsg.NodeIDHeader, "node-p1-ok")
-			_ = m.RespondMsg(resp)
-		})
+		owner(t, d, "node-p1-ok", func(m *nats.Msg) { _ = m.Respond([]byte(`{"started":true}`)) })
 
 		reply := requestHandler(t, d.natsConn, "ec2.start.p1-ok", asMsgHandler(d.handleEC2StartStoppedInstance), testAccountID, body)
 		assert.JSONEq(t, `{"started":true}`, string(reply.Data))
-		assert.Equal(t, "node-p1-ok", reply.Header.Get(natsmsg.NodeIDHeader),
-			"the relayed verdict is the owner's, headers included")
 	})
 
 	t.Run("owner at capacity falls back locally", func(t *testing.T) {
@@ -628,15 +616,12 @@ func TestHandleEC2StartStoppedInstance_Forwarding(t *testing.T) {
 			resp := nats.NewMsg(m.Reply)
 			resp.Data = awserrors.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity)
 			resp.Header.Set(natsmsg.UnsealedWritesHeader, "true")
-			resp.Header.Set(natsmsg.NodeIDHeader, "node-p1-dirty")
 			_ = m.RespondMsg(resp)
 		})
 
 		reply := requestHandler(t, d.natsConn, "ec2.start.p1-dirty", asMsgHandler(d.handleEC2StartStoppedInstance), testAccountID, body)
 		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, decodeError(t, reply.Data)["Code"],
 			"an owner that still holds unsealed writes must not be bypassed by a local start")
-		assert.Equal(t, "true", reply.Header.Get(natsmsg.UnsealedWritesHeader))
-		assert.Equal(t, "node-p1-dirty", reply.Header.Get(natsmsg.NodeIDHeader))
 	})
 
 	t.Run("owner not subscribed falls back locally", func(t *testing.T) {
@@ -657,71 +642,6 @@ func TestHandleEC2StartStoppedInstance_Forwarding(t *testing.T) {
 	})
 }
 
-// A start refused because another node holds a volume crosses the owner's
-// reply, the ec2.start forwarder's relay and the StartInstances client, and
-// must still match the sentinel at the end, whatever words the provider used.
-func TestStartInstances_VolumeHeldElsewhereSurvivesTheForward(t *testing.T) {
-	ns, nc := testutil.StartTestNATS(t)
-	store := vmmock.New()
-	v := stoppedVMFixture("i-0abc0000000000001", testAccountID)
-	v.LastNode = "node-owner"
-	store.Stopped[v.ID] = v
-	d := createTestDaemon(t, ns.ClientURL())
-	d.stateStore = store
-	d.instanceService = ec2instance.NewInstanceServiceImpl(
-		d.config, d.resourceMgr.instanceTypes, d.natsConn,
-		objectstore.NewMemoryObjectStore(), d.vmMgr, d.resourceMgr, store,
-	)
-
-	held := fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere,
-		awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", "volume vol-1 is published to node-a"))
-	owner := func(context.Context, *ec2instance.StartStoppedInstanceInput, string) (*ec2instance.StartStoppedInstanceOutput, error) {
-		return nil, held
-	}
-	for subject, fn := range map[string]nats.MsgHandler{
-		"ec2.start.node-owner": asMsgHandler(handleNATSRequest("node-owner", owner)),
-		"ec2.start":            asMsgHandler(d.handleEC2StartStoppedInstance),
-	} {
-		sub, err := nc.Subscribe(subject, fn)
-		require.NoError(t, err)
-		t.Cleanup(func() { _ = sub.Unsubscribe() })
-	}
-
-	_, err := ec2instanceapi.StartInstances(t.Context(),
-		&ec2.StartInstancesInput{InstanceIds: []*string{aws.String(v.ID)}}, nc, testAccountID)
-	require.Error(t, err)
-	assert.ErrorIs(t, err, ec2v1.ErrVolumeHeldElsewhere, "the caller cannot read this refusal from its words")
-	code, message, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorIncorrectState, code)
-	assert.Contains(t, message, "node-a", "the operator still learns where the volume is")
-}
-
-// The header is the one signal that a refusal was a volume held elsewhere, so
-// the owner must not put it on any other start failure in the same AWS code.
-func TestStartOwner_HeldElsewhereHeaderOnlyOnAHeldVolume(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
-	for name, tc := range map[string]struct {
-		err  error
-		want string
-	}{
-		"held elsewhere": {fmt.Errorf("%w: %w", ec2v1.ErrVolumeHeldElsewhere,
-			awserrors.Errorf(awserrors.ErrorIncorrectState, "%s", "volume vol-1 is published to node-a")), "true"},
-		"other incorrect state": {awserrors.Errorf(awserrors.ErrorIncorrectState, "%s",
-			"instance i-0abc0000000000003 is not stopped"), ""},
-		"internal fault": {errors.New(awserrors.ErrorServerInternal), ""},
-	} {
-		t.Run(name, func(t *testing.T) {
-			owner := func(context.Context, *ec2instance.StartStoppedInstanceInput, string) (*ec2instance.StartStoppedInstanceOutput, error) {
-				return nil, tc.err
-			}
-			reply := requestHandler(t, nc, "ec2.start.owner-"+strings.ReplaceAll(name, " ", "-"),
-				asMsgHandler(handleNATSRequest("node-owner", owner)), testAccountID, []byte(`{}`))
-			assert.Equal(t, tc.want, reply.Header.Get(natsmsg.VolumeHeldElsewhereHeader))
-		})
-	}
-}
-
 // --- handleSetInstanceTags / handleSetInstanceMonitoring ---
 
 func setTagsCommand(id string) ec2v1.EC2InstanceCommand {

spinifex/runtime/daemon/daemon_handlers_volume.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
@@ -138,6 +139,50 @@ func (d *Daemon) handleDetachVolume(ctx context.Context, msg *nats.Msg, command
 	return outcomeSuccess
 }
 
+// handleDrainVolume flushes the volume's in-flight writes to S3 by dialing the
+// drain socket the local NBD plugin serves. ec2.CreateSnapshot is answered by
+// any node in the cluster, but only the node hosting the instance subscribes
+// ec2.cmd.{instanceID}, so routing the drain here lands it where the writes are.
+//
+// Runs on its own goroutine (see handleEC2Events) so a long flush cannot hold
+// the instance's command subscription.
+func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command ec2v1.EC2InstanceCommand, instance *vm.VM) string {
+	ctx, span := startOpSpan(ctx, "ec2.DrainVolume", command.ID)
+	var err error
+	defer func() { endOpSpan(span, err) }()
+
+	if command.DrainVolumeData == nil || command.DrainVolumeData.VolumeID == "" {
+		slog.ErrorContext(ctx, "DrainVolume: missing drain volume data", "instanceId", command.ID)
+		return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue)
+	}
+
+	volumeID := command.DrainVolumeData.VolumeID
+
+	// Only stable post-teardown states prove the volume was sealed. Transitional
+	// states still have a live or shutting-down writer, so they must drain or
+	// fail rather than acknowledge an older checkpoint as current.
+	status := d.vmMgr.Status(instance)
+	if status == vm.StateStopped || status == vm.StateTerminated {
+		slog.InfoContext(ctx, "DrainVolume: instance teardown is complete, nothing to drain",
+			"volumeId", volumeID, "instanceId", command.ID, "status", status)
+		respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusNotRunning})
+		return outcomeSuccess
+	}
+
+	slog.InfoContext(ctx, "Draining volume before snapshot", "volumeId", volumeID, "instanceId", command.ID)
+
+	// A missing or unresponsive socket under a running instance means the writes
+	// cannot be made current, so the caller must fail its snapshot rather than
+	// read a stale checkpoint.
+	if err = ec2snapshot.DrainVolumeSocket(d.config.DataDir, volumeID); err != nil {
+		slog.ErrorContext(ctx, "DrainVolume: drain failed", "volumeId", volumeID, "instanceId", command.ID, "err", err)
+		return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal)
+	}
+
+	respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
+	return outcomeSuccess
+}
+
 // attachDetachErrorCode maps a vm.Manager error returned by AttachVolume
 // or DetachVolume to the AWS API error code that the SDK expects.
 func attachDetachErrorCode(err error) string {

spinifex/runtime/daemon/daemon_handlers_volume_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
 	"testing"
 	"time"
@@ -11,14 +12,17 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
 )
 
 // TestAttachDetachErrorCode locks down the manager-error → AWS-API-code
@@ -366,6 +370,257 @@ func TestAttachVolume_IdempotentSameInstance_DeviceMismatch(t *testing.T) {
 	assert.Contains(t, string(resp.Data), awserrors.ErrorVolumeInUse)
 }
 
+// drainCommandFor builds the drain command a snapshot addresses to the node
+// hosting instanceID.
+func drainCommandFor(t *testing.T, instanceID, volumeID string) []byte {
+	t.Helper()
+	command := ec2v1.EC2InstanceCommand{
+		ID:              instanceID,
+		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
+		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
+	}
+	data, err := json.Marshal(command)
+	require.NoError(t, err)
+	return data
+}
+
+// drainTestDaemon returns a daemon owning a running instance, with a data dir
+// short enough to hold the drain socket path.
+func drainTestDaemon(t *testing.T, instanceID string) *Daemon {
+	t.Helper()
+	return drainTestDaemonInState(t, instanceID, vm.StateRunning)
+}
+
+// drainTestDaemonInState is drainTestDaemon for an instance the node still
+// holds in some other state.
+func drainTestDaemonInState(t *testing.T, instanceID string, status vm.InstanceState) *Daemon {
+	t.Helper()
+	daemon := createTestDaemon(t, sharedNATSURL)
+	daemon.config.DataDir = testutil.SocketTempDir(t)
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:           instanceID,
+		Status:       status,
+		AccountID:    testAccountID,
+		InstanceType: getTestInstanceType(t),
+		Instance:     &ec2.Instance{},
+	})
+	return daemon
+}
+
+// drainRequest issues a drain on the instance's command subject, returning the
+// raw reply.
+func drainRequest(t *testing.T, daemon *Daemon, instanceID, volumeID string) *nats.Msg {
+	t.Helper()
+	reply, err := sendDrainCommand(daemon, instanceID, drainCommandFor(t, instanceID, volumeID))
+	require.NoError(t, err)
+	return reply
+}
+
+// sendDrainCommand is drainRequest without the assertions, so a test can issue
+// one from a goroutine and assert on the result back on the test goroutine.
+func sendDrainCommand(daemon *Daemon, instanceID string, command []byte) (*nats.Msg, error) {
+	msg := nats.NewMsg("ec2.cmd." + instanceID)
+	msg.Data = command
+	msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
+	return daemon.natsConn.RequestMsg(msg, 5*time.Second)
+}
+
+// subscribeEC2Commands wires the daemon's real dispatcher onto the per-instance
+// command subject, so tests exercise the same serial delivery production has.
+func subscribeEC2Commands(t *testing.T, daemon *Daemon, instanceID string) {
+	t.Helper()
+	sub, err := daemon.natsConn.Subscribe("ec2.cmd."+instanceID, daemon.handleEC2Events)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+// A drain routed to the node hosting the instance reaches the local socket and
+// acks, which is what lets a snapshot answered elsewhere read a current
+// checkpoint.
+func TestDrainVolume_HostNodeAcksLocalSocket(t *testing.T) {
+	const instanceID, volumeID = "i-drain-ok", "vol-drain-ok"
+	daemon := drainTestDaemon(t, instanceID)
+	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
+
+	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+		testAccountID, drainCommandFor(t, instanceID, volumeID))
+
+	var ack ec2v1.DrainVolumeResponse
+	require.NoError(t, json.Unmarshal(resp.Data, &ack))
+	assert.Equal(t, volumeID, ack.VolumeID)
+	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+}
+
+// The owning node having no socket for the volume means the writes cannot be
+// flushed. That must reach the caller as an error, never as a silent success:
+// the caller would otherwise snapshot a stale checkpoint.
+func TestDrainVolume_HostNodeWithoutSocketFails(t *testing.T) {
+	const instanceID, volumeID = "i-drain-nosock", "vol-drain-nosock"
+	daemon := drainTestDaemon(t, instanceID)
+
+	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+		testAccountID, drainCommandFor(t, instanceID, volumeID))
+
+	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
+}
+
+// A plugin that refuses the drain (ERR rather than OK) is a failure, not an ack.
+func TestDrainVolume_HostNodeRelaysSocketFailure(t *testing.T) {
+	const instanceID, volumeID = "i-drain-err", "vol-drain-err"
+	daemon := drainTestDaemon(t, instanceID)
+	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "ERR\n")
+
+	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+		testAccountID, drainCommandFor(t, instanceID, volumeID))
+
+	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
+}
+
+// A drain command with no volume is a caller error, not a server fault.
+func TestDrainVolume_MissingVolumeDataIsInvalidParameter(t *testing.T) {
+	const instanceID = "i-drain-novol"
+	daemon := drainTestDaemon(t, instanceID)
+
+	command := ec2v1.EC2InstanceCommand{
+		ID:         instanceID,
+		Attributes: ec2v1.EC2CommandAttributes{DrainVolume: true},
+	}
+	data, err := json.Marshal(command)
+	require.NoError(t, err)
+
+	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+		testAccountID, data)
+
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, resp.Data))
+}
+
+// Only the instance's owner may drain its volume: the command carries the same
+// ownership gate as every other per-instance command.
+func TestDrainVolume_ForeignAccountRejected(t *testing.T) {
+	const instanceID, volumeID = "i-drain-foreign", "vol-drain-foreign"
+	daemon := drainTestDaemon(t, instanceID)
+	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
+
+	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+		"999988887777", drainCommandFor(t, instanceID, volumeID))
+
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, replyErrCode(t, resp.Data))
+}
+
+// Only stable post-teardown states prove the volume is sealed. These states
+// acknowledge not-running without touching the absent plugin socket.
+func TestDrainVolume_CompletedTeardownAcksNotRunning(t *testing.T) {
+	for _, status := range []vm.InstanceState{vm.StateStopped, vm.StateTerminated} {
+		t.Run(string(status), func(t *testing.T) {
+			instanceID := "i-drain-" + string(status)
+			volumeID := "vol-drain-" + string(status)
+			daemon := drainTestDaemonInState(t, instanceID, status)
+
+			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+				testAccountID, drainCommandFor(t, instanceID, volumeID))
+
+			var ack ec2v1.DrainVolumeResponse
+			require.NoError(t, json.Unmarshal(resp.Data, &ack))
+			assert.Equal(t, volumeID, ack.VolumeID)
+			assert.Equal(t, ec2v1.DrainVolumeStatusNotRunning, ack.Status)
+		})
+	}
+}
+
+// Transitional and uncertain states are not proof of a sealed checkpoint. An
+// absent socket in any of them must fail rather than silently snapshot stale
+// data while launch or teardown may still own the volume.
+func TestDrainVolume_UnsealedStatesRequireDrain(t *testing.T) {
+	statuses := []vm.InstanceState{
+		vm.StateRunning,
+		vm.StateStopping,
+		vm.StateShuttingDown,
+		vm.StatePending,
+		vm.StateProvisioning,
+		vm.StateError,
+	}
+	for _, status := range statuses {
+		t.Run(string(status), func(t *testing.T) {
+			instanceID := "i-drain-" + string(status)
+			volumeID := "vol-drain-" + string(status)
+			daemon := drainTestDaemonInState(t, instanceID, status)
+
+			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
+				testAccountID, drainCommandFor(t, instanceID, volumeID))
+
+			assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
+		})
+	}
+}
+
+// nats.go delivers a subscription's messages serially, so a drain run inline
+// would hold ec2.cmd.{instanceID} for the length of the flush — stalling stop,
+// terminate and hot-plug for that instance. A drain still flushing must not
+// keep the next command waiting.
+func TestDrainVolume_SlowDrainDoesNotBlockTheCommandSubject(t *testing.T) {
+	const instanceID = "i-drain-parallel"
+	daemon := drainTestDaemon(t, instanceID)
+	accepted, release := testutil.StartBlockingDrainSocket(t, daemon.config.DataDir, "vol-drain-slow", "OK\n")
+	testutil.StartDrainSocket(t, daemon.config.DataDir, "vol-drain-fast", "OK\n")
+	subscribeEC2Commands(t, daemon, instanceID)
+
+	type reply struct {
+		msg *nats.Msg
+		err error
+	}
+	slowCommand := drainCommandFor(t, instanceID, "vol-drain-slow")
+	slow := make(chan reply, 1)
+	go func() {
+		msg, err := sendDrainCommand(daemon, instanceID, slowCommand)
+		slow <- reply{msg, err}
+	}()
+
+	// Only assert once the slow drain is genuinely mid-flush.
+	select {
+	case <-accepted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the slow drain never reached its socket")
+	}
+
+	fast := drainRequest(t, daemon, instanceID, "vol-drain-fast")
+	var ack ec2v1.DrainVolumeResponse
+	require.NoError(t, json.Unmarshal(fast.Data, &ack))
+	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+
+	release()
+	select {
+	case got := <-slow:
+		require.NoError(t, got.err)
+		require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
+		assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+	case <-time.After(5 * time.Second):
+		t.Fatal("the released drain never replied")
+	}
+}
+
+// Running the drain off the delivery goroutine must not leak it: the goroutine
+// has to be gone once the ack is on the wire, or a snapshot loop against a
+// wedged plugin accumulates them.
+func TestDrainVolume_DispatchedGoroutineDoesNotLeak(t *testing.T) {
+	const instanceID, volumeID = "i-drain-leak", "vol-drain-leak"
+	daemon := drainTestDaemon(t, instanceID)
+	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
+	subscribeEC2Commands(t, daemon, instanceID)
+
+	// Take the baseline after one full round trip: the first request is what
+	// makes nats.go stand up its long-lived response-inbox goroutine, which
+	// would otherwise read as the leak.
+	var ack ec2v1.DrainVolumeResponse
+	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
+	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+	ignoreExisting := goleak.IgnoreCurrent()
+
+	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
+	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+
+	goleak.VerifyNone(t, ignoreExisting)
+}
+
 // seedProviderVolume allocates volumeID in the daemon's EBS provider so
 // operations that reach the provider (expand, delete) find it there.
 func seedProviderVolume(t *testing.T, daemon *Daemon, volumeID string, sizeGiB int64) {

spinifex/runtime/daemon/daemon_test.go

@@ -50,7 +50,6 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
@@ -138,10 +137,10 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	// Wire the minimum vm.Deps that handler tests rely on. Lifecycle (Run/Start/
 	// Stop/Terminate) tests still set up their own deps; this gives the
 	// AttachVolume / DetachVolume manager methods enough plumbing to drive
-	// publish/unpublish over NATS using the test's connection.
+	// ebs.mount/unmount over NATS using the test's connection.
 	daemon.vmMgr.SetDeps(vm.Deps{
 		NodeID:               daemon.node,
-		VolumeMounter:        newNATSMounter(daemon.natsConn, daemon.node, daemon.volumeService),
+		VolumeMounter:        newVolumeMounterAdapter(daemon.natsConn, daemon.node, daemon.volumeService),
 		VolumeStateUpdater:   daemon.volumeService,
 		DetachDelay:          daemon.detachDelay,
 		DeviceDeletedTimeout: daemon.deviceDeletedTimeout,
@@ -2353,13 +2352,18 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	// Stand in for the provider's unpublish.
+	// Subscribe a mock ebs.unmount handler
 	ebsUnmountCalled := make(chan string, 1)
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
-		ebsUnmountCalled <- volumeID
-		return ebsprovider.UnpublishVolumeResponse{}
+	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		ebsUnmountCalled <- req.Name
+		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
 	})
+	require.NoError(t, err)
+	defer ebsSub.Unsubscribe()
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2400,12 +2404,12 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	assert.Equal(t, []string{"device_del", "blockdev-del", "object-del"}, qmpCommands)
 	mu.Unlock()
 
-	// Verify unpublish was called
+	// Verify ebs.unmount was called
 	select {
 	case unmountedVol := <-ebsUnmountCalled:
 		assert.Equal(t, volumeID, unmountedVol)
 	case <-time.After(5 * time.Second):
-		t.Fatal("unpublish was not called")
+		t.Fatal("ebs.unmount was not called")
 	}
 
 	// Verify volume removed from EBSRequests
@@ -2489,7 +2493,14 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	// Mock ebs.unmount
+	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: false}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer ebsSub.Unsubscribe()
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2686,7 +2697,13 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: false}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer ebsSub.Unsubscribe()
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2756,9 +2773,14 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	// The provider publishes the volume with a new NBD URI.
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://new:2222") })
+	// Mock ebs.mount to return success with a new NBDURI
+	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://new:2222"}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer ebsSub.Unsubscribe()
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -3313,33 +3335,95 @@ func TestNewDaemon_WalDirPreservedIfSet(t *testing.T) {
 	assert.Equal(t, "/fast-ssd/wal", d.config.WalDir)
 }
 
-// TestVolumeMounterAdapter_UnmountOne_UnmountError verifies that UnmountOne
-// propagates an unpublish refusal. DetachVolume gates the volume's available
-// transition on this error, so it must not be swallowed.
-func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
-	daemon := createTestDaemon(t, sharedNATSURL)
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
-		return unpublishRefusal(ebsprovider.ErrorCodeInternal, "seal volume: device busy")
-	})
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+// TestVolumeMounterAdapter_UnmountOne_Success verifies that the adapter's
+// UnmountOne sends an ebs.unmount NATS request and handles a successful
+// response. UnmountOne is shared by the AttachVolume rollback path and the
+// DetachVolume ebs.unmount step.
+func TestVolumeMounterAdapter_UnmountOne_Success(t *testing.T) {
+	natsURL := sharedNATSURL
 
-	err := adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-rollback-err"})
-	require.Error(t, err, "an unpublish refusal must propagate so DetachVolume can keep the volume attached")
-	assert.Contains(t, err.Error(), "device busy")
+	daemon := createTestDaemon(t, natsURL)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	unmountCalled := make(chan string, 1)
+
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		unmountCalled <- req.Name
+		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{
+		Name:       "vol-rollback-test",
+		DeviceName: "/dev/sdf",
+	})
+
+	select {
+	case volName := <-unmountCalled:
+		assert.Equal(t, "vol-rollback-test", volName)
+	case <-time.After(2 * time.Second):
+		t.Fatal("ebs.unmount was not called")
+	}
+}
+
+// TestVolumeMounterAdapter_UnmountOne_UnmountError verifies that UnmountOne
+// propagates an error response from ebs.unmount. DetachVolume gates the volume's
+// available transition on this error, so it must not be swallowed.
+func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
+	natsURL := sharedNATSURL
+
+	daemon := createTestDaemon(t, natsURL)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSUnMountResponse{Error: "unmount failed: device busy"}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-rollback-err"}),
+		"an ebs.unmount error response must propagate so DetachVolume can keep the volume attached")
+}
+
+// TestVolumeMounterAdapter_UnmountOne_StillMounted verifies that UnmountOne
+// returns an error when the unmount response reports the volume still mounted.
+func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
+	natsURL := sharedNATSURL
+
+	daemon := createTestDaemon(t, natsURL)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: true}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-still-mounted"}),
+		"a still-mounted response must propagate as an error")
 }
 
 // TestVolumeMounterAdapter_UnmountOne_RequestFailure verifies that UnmountOne
-// surfaces a failed unpublish request as an error. A closed connection fails
-// immediately rather than waiting out the provider's seal budget.
+// surfaces a failed ebs.unmount NATS request as an error. A closed connection
+// fails immediately, rather than blocking on the full unmountSealTimeout (120s)
+// that a no-subscriber timeout would incur in every preflight run.
 func TestVolumeMounterAdapter_UnmountOne_RequestFailure(t *testing.T) {
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	nc.Close()
 
-	adapter := newNATSMounter(nc, "node-1", nil)
-	require.ErrorIs(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-timeout"}), nats.ErrConnectionClosed,
-		"a failed unpublish request must propagate as an error")
+	adapter := newVolumeMounterAdapter(nc, "node-1", nil)
+	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-timeout"}),
+		"a failed ebs.unmount request must propagate as an error")
 }
 
 // recordingVolState records UpdateVolumeState calls so the bulk-unmount test can
@@ -3385,14 +3469,20 @@ var _ vm.VolumeStateUpdater = (*recordingVolState)(nil)
 func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
-		if volumeID == "vol-fail" {
-			return unpublishRefusal(ebsprovider.ErrorCodeInternal, "seal volume: predastore unreachable")
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
+
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		if req.Name == "vol-fail" {
+			resp.Error = "seal volume: predastore unreachable"
 		}
-		return ebsprovider.UnpublishVolumeResponse{}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
 	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
 
 	inst := &vm.VM{
 		ID: "i-1",
@@ -3400,7 +3490,7 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 			Requests: []viperblocklegacyv1.EBSRequest{{Name: "vol-fail"}, {Name: "vol-ok"}},
 		},
 	}
-	err := adapter.Unmount(t.Context(), inst)
+	err = adapter.Unmount(t.Context(), inst)
 	require.Error(t, err, "bulk Unmount must report a per-volume seal failure")
 	assert.Contains(t, err.Error(), "vol-fail", "the error must name the volume that did not seal")
 
@@ -3411,18 +3501,29 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 		"a successful seal must still transition the volume to available")
 }
 
-// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies that an
-// unpublish answered not_found, which means the provider holds no export of
-// the volume, still flips a non-boot data volume to available, while the
-// boot/EFI gate stays intact.
+// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies the
+// timeout-then-completed-seal retry path: an ebs.unmount response reporting
+// NotFound (the seal already completed on a prior, client-timed-out request)
+// must still flip a non-boot data volume to available, while the boot/EFI
+// gate stays intact.
 func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
-		return unpublishRefusal(ebsprovider.ErrorCodeNotFound, fmt.Sprintf("Volume %s not found", volumeID))
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
+
+	sub, err := daemon.natsConn.Subscribe(viperblocklegacyv1.UnmountSubject(daemon.node), func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		resp := viperblocklegacyv1.EBSUnMountResponse{
+			Volume:   req.Name,
+			NotFound: true,
+			Error:    fmt.Sprintf("Volume %s not found", req.Name),
+		}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
 	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
 
 	inst := &vm.VM{
 		ID:        "i-unmount-retry",
@@ -3435,86 +3536,130 @@ func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
 		},
 	}
 	require.NoError(t, adapter.Unmount(t.Context(), inst))
-	require.NoError(t, adapter.UnmountOne(t.Context(), "000000000042", viperblocklegacyv1.EBSRequest{Name: "vol-data-retry"}),
-		"a hot detach of a volume the provider no longer holds is already done")
 
 	calls := volState.snapshot()
 	assert.Contains(t, calls, "vol-data-retry:available",
-		"a not_found unpublish must still flip a non-boot data volume to available")
+		"a NotFound response (already-completed seal) must still flip a non-boot data volume to available")
 	assert.NotContains(t, calls, "vol-boot-retry:available",
-		"a boot volume must stay attached even when the unpublish reports not_found")
+		"a boot volume must stay attached even when the seal retry reports NotFound")
 	assert.Equal(t, []string{"000000000042"}, volState.accountsSnapshot(),
 		"the state write must be keyed on the instance's own account, not its ID")
 }
 
-// TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
-// the second of three volumes is refused, the adapter unpublishes the volume
-// it already published in the same Mount() call and nothing else. Without
-// rollback, a launch failure would leave the first volume's export live.
-func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
-		if volumeID == "vol-2" {
-			return ebsprovider.PublishVolumeResponse{Error: providerRefusal(ebsprovider.ErrorCodeInternal, "simulated mount failure")}
-		}
-		return publishedNBD("nbd://mounted-" + volumeID)
+// TestUnmountResponseError covers the shared seal-result parser used by both the
+// gated DetachVolume path (unmountOne) and the tolerated teardown path (Unmount).
+func TestUnmountResponseError(t *testing.T) {
+	t.Run("success (not mounted, no error)", func(t *testing.T) {
+		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Mounted: false})
+		require.NoError(t, unmountResponseError(data))
+	})
+	t.Run("seal error propagates", func(t *testing.T) {
+		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Error: "seal volume: boom"})
+		require.Error(t, unmountResponseError(data))
+	})
+	t.Run("still mounted propagates", func(t *testing.T) {
+		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Mounted: true})
+		require.Error(t, unmountResponseError(data))
+	})
+	t.Run("malformed payload propagates", func(t *testing.T) {
+		require.Error(t, unmountResponseError([]byte("not json")))
+	})
+	t.Run("not found treated as idempotent success", func(t *testing.T) {
+		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", NotFound: true, Error: "Volume v not found"})
+		require.NoError(t, unmountResponseError(data),
+			"a NotFound response means the seal already completed on a prior request; a timeout-then-retry must not be treated as a failure")
 	})
-	adapter := newNATSMounter(nc, node, nil)
-
-	err := adapter.Mount(t.Context(), mounterTestVM("vol-1", "vol-2", "vol-3"))
-	require.Error(t, err, "Mount should propagate the vol-2 failure")
-	assert.Contains(t, err.Error(), "simulated mount failure")
-	assert.Equal(t, []string{"vol-1"}, fake.unpublished(),
-		"only the previously published vol-1 is rolled back; a refused publish exported nothing")
 }
 
-// A publish reply that cannot be decoded may still have started an export, so
-// the failing volume is torn down along with the one already published.
-func TestVolumeMounterAdapter_Mount_UndecodableReplyRollsBackBoth(t *testing.T) {
-	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+// TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
+// any of Mount's per-volume failure paths fires mid-fan-out, the adapter
+// unmounts the volumes already successfully mounted in the same Mount()
+// call. Each subtest exercises a distinct rollback trigger so a regression
+// that drops rollback() from one branch (mount-response error, malformed
+// response, NATS layer failure) is caught individually. Without rollback,
+// a launch failure on the second of three volumes would leave the first
+// volume's viperblockd attached and the NBD socket live, leaking
+// resources every retry.
+func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
+	tests := []struct {
+		name        string
+		respondVol2 func(msg *nats.Msg)
+		wantErrSub  string
+	}{
+		{
+			name: "MountResponseError",
+			respondVol2: func(msg *nats.Msg) {
+				resp := viperblocklegacyv1.EBSMountResponse{Error: "simulated mount failure"}
+				data, _ := json.Marshal(resp)
+				msg.Respond(data)
+			},
+			wantErrSub: "simulated mount failure",
+		},
+		{
+			name: "MalformedResponse",
+			respondVol2: func(msg *nats.Msg) {
+				msg.Respond([]byte("not-valid-json"))
+			},
+			wantErrSub: "invalid character",
+		},
+	}
 
-	publishSubject, err := ebsprovider.PublishSubject(daemon.node)
-	require.NoError(t, err)
-	mountSub, err := daemon.natsConn.Subscribe(publishSubject, func(msg *nats.Msg) {
-		var req ebsprovider.PublishVolumeRequest
-		require.NoError(t, json.Unmarshal(msg.Data, &req))
-		if req.VolumeID == "vol-2" {
-			msg.Respond([]byte("not-valid-json"))
-			return
-		}
-		resp := publishedNBD("nbd://mounted-" + req.VolumeID)
-		resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer mountSub.Unsubscribe()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			daemon := createTestDaemon(t, sharedNATSURL)
+			adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
 
-	unmounted := make(chan string, 3)
-	unpublishSubject, err := ebsprovider.UnpublishSubject(daemon.node)
-	require.NoError(t, err)
-	unmountSub, err := daemon.natsConn.Subscribe(unpublishSubject, func(msg *nats.Msg) {
-		var req ebsprovider.UnpublishVolumeRequest
-		require.NoError(t, json.Unmarshal(msg.Data, &req))
-		unmounted <- req.VolumeID
-		data, _ := json.Marshal(ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)})
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer unmountSub.Unsubscribe()
+			mountSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+				var req viperblocklegacyv1.EBSRequest
+				require.NoError(t, json.Unmarshal(msg.Data, &req))
+				if req.Name == "vol-2" {
+					tt.respondVol2(msg)
+					return
+				}
+				resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-" + req.Name}
+				data, _ := json.Marshal(resp)
+				msg.Respond(data)
+			})
+			require.NoError(t, err)
+			defer mountSub.Unsubscribe()
 
-	err = adapter.Mount(t.Context(), mounterTestVM("vol-1", "vol-2", "vol-3"))
-	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
-	require.ErrorIs(t, err, ebsprovider.ErrUnreadableReply)
-	assert.Contains(t, err.Error(), "invalid character")
+			unmounted := make(chan string, 3)
+			unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+				var req viperblocklegacyv1.EBSRequest
+				require.NoError(t, json.Unmarshal(msg.Data, &req))
+				unmounted <- req.Name
+				resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+				data, _ := json.Marshal(resp)
+				msg.Respond(data)
+			})
+			require.NoError(t, err)
+			defer unmountSub.Unsubscribe()
 
-	// Mount returns only after rollback completes, and the subscriber sends on
-	// the buffered channel before responding, so every rollback is seen here.
-	require.Len(t, unmounted, 2, "vol-1 and the ambiguous vol-2 are rolled back")
-	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, []string{<-unmounted, <-unmounted})
+			instance := &vm.VM{
+				ID:        "i-mount-rollback",
+				AccountID: testAccountID,
+				EBSRequests: vm.EBSRequests{
+					Requests: []viperblocklegacyv1.EBSRequest{
+						{Name: "vol-1"},
+						{Name: "vol-2"},
+						{Name: "vol-3"},
+					},
+				},
+			}
+
+			err = adapter.Mount(t.Context(), instance)
+			require.Error(t, err, "Mount should propagate the vol-2 failure")
+			assert.Contains(t, err.Error(), tt.wantErrSub)
+
+			// Mount returns only after rollback completes (the unmount NATS
+			// round-trip is synchronous), and the unmount subscriber sends on
+			// the buffered channel before responding — so by the time Mount
+			// returns, every rollback unmount is observable in `unmounted`.
+			require.Len(t, unmounted, 1,
+				"exactly one volume (vol-1, the previously mounted one) should be rolled back")
+			assert.Equal(t, "vol-1", <-unmounted)
+		})
+	}
 }
 
 // TestVolumeMounterAdapter_Mount_RollbackFailurePropagates verifies that
@@ -3523,17 +3668,31 @@ func TestVolumeMounterAdapter_Mount_UndecodableReplyRollsBackBoth(t *testing.T)
 // attached without surfacing the leak to the caller.
 func TestVolumeMounterAdapter_Mount_RollbackFailurePropagates(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
-	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
-	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
-		if volumeID == "vol-2" {
-			return ebsprovider.PublishVolumeResponse{Error: providerRefusal(ebsprovider.ErrorCodeInternal, "primary mount failure")}
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	mountSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		require.NoError(t, json.Unmarshal(msg.Data, &req))
+		if req.Name == "vol-2" {
+			resp := viperblocklegacyv1.EBSMountResponse{Error: "primary mount failure"}
+			data, _ := json.Marshal(resp)
+			msg.Respond(data)
+			return
 		}
-		return publishedNBD("nbd://mounted-" + volumeID)
+		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-" + req.Name}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
 	})
-	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
-		return unpublishRefusal(ebsprovider.ErrorCodeInternal, "rollback unmount failed")
+	require.NoError(t, err)
+	defer mountSub.Unsubscribe()
+
+	unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSUnMountResponse{Error: "rollback unmount failed"}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
 	})
+	require.NoError(t, err)
+	defer unmountSub.Unsubscribe()
 
 	instance := &vm.VM{
 		ID:        "i-rollback-failure",
@@ -3546,7 +3705,7 @@ func TestVolumeMounterAdapter_Mount_RollbackFailurePropagates(t *testing.T) {
 		},
 	}
 
-	err := adapter.Mount(t.Context(), instance)
+	err = adapter.Mount(t.Context(), instance)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "primary mount failure",
 		"original mount error must be preserved")

spinifex/runtime/daemon/instance_records.go

@@ -3,10 +3,13 @@ package daemon
 import (
 	"context"
 	"errors"
+	"fmt"
 	"log/slog"
+	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
 )
 
 // InstanceRecordPrefix is the key prefix for the per-resource instance record
@@ -215,3 +218,99 @@ func loadRecord[T any](store *kvstore.Store[T], key string) (*T, error) {
 	}
 	return record, nil
 }
+
+// TerminateDecided reports whether instanceID is terminated or terminating, or
+// has no instance record at all, read from the KV store at call time. A read
+// error is returned rather than guessed, so a partition never reads as absent.
+func (m *JetStreamManager) TerminateDecided(instanceID string) (bool, error) {
+	terminated, err := m.LoadTerminatedInstance(instanceID)
+	if err != nil {
+		return false, err
+	}
+	if terminated != nil {
+		return true, nil
+	}
+	record, err := m.LoadInstanceRecord(instanceID)
+	if err != nil {
+		return false, err
+	}
+	if record == nil {
+		return true, nil
+	}
+	switch record.Status.Status {
+	case vm.StateShuttingDown, vm.StateTerminated:
+		return true, nil
+	}
+	return false, nil
+}
+
+// OpenTeardowns lists the terminated instances whose dependency entry dep is
+// recorded and not yet done. A record that has expired is not listed, and a
+// failed listing is returned whole rather than as a partial set.
+func (m *JetStreamManager) OpenTeardowns(dep string) ([]string, error) {
+	terminated, err := m.ListTerminatedInstances()
+	if err != nil {
+		return nil, err
+	}
+	var ids []string
+	for _, v := range terminated {
+		state, ok := v.Teardown[dep]
+		if ok && vm.TeardownState(state) != vm.TeardownDone && v.Status == vm.StateTerminated {
+			ids = append(ids, v.ID)
+		}
+	}
+	return ids, nil
+}
+
+// MarkTeardownDone sets the terminated record's dependency entry dep done. An
+// absent or expired record is not an error: there is no entry left to close.
+func (m *JetStreamManager) MarkTeardownDone(instanceID, dep string) error {
+	terminated, err := m.LoadTerminatedInstance(instanceID)
+	if err != nil {
+		return err
+	}
+	if terminated == nil || vm.TeardownState(terminated.Teardown[dep]) == vm.TeardownDone {
+		return nil
+	}
+	_, err = m.UpdateTerminatedInstance(instanceID, func(v *vm.VM) {
+		if v.Teardown == nil {
+			v.Teardown = make(map[string]string)
+		}
+		v.Teardown[dep] = string(vm.TeardownDone)
+	})
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// KnownInstanceIDs lists every instance ID the instance-state and terminated
+// buckets hold a record for, in any key space, by key alone so an undecodable
+// record still counts. It attaches to the buckets and fails if one is missing.
+func KnownInstanceIDs(ctx context.Context, js jetstream.JetStream) (map[string]bool, error) {
+	ids := make(map[string]bool)
+	buckets := map[string][]string{
+		InstanceStateBucket:      {InstanceRecordPrefix, StoppedInstancePrefix},
+		TerminatedInstanceBucket: {InstanceRecordPrefix, TerminatedInstancePrefix},
+	}
+	for name, prefixes := range buckets {
+		kv, err := js.KeyValue(ctx, name)
+		if err != nil {
+			return nil, fmt.Errorf("open %s: %w", name, err)
+		}
+		lister, err := kv.ListKeys(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("list %s: %w", name, err)
+		}
+		for key := range lister.Keys() {
+			for _, prefix := range prefixes {
+				if id, ok := strings.CutPrefix(key, prefix); ok {
+					ids[id] = true
+				}
+			}
+		}
+		// The lister closes itself once drained, so Stop only reports that.
+		_ = lister.Stop()
+	}
+	return ids, nil
+}

spinifex/runtime/daemon/instance_recovery.go

@@ -10,7 +10,7 @@ import (
 	"slices"
 	"time"
 
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/cache"
@@ -301,7 +301,7 @@ func (f recoveryFault) stateReasonCode() string {
 
 // classifyRecoveryFault reads what the launch refused with.
 func classifyRecoveryFault(err error) recoveryFault {
-	if errors.Is(err, ec2v1.ErrVolumeHeldElsewhere) {
+	if errors.Is(err, ec2instance.ErrVolumeHeldElsewhere) {
 		return recoveryFaultLeaseHeld
 	}
 	if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorInsufficientInstanceCapacity {

spinifex/runtime/daemon/instance_recovery_coverage_test.go

@@ -251,7 +251,7 @@ func TestRecoveryAttemptSucceeds(t *testing.T) {
 	// starting QEMU, which is the one part of a recovery this cannot do.
 	d.vmMgr.SetDeps(vm.Deps{
 		NodeID:             d.node,
-		VolumeMounter:      newNATSMounter(d.natsConn, d.node, d.volumeService),
+		VolumeMounter:      newVolumeMounterAdapter(d.natsConn, d.node, d.volumeService),
 		VolumeStateUpdater: d.volumeService,
 		Hooks: vm.ManagerHooks{BeforeInstanceRelaunch: func(v *vm.VM) error {
 			d.vmMgr.Inspect(v, func(v *vm.VM) { v.Status = vm.StateShuttingDown })

spinifex/runtime/daemon/instance_recovery_test.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -404,7 +403,7 @@ func TestWhatTheLaunchRefusalMeansForTheNextAttempt(t *testing.T) {
 		{
 			name: "the old owner still holds the volume lease",
 			err: fmt.Errorf("mount refused: %w", fmt.Errorf("%w: %w",
-				ec2v1.ErrVolumeHeldElsewhere,
+				ec2instance.ErrVolumeHeldElsewhere,
 				awserrors.Errorf(awserrors.ErrorIncorrectState, "volume is leased by another owner: node-1"))),
 			want: recoveryFaultLeaseHeld,
 			code: "Server.HostRecoveryFailed",

spinifex/runtime/daemon/jetstream.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
@@ -88,6 +89,7 @@ type JetStreamManager struct {
 	termRecords   *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
 	clusterKV     jetstream.KeyValue                // spinifex-cluster-state
 	clusterConfig *kvstore.Bucket                   // spinifex-cluster-config
+	uefiStores    *uefistore.Store                  // spinifex-uefi-stores
 	obs           KVSyncObserver
 	running       runningSetState
 }
@@ -232,6 +234,18 @@ func (m *JetStreamManager) InitClusterConfigBucket() error {
 	return err
 }
 
+// InitUEFIStoreBucket opens the UEFI store-record bucket, creating it if it
+// doesn't exist, so a bucket that cannot be created fails startup.
+func (m *JetStreamManager) InitUEFIStoreBucket() error {
+	m.uefiStores = uefistore.NewStore(m.js)
+	return m.uefiStores.Open(context.Background())
+}
+
+// UEFIStores returns the store-record bucket opened by InitUEFIStoreBucket.
+func (m *JetStreamManager) UEFIStores() *uefistore.Store {
+	return m.uefiStores
+}
+
 // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
 // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
 func (m *JetStreamManager) InitTerminatedInstanceBucket() error {

spinifex/runtime/daemon/nats_classification_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/stretchr/testify/assert"
@@ -128,6 +129,7 @@ func TestEC2CommandNameCoversEveryCommand(t *testing.T) {
 	}{
 		{"AttachVolume", func(a *ec2v1.EC2CommandAttributes) { a.AttachVolume = true }},
 		{"DetachVolume", func(a *ec2v1.EC2CommandAttributes) { a.DetachVolume = true }},
+		{"DrainVolume", func(a *ec2v1.EC2CommandAttributes) { a.DrainVolume = true }},
 		{"AttachNetworkInterface", func(a *ec2v1.EC2CommandAttributes) { a.AttachENI = true }},
 		{"DetachNetworkInterface", func(a *ec2v1.EC2CommandAttributes) { a.DetachENI = true }},
 		{"AssociateIamInstanceProfile", func(a *ec2v1.EC2CommandAttributes) { a.AssociateIamInstanceProfile = true }},
@@ -185,3 +187,28 @@ func TestEC2CommandRecordsUnderTheCommandNotTheInstance(t *testing.T) {
 	assert.Empty(t, requestOutcomesFor(t, subject),
 		"the instance id must not reach the metric dimension")
 }
+
+// A drain runs off-thread so the command subscription is not held for the
+// length of the flush. Its point therefore has to be recorded by the goroutine
+// once the work finishes: recording at dispatch would call every drain an
+// instant success.
+func TestDrainVolumeRecordsItsPointAfterTheWork(t *testing.T) {
+	installOutcomeReader(t)
+	const instanceID, volumeID = "i-drain-metric", "vol-drain-metric"
+	daemon := drainTestDaemon(t, instanceID)
+	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
+	subscribeEC2Commands(t, daemon, instanceID)
+
+	before := requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess)
+	reply := drainRequest(t, daemon, instanceID, volumeID)
+
+	var ack ec2v1.DrainVolumeResponse
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
+
+	// The goroutine records after it replies, so poll rather than assume the
+	// point has landed by the time the reply arrives.
+	require.Eventually(t, func() bool {
+		return requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess) > before
+	}, 5*time.Second, 20*time.Millisecond, "the drain goroutine must record its own success point")
+}

spinifex/runtime/daemon/shutdown_test.go

@@ -13,7 +13,6 @@ import (
 
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go"
@@ -677,16 +676,36 @@ func TestHandleShutdownDrain(t *testing.T) {
 	})
 }
 
-// stubEBSUnmount answers the daemon's unpublish subject, standing in for its
-// provider, with refusal when one is given. Returns the volumes it was asked
-// to unmount.
-func stubEBSUnmount(t *testing.T, d *Daemon, refusal *ebsprovider.ProviderError) func() []string {
+// stubEBSUnmount answers the daemon's ebs.<node>.unmount subject with a fixed
+// response, standing in for viperblockd. Returns the volumes it was asked to
+// unmount once the subscription has been drained.
+func stubEBSUnmount(t *testing.T, d *Daemon, resp viperblocklegacyv1.EBSUnMountResponse) func() []string {
 	t.Helper()
-	fake := serveFakeNodeProvider(t, d.natsConn, d.node)
-	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
-		return ebsprovider.UnpublishVolumeResponse{Error: refusal}
+	var (
+		mu      sync.Mutex
+		volumes []string
+	)
+	sub, err := d.natsConn.Subscribe("ebs."+d.node+".unmount", func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		require.NoError(t, json.Unmarshal(msg.Data, &req))
+		mu.Lock()
+		volumes = append(volumes, req.Name)
+		mu.Unlock()
+		reply := resp
+		reply.Volume = req.Name
+		data, marshalErr := json.Marshal(reply)
+		require.NoError(t, marshalErr)
+		require.NoError(t, msg.Respond(data))
 	})
-	return fake.unpublished
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, d.natsConn.Flush())
+
+	return func() []string {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]string(nil), volumes...)
+	}
 }
 
 // drainRunningInstance is a guest in the state DRAIN finds it in: running, with
@@ -712,8 +731,10 @@ func TestHandleShutdownDrain_SealFailure(t *testing.T) {
 		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
 		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
 
-		unmounted := stubEBSUnmount(t, daemon, providerRefusal(ebsprovider.ErrorCodeInternal,
-			"seal volume to predastore: transfer stalled: idle timeout"))
+		unmounted := stubEBSUnmount(t, daemon, viperblocklegacyv1.EBSUnMountResponse{
+			Mounted: true,
+			Error:   "seal volume to predastore: transfer stalled: idle timeout",
+		})
 		daemon.vmMgr.Insert(drainRunningInstance("i-drain-seal-fail", "vol-drain-seal-fail"))
 
 		subject := "spinifex.cluster.shutdown.drain.sealfail"
@@ -749,7 +770,7 @@ func TestHandleShutdownDrain_SealFailure(t *testing.T) {
 		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
 		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
 
-		unmounted := stubEBSUnmount(t, daemon, nil)
+		unmounted := stubEBSUnmount(t, daemon, viperblocklegacyv1.EBSUnMountResponse{})
 		instance := drainRunningInstance("i-drain-seal-ok", "vol-drain-seal-ok")
 		daemon.vmMgr.Insert(instance)
 

spinifex/runtime/daemon/uefi_store_bucket_test.go

@@ -0,0 +1,36 @@
+package daemon_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The daemon creates the store-record bucket at startup and hands out the
+// store it opened, so later readers share one handle.
+func TestInitUEFIStoreBucket_CreatesBucketAndServesTheStore(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	m, err := daemon.NewJetStreamManager(nc)
+	require.NoError(t, err)
+
+	require.NoError(t, m.InitUEFIStoreBucket())
+
+	_, err = js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err, "init must create the bucket rather than leave it to first use")
+
+	rec := &uefistore.UEFIStoreRecord{InstanceID: "i-0123456789abcdef0", State: uefistore.StateLive}
+	rev, err := m.UEFIStores().Create(t.Context(), rec)
+	require.NoError(t, err)
+	got, gotRev, err := m.UEFIStores().Get(t.Context(), rec.InstanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, rec, got)
+
+	_, err = m.UEFIStores().Create(t.Context(), rec)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+}

spinifex/runtime/daemon/uefi_store_reaper.go

@@ -0,0 +1,212 @@
+package daemon
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// uefiStoreKVTimeout bounds the one record read and write a terminate makes.
+const uefiStoreKVTimeout = 10 * time.Second
+
+// uefiLocalDirName is the directory under the daemon's data directory that
+// holds each instance's local copy of its variable store.
+const uefiLocalDirName = "uefi"
+
+// uefiLocalDir is the directory holding instanceID's local variable store.
+func uefiLocalDir(dataDir, instanceID string) string {
+	if dataDir == "" {
+		dataDir = DefaultDataDir
+	}
+	return filepath.Join(dataDir, uefiLocalDirName, instanceID)
+}
+
+// uefiStoreTerminator satisfies vm.UEFIStoreTerminator over the store-record
+// bucket and this node's data directory.
+type uefiStoreTerminator struct {
+	stores  *uefistore.Store
+	dataDir string
+}
+
+var _ vm.UEFIStoreTerminator = (*uefiStoreTerminator)(nil)
+
+// newUEFIStoreTerminator returns nil, not a typed nil, when the bucket is not
+// open, so the manager sees no terminator rather than one that panics.
+func (d *Daemon) newUEFIStoreTerminator() vm.UEFIStoreTerminator {
+	if d.jsManager == nil || d.jsManager.UEFIStores() == nil {
+		return nil
+	}
+	return &uefiStoreTerminator{stores: d.jsManager.UEFIStores(), dataDir: d.config.DataDir}
+}
+
+func (t *uefiStoreTerminator) MarkUEFIStoreDeleting(instanceID string) (bool, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), uefiStoreKVTimeout)
+	defer cancel()
+	return t.stores.MarkDeleting(ctx, instanceID)
+}
+
+// RemoveLocalUEFIStore refuses anything but a bare instance ID, so a malformed
+// ID cannot point the removal outside the instance's own directory.
+func (t *uefiStoreTerminator) RemoveLocalUEFIStore(instanceID string) error {
+	if !strings.HasPrefix(instanceID, "i-") || strings.ContainsAny(instanceID, `/\`) || strings.Contains(instanceID, "..") {
+		return fmt.Errorf("refusing to remove a local UEFI store for %q", instanceID)
+	}
+	return os.RemoveAll(uefiLocalDir(t.dataDir, instanceID))
+}
+
+// uefiInstanceRecords is what the store reaper asks of the instance domain:
+// whether a terminate is decided, and the terminated records' uefi entries.
+type uefiInstanceRecords interface {
+	TerminateDecided(instanceID string) (bool, error)
+	OpenTeardowns(dep string) ([]string, error)
+	MarkTeardownDone(instanceID, dep string) error
+}
+
+var _ uefiInstanceRecords = (*JetStreamManager)(nil)
+
+// uefiStoreReaper deletes the variable stores whose record is deleting: each
+// object under the instance's prefix, then the record. Terminate, or an
+// operator's forget, sets deleting and returns, so this decides nothing.
+//
+// It also completes a terminate whose mark failed, from the terminated
+// record's pending or failed uefi entry. That record is the terminate itself,
+// so moving the store record to deleting there is still not the reaper's call.
+//
+// Cluster-wide: the store outlives the node that ran the instance, and one
+// sweeper keeps two deletes off one prefix.
+type uefiStoreReaper struct {
+	stores    *uefistore.Store
+	objects   objectstore.ObjectStore
+	bucket    string
+	instances uefiInstanceRecords
+}
+
+var _ vm.Reaper = (*uefiStoreReaper)(nil)
+
+// newUEFIStoreReaper builds the reaper, or nil when this node has no store
+// bucket or object store to reap with.
+func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
+	if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
+		return nil
+	}
+	return &uefiStoreReaper{
+		stores: d.jsManager.UEFIStores(),
+		objects: objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(d.config.Predastore.Host),
+			d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
+		bucket:    d.config.Predastore.Bucket,
+		instances: d.jsManager,
+	}
+}
+
+func (r *uefiStoreReaper) Class() string         { return "uefi-store" }
+func (r *uefiStoreReaper) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
+
+// Sweep deletes every deleting store it can and reports how many records it
+// retired. A store whose delete fails keeps its record, which has no TTL, and
+// is retried next sweep however long predastore is away.
+func (r *uefiStoreReaper) Sweep(ctx context.Context) (int, error) {
+	r.completeTerminatedMarks(ctx)
+
+	ids, err := r.stores.InstanceIDs(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list UEFI store records: %w", err)
+	}
+	reaped := 0
+	for _, id := range ids {
+		if err := ctx.Err(); err != nil {
+			return reaped, err
+		}
+		if r.reap(ctx, id) {
+			reaped++
+		}
+	}
+	return reaped, nil
+}
+
+// reap deletes one store if its record is deleting, and reports whether the
+// record was retired.
+func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
+	rec, rev, err := r.stores.Get(ctx, id)
+	if err != nil {
+		if !errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "uefi-store: unreadable store record, leaving it", "instanceId", id, "err", err)
+		}
+		return false
+	}
+	if rec.State != uefistore.StateDeleting {
+		return false
+	}
+
+	ok, err := r.instances.TerminateDecided(id)
+	if err != nil {
+		slog.WarnContext(ctx, "uefi-store: cannot read the instance record, retrying next sweep", "instanceId", id, "err", err)
+		return false
+	}
+	if !ok {
+		// Deleting is only ever set by a decided terminate or a forget, so a
+		// live instance here is a defect; its store is kept for an operator.
+		slog.ErrorContext(ctx, "uefi-store: refusing to delete the store of an instance that is not terminating",
+			"instanceId", id, "accountId", rec.AccountID)
+		return false
+	}
+
+	start := time.Now()
+	deleted, err := uefistore.DeleteObjects(ctx, r.objects, r.bucket, id)
+	if err != nil {
+		slog.WarnContext(ctx, "uefi-store: delete incomplete, retrying next sweep",
+			"instanceId", id, "deleted", deleted, "err", err)
+		return false
+	}
+	if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
+		slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record, retrying next sweep", "instanceId", id, "err", err)
+		return false
+	}
+	// Removed last: while it exists the obligation is readable, so a sweep
+	// interrupted above resumes rather than leaving objects nobody owns.
+	if err := r.stores.CompareAndDelete(ctx, id, rev); err != nil {
+		slog.WarnContext(ctx, "uefi-store: failed to remove the store record, retrying next sweep", "instanceId", id, "err", err)
+		return false
+	}
+	slog.InfoContext(ctx, "uefi-store: store deleted", "instanceId", id, "accountId", rec.AccountID,
+		"objects", deleted, "elapsed_ms", otelsetup.Millis(time.Since(start)))
+	return true
+}
+
+// completeTerminatedMarks finishes the mark of each terminate that recorded a
+// uefi entry but did not land its store record in deleting.
+func (r *uefiStoreReaper) completeTerminatedMarks(ctx context.Context) {
+	ids, err := r.instances.OpenTeardowns(vm.TeardownUEFI)
+	if err != nil {
+		slog.WarnContext(ctx, "uefi-store: cannot list terminated instances", "err", err)
+		return
+	}
+	for _, id := range ids {
+		rec, _, err := r.stores.Get(ctx, id)
+		switch {
+		case errors.Is(err, kvstore.ErrNotFound):
+			// Nothing left to delete: the store is gone or never existed.
+			if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
+				slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record", "instanceId", id, "err", err)
+			}
+		case err != nil:
+			slog.WarnContext(ctx, "uefi-store: unreadable store record", "instanceId", id, "err", err)
+		case rec.State == uefistore.StateLive:
+			if _, err := r.stores.MarkDeleting(ctx, id); err != nil {
+				slog.WarnContext(ctx, "uefi-store: failed to record a terminated instance's store for deletion",
+					"instanceId", id, "err", err)
+			}
+		}
+	}
+}

spinifex/runtime/daemon/uefi_store_reaper_test.go

@@ -0,0 +1,255 @@
+//test:in-package — the reaper and the terminator adapter are unexported daemon types
+package daemon
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const uefiTestBucket = "spinifex"
+
+type uefiReaperFixture struct {
+	m       *JetStreamManager
+	objects *objectstore.MemoryObjectStore
+	r       *uefiStoreReaper
+}
+
+func newUEFIReaperFixture(t *testing.T) *uefiReaperFixture {
+	t.Helper()
+	m := newRecordManagerInPackage(t)
+	require.NoError(t, m.InitUEFIStoreBucket())
+	objects := objectstore.NewMemoryObjectStore()
+	return &uefiReaperFixture{
+		m:       m,
+		objects: objects,
+		r:       &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m},
+	}
+}
+
+// seedStore writes a live store record and n objects under its prefix, the
+// state a launch leaves, then marks it deleting if asked, as terminate does.
+func (f *uefiReaperFixture) seedStore(t *testing.T, id string, n int, deleting bool) {
+	t.Helper()
+	_, err := f.m.UEFIStores().Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: id, AccountID: "111122223333", State: uefistore.StateLive})
+	require.NoError(t, err)
+	for i := range n {
+		_, err := f.objects.PutObject(t.Context(), &s3.PutObjectInput{
+			Bucket: aws.String(uefiTestBucket),
+			Key:    aws.String(uefistore.KeyPrefix(id) + string(rune('a'+i))),
+			Body:   bytes.NewReader([]byte("vars")),
+		})
+		require.NoError(t, err)
+	}
+	if deleting {
+		_, err := f.m.UEFIStores().MarkDeleting(t.Context(), id)
+		require.NoError(t, err)
+	}
+}
+
+func (f *uefiReaperFixture) writeInstance(t *testing.T, id string, status vm.InstanceState) {
+	t.Helper()
+	v := &vm.VM{ID: id, Status: status, AccountID: "111122223333", Instance: &ec2.Instance{InstanceId: aws.String(id)}}
+	require.NoError(t, f.m.WriteInstanceRecord(id, v.Record()))
+}
+
+func (f *uefiReaperFixture) writeTerminated(t *testing.T, id string, uefi vm.TeardownState) {
+	t.Helper()
+	v := &vm.VM{ID: id, Status: vm.StateTerminated, AccountID: "111122223333", Instance: &ec2.Instance{InstanceId: aws.String(id)},
+		Teardown: map[string]string{vm.TeardownUEFI: string(uefi)}}
+	require.NoError(t, f.m.WriteTerminatedInstance(id, v))
+}
+
+func (f *uefiReaperFixture) objectCount(t *testing.T, id string) int {
+	t.Helper()
+	keys, err := uefistore.ListObjects(t.Context(), f.objects, uefiTestBucket, id)
+	require.NoError(t, err)
+	return len(keys)
+}
+
+func (f *uefiReaperFixture) recordState(t *testing.T, id string) string {
+	t.Helper()
+	rec, _, err := f.m.UEFIStores().Get(t.Context(), id)
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return "absent"
+	}
+	require.NoError(t, err)
+	return rec.State
+}
+
+func (f *uefiReaperFixture) terminatedUEFIMark(t *testing.T, id string) string {
+	t.Helper()
+	v, err := f.m.LoadTerminatedInstance(id)
+	require.NoError(t, err)
+	require.NotNil(t, v)
+	return v.Teardown[vm.TeardownUEFI]
+}
+
+// TestS7_UEFIStoreReaperDeletesOnlyAStoreATerminateMarkedDeleting enforces
+// ADR-0003:S7 for the store reaper: it deletes the objects and record of a
+// store a terminate set deleting, and never a live store, whether its
+// instance runs, is stopped, or has no record the reaper could judge it by.
+func TestS7_UEFIStoreReaperDeletesOnlyAStoreATerminateMarkedDeleting(t *testing.T) {
+	f := newUEFIReaperFixture(t)
+	const (
+		running     = "i-0000000000000running"
+		stopped     = "i-0000000000000stopped"
+		unowned     = "i-0000000000000unowned"
+		terminated  = "i-000000000terminated"
+		terminating = "i-00000000terminating"
+	)
+	f.writeInstance(t, running, vm.StateRunning)
+	f.seedStore(t, running, 3, false)
+	f.writeInstance(t, stopped, vm.StateStopped)
+	f.seedStore(t, stopped, 3, false)
+	f.seedStore(t, unowned, 3, false)
+	f.writeTerminated(t, terminated, vm.TeardownPending)
+	f.seedStore(t, terminated, 3, true)
+	f.writeInstance(t, terminating, vm.StateShuttingDown)
+	f.seedStore(t, terminating, 2, true)
+
+	reaped, err := f.r.Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, 2, reaped)
+
+	for _, id := range []string{running, stopped, unowned} {
+		assert.Equalf(t, 3, f.objectCount(t, id), "ADR-0003:S7: the reaper deleted objects of the live store %s", id)
+		assert.Equalf(t, uefistore.StateLive, f.recordState(t, id), "ADR-0003:S7: the reaper changed the live store record %s", id)
+	}
+	for _, id := range []string{terminated, terminating} {
+		assert.Zerof(t, f.objectCount(t, id), "a store a terminate marked deleting must be deleted: %s", id)
+		assert.Equalf(t, "absent", f.recordState(t, id), "the record goes once its prefix is gone: %s", id)
+	}
+	assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, terminated))
+}
+
+// TestS7_UEFIStoreReaperRefusesADeletingStoreOfALiveInstance enforces
+// ADR-0003:S7 when the record and the instance disagree: a deleting record
+// whose instance still runs or is stopped is kept, objects and all.
+func TestS7_UEFIStoreReaperRefusesADeletingStoreOfALiveInstance(t *testing.T) {
+	for _, status := range []vm.InstanceState{vm.StateRunning, vm.StateStopped, vm.StatePending} {
+		t.Run(string(status), func(t *testing.T) {
+			f := newUEFIReaperFixture(t)
+			const id = "i-0000000000000000a"
+			f.writeInstance(t, id, status)
+			f.seedStore(t, id, 3, true)
+
+			reaped, err := f.r.Sweep(t.Context())
+			require.NoError(t, err)
+			assert.Zero(t, reaped)
+			assert.Equal(t, 3, f.objectCount(t, id), "ADR-0003:S7: a live instance's store must survive a deleting record")
+			assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
+		})
+	}
+}
+
+// failingObjects stands in for predastore while it is down: every list and
+// delete fails.
+type failingObjects struct {
+	objectstore.ObjectStore
+}
+
+func (failingObjects) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return nil, errors.New("predastore: connection refused")
+}
+
+func (failingObjects) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	return nil, errors.New("predastore: connection refused")
+}
+
+// The obligation lives in the store record, which has no TTL, so a terminate
+// still completes when predastore is down for longer than the terminated
+// record lives.
+func TestUEFIStoreReaper_CompletesAfterTheTerminatedRecordExpires(t *testing.T) {
+	f := newUEFIReaperFixture(t)
+	const id = "i-0000000000000000a"
+	f.writeTerminated(t, id, vm.TeardownPending)
+	f.seedStore(t, id, 3, true)
+
+	f.r.objects = failingObjects{f.objects}
+	reaped, err := f.r.Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id), "a failed delete keeps the obligation")
+	assert.Equal(t, string(vm.TeardownPending), f.terminatedUEFIMark(t, id))
+
+	// The bucket's TTL expires the terminated record while predastore is away.
+	require.NoError(t, f.m.DeleteTerminatedInstance(id))
+	f.r.objects = f.objects
+
+	reaped, err = f.r.Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	assert.Zero(t, f.objectCount(t, id))
+	assert.Equal(t, "absent", f.recordState(t, id))
+}
+
+// A terminate whose mark failed left the store live and the terminated
+// record's entry failed; the reaper completes that terminate.
+func TestUEFIStoreReaper_CompletesATerminateWhoseMarkFailed(t *testing.T) {
+	f := newUEFIReaperFixture(t)
+	const id = "i-0000000000000000a"
+	f.seedStore(t, id, 3, false)
+	f.writeTerminated(t, id, vm.TeardownFailed)
+
+	reaped, err := f.r.Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	assert.Zero(t, f.objectCount(t, id))
+	assert.Equal(t, "absent", f.recordState(t, id))
+	assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, id))
+}
+
+// A pending entry whose store record is already gone has nothing left to
+// delete, so the entry is closed and the terminated record can be purged.
+func TestUEFIStoreReaper_ClosesAPendingEntryWithNoStore(t *testing.T) {
+	f := newUEFIReaperFixture(t)
+	const id = "i-0000000000000000a"
+	f.writeTerminated(t, id, vm.TeardownPending)
+
+	_, err := f.r.Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, id))
+}
+
+func TestUEFIStoreTerminator_MarksTheRecordAndRemovesOnlyTheInstanceDirectory(t *testing.T) {
+	f := newUEFIReaperFixture(t)
+	const id = "i-0000000000000000a"
+	f.seedStore(t, id, 1, false)
+	dataDir := t.TempDir()
+	mine := uefiLocalDir(dataDir, id)
+	other := uefiLocalDir(dataDir, "i-0000000000000000b")
+	for _, dir := range []string{mine, other} {
+		require.NoError(t, os.MkdirAll(dir, 0o750))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "vars.fd"), []byte("vars"), 0o600))
+	}
+	term := &uefiStoreTerminator{stores: f.m.UEFIStores(), dataDir: dataDir}
+
+	has, err := term.MarkUEFIStoreDeleting(id)
+	require.NoError(t, err)
+	assert.True(t, has)
+	assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
+
+	require.NoError(t, term.RemoveLocalUEFIStore(id))
+	assert.NoDirExists(t, mine)
+	assert.DirExists(t, other)
+	require.NoError(t, term.RemoveLocalUEFIStore(id), "removal is idempotent")
+
+	for _, bad := range []string{"", "..", "i-0/../..", "i-0000000000000000b/vars.fd", "vol-0123"} {
+		require.Errorf(t, term.RemoveLocalUEFIStore(bad), "%q", bad)
+	}
+	assert.FileExists(t, filepath.Join(other, "vars.fd"))
+}

spinifex/runtime/daemon/vm_adapters.go

@@ -19,7 +19,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -93,129 +92,26 @@ func (a *stateStoreAdapter) DeleteTerminatedInstance(id string) error {
 	return a.js.DeleteTerminatedInstance(id)
 }
 
-// volumeMounterAdapter satisfies vm.VolumeMounter over the EBS contract's
-// PublishVolume and UnpublishVolume, sent at the cluster's one contract
-// version to this node's provider. Unmount also marks volumes available.
+// volumeMounterAdapter satisfies vm.VolumeMounter by routing ebs.mount /
+// ebs.unmount NATS requests. Unmount also updates volume state via VolumeStateUpdater.
 type volumeMounterAdapter struct {
+	nc       *nats.Conn
 	node     string
 	volState vm.VolumeStateUpdater
-	provider mounterProvider
-	// abandonTimeout bounds each no-seal unpublish Abandon sends.
-	abandonTimeout time.Duration
 }
 
-// mounterProvider is the part of the EBS contract the mounter drives.
-type mounterProvider interface {
-	PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
-	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error)
-}
-
-var _ mounterProvider = (*ebsprovider.NATSProvider)(nil)
-
 var _ vm.VolumeMounter = (*volumeMounterAdapter)(nil)
 
-func newVolumeMounterAdapter(provider mounterProvider, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
-	return &volumeMounterAdapter{
-		node:           node,
-		volState:       volState,
-		provider:       provider,
-		abandonTimeout: abandonTimeout,
-	}
-}
+// unmountSealTimeout bounds the ebs.unmount NATS request. The handler now drives
+// a synchronous block-map seal to predastore (S3 I/O), which runs AFTER
+// viperblockd's SIGKILL once its 120s KillProcess grace elapses — so this
+// must strictly exceed that grace plus a seal allowance, never merely match
+// it: an unmount that consumes the whole grace could otherwise never be
+// observed as successful.
+const unmountSealTimeout = 180 * time.Second
 
-// publish mounts req's volume on this node through PublishVolume and records
-// the export on req. A failure worth retrying is vm.ErrMountRetryable; one
-// that leaves unknown whether the provider exported it is vm.ErrMountAmbiguous.
-func (a *volumeMounterAdapter) publish(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
-	// A caller already gone sends nothing, so there is no export to tear down;
-	// only a cancellation after the request went out leaves the outcome unknown.
-	if err := ctx.Err(); err != nil {
-		return fmt.Errorf("publish volume %s: %w", req.Name, err)
-	}
-	published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{
-		Versioned: ebsprovider.NewLatestVersioned(),
-		VolumeID:  req.Name,
-		NodeID:    a.node,
-	})
-	if err != nil {
-		switch {
-		case publishRetryable(err):
-			return fmt.Errorf("publish volume %s: %w: %w", req.Name, err, vm.ErrMountRetryable)
-		case publishOutcomeUnknown(err):
-			return fmt.Errorf("publish volume %s: %w: %w", req.Name, err, vm.ErrMountAmbiguous)
-		}
-		return fmt.Errorf("publish volume %s: %w", req.Name, err)
-	}
-	applyExport(req, published.Export)
-	slog.InfoContext(ctx, "Published volume", "volume", req.Name, "node", a.node, "export_kind", string(published.Export.Kind))
-	return nil
-}
-
-// publishRetryable reports whether a failed publish is worth repeating later.
-// A connection that is down never sent the request, and not_found is how a
-// volume whose state is not loadable yet is reported; restore backs off on both.
-func publishRetryable(err error) bool {
-	if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrConnectionClosed) {
-		return true
-	}
-	var providerErr *ebsprovider.ProviderError
-	if !errors.As(err, &providerErr) {
-		return false
-	}
-	return providerErr.Code.Retryable() || providerErr.Code == ebsprovider.ErrorCodeNotFound
-}
-
-// publishOutcomeUnknown reports whether the provider may have exported the
-// volume despite the error: the request was sent but no usable reply came back,
-// or the caller stopped waiting for it. A connection found down sends nothing.
-func publishOutcomeUnknown(err error) bool {
-	for _, unknown := range []error{
-		context.DeadlineExceeded, context.Canceled, nats.ErrTimeout, ebsprovider.ErrUnreadableReply,
-		ebsprovider.ErrNoPublishedVolume, ebsprovider.ErrInvalidExport,
-	} {
-		if errors.Is(err, unknown) {
-			return true
-		}
-	}
-	return false
-}
-
-// applyExport records export on req as the one export field QEMU attaches.
-// The provider client has already validated it.
-func applyExport(req *viperblocklegacyv1.EBSRequest, export ebsprovider.VolumeExport) {
-	switch export.Kind {
-	case ebsprovider.ExportKindBlock:
-		req.NBDURI = ""
-		req.BlockDevice = export.Block.DevicePath
-		req.BlockSizeBytes = export.Block.SizeBytes
-	default:
-		req.NBDURI = export.NBD.NBDURI
-		req.BlockDevice = ""
-		req.BlockSizeBytes = 0
-	}
-}
-
-// unpublish tears down this node's export of req's volume through
-// UnpublishVolume, which seals first, and returns nil only once its block map
-// is sealed or the provider holds no export of it.
-func (a *volumeMounterAdapter) unpublish(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
-	// The seal takes the provider's data-movement budget, not what is left of
-	// the caller's; the caller's trace is kept.
-	sealCtx := ebsprovider.WithAccountID(context.WithoutCancel(ctx), accountID)
-	_, err := a.provider.UnpublishVolume(sealCtx, ebsprovider.UnpublishVolumeRequest{
-		Versioned: ebsprovider.NewLatestVersioned(),
-		VolumeID:  req.Name,
-		NodeID:    a.node,
-	})
-	switch {
-	case err == nil:
-		return nil
-	case errors.Is(err, ebsprovider.ErrNotFound):
-		slog.InfoContext(ctx, "Unpublish found no volume, counting it as done", "volume", req.Name, "node", a.node)
-		return nil
-	default:
-		return fmt.Errorf("unpublish volume %s: %w", req.Name, err)
-	}
+func newVolumeMounterAdapter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
+	return &volumeMounterAdapter{nc: nc, node: node, volState: volState}
 }
 
 // ebsRequestWithTrace sends an ebs.* NATS request, opening a client span under
@@ -227,10 +123,13 @@ func (a *volumeMounterAdapter) unpublish(ctx context.Context, accountID string,
 // sweep or a recovery belongs to nobody, and crediting it to whoever happens
 // to be admin would be worse than leaving it blank.
 func ebsRequestWithTrace(ctx context.Context, nc *nats.Conn, accountID, subject string, data []byte, timeout time.Duration) (msg *nats.Msg, err error) {
-	attrs := append([]attribute.KeyValue{
+	attrs := []attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", subject),
-	}, natsmsg.AccountAttrs(accountID)...)
+	}
+	if accountID != "" {
+		attrs = append(attrs, attribute.String(natsmsg.AttrAccountID, accountID))
+	}
 
 	ctx, span := otel.Tracer(daemonTracerName).Start(ctx, "NATS "+subject,
 		trace.WithSpanKind(trace.SpanKindClient),
@@ -263,7 +162,7 @@ func (a *volumeMounterAdapter) Mount(ctx context.Context, instance *vm.VM) error
 		var rbErrs []error
 		for _, idx := range mounted {
 			req := instance.EBSRequests.Requests[idx]
-			if err := a.unpublish(ctx, instance.AccountID, req); err != nil {
+			if err := a.unmountOne(ctx, instance.AccountID, req); err != nil {
 				slog.Error("Mount rollback: unmount failed",
 					"volume", req.Name, "err", err)
 				rbErrs = append(rbErrs, fmt.Errorf("unmount %s: %w", req.Name, err))
@@ -276,14 +175,42 @@ func (a *volumeMounterAdapter) Mount(ctx context.Context, instance *vm.VM) error
 	}
 
 	for k, v := range instance.EBSRequests.Requests {
-		if err := a.publish(ctx, instance.AccountID, &instance.EBSRequests.Requests[k]); err != nil {
-			slog.Error("Failed to publish volume", "volume", v.Name, "err", err)
-			// The provider may have started this export, so it is torn down too.
-			if errors.Is(err, vm.ErrMountAmbiguous) {
-				mounted = append(mounted, k)
+		ebsMountRequest, err := json.Marshal(v)
+		if err != nil {
+			slog.Error("Failed to marshal volume payload", "err", err)
+			return rollback(err)
+		}
+
+		reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
+
+		slog.Info("Mounting volume", "Vol", v.Name, "NBDURI", v.NBDURI)
+
+		if err != nil {
+			slog.Error("Failed to request EBS mount", "err", err)
+			// A missing responder means viperblockd has not finished starting;
+			// during recovery that is transient, so let the caller retry.
+			if errors.Is(err, nats.ErrNoResponders) {
+				return rollback(fmt.Errorf("ebs mount had no responder: %w", vm.ErrMountRetryable))
 			}
 			return rollback(err)
 		}
+
+		var ebsMountResponse viperblocklegacyv1.EBSMountResponse
+		if err := json.Unmarshal(reply.Data, &ebsMountResponse); err != nil {
+			slog.Error("Failed to unmarshal volume response:", "err", err)
+			return rollback(err)
+		}
+
+		if ebsMountResponse.Error != "" {
+			slog.Error("Failed to mount volume", "error", ebsMountResponse.Error, "retryable", ebsMountResponse.Retryable)
+			if ebsMountResponse.Retryable {
+				return rollback(fmt.Errorf("failed to mount volume: %s: %w", ebsMountResponse.Error, vm.ErrMountRetryable))
+			}
+			return rollback(fmt.Errorf("failed to mount volume: %s", ebsMountResponse.Error))
+		}
+
+		slog.Debug("Mounted volume successfully", "response", ebsMountResponse.URI)
+		instance.EBSRequests.Requests[k].NBDURI = ebsMountResponse.URI
 		mounted = append(mounted, k)
 	}
 
@@ -296,22 +223,39 @@ func (a *volumeMounterAdapter) Unmount(ctx context.Context, instance *vm.VM) err
 
 	var sealErrs []error
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		// Every volume is attempted and the seal failures aggregated. An unsealed
-		// volume must NOT go available: a reattach on a node without the local
-		// WAL would find no checkpoint (bad superblock).
-		if err := a.unpublish(ctx, instance.AccountID, ebsRequest); err != nil {
-			slog.Error("Volume seal failed, leaving volume non-available",
-				"instance", instance.ID, "volume", ebsRequest.Name, "err", err)
-			sealErrs = append(sealErrs, fmt.Errorf("seal %s: %w", ebsRequest.Name, err))
+		ebsUnMountRequest, err := json.Marshal(ebsRequest)
+		if err != nil {
+			slog.Error("Failed to marshal volume payload for unmount", "err", err)
+			sealErrs = append(sealErrs, fmt.Errorf("marshal unmount request for %s: %w", ebsRequest.Name, err))
 			continue
 		}
-		slog.Info("Unmounted volume", "instance", instance.ID, "volume", ebsRequest.Name)
+
+		// ebs.unmount seals the block map to predastore. Every volume is
+		// attempted, and the failures are aggregated for the caller: the
+		// volume must NOT go available, because a reattach on a node without
+		// the local WAL would find no checkpoint (bad superblock).
+		sealed := true
+		msg, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.UnmountSubject(a.node), ebsUnMountRequest, unmountSealTimeout)
+		if err != nil {
+			slog.Error("Failed to unmount volume",
+				"name", ebsRequest.Name, "instance", instance.ID, "err", err)
+			sealErrs = append(sealErrs, fmt.Errorf("unmount %s: %w", ebsRequest.Name, err))
+			sealed = false
+		} else if sealErr := unmountResponseError(msg.Data); sealErr != nil {
+			slog.Error("Volume unmount seal failed, leaving volume non-available",
+				"instance", instance.ID, "volume", ebsRequest.Name, "err", sealErr)
+			sealErrs = append(sealErrs, fmt.Errorf("seal %s: %w", ebsRequest.Name, sealErr))
+			sealed = false
+		} else {
+			slog.Info("Unmounted volume",
+				"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
+		}
 
 		// Boot/root volumes must stay attached across stop/crash unmount: EFI is
 		// the wrong proxy for "boot" (BIOS-boot roots are Boot && !EFI), so gate on
 		// Boot too. Only DetachVolume and terminate release a boot volume; flipping
 		// it available here while the instance restarts splits the state record.
-		if !ebsRequest.EFI && !ebsRequest.Boot && a.volState != nil {
+		if sealed && !ebsRequest.EFI && !ebsRequest.Boot && a.volState != nil {
 			if err := a.volState.UpdateVolumeState(instance.AccountID, ebsRequest.Name, "available", "", ""); err != nil {
 				slog.Error("Failed to update volume state to available after unmount",
 					"volumeId", ebsRequest.Name, "err", err)
@@ -322,71 +266,87 @@ func (a *volumeMounterAdapter) Unmount(ctx context.Context, instance *vm.VM) err
 	return errors.Join(sealErrs...)
 }
 
-// abandonTimeout bounds one no-seal unpublish. Generous enough to cover a
-// provider killing an export stuck in the storage path, and far short of the
-// seal budget because nothing here seals.
+// abandonTimeout bounds one ebs.abandon request. Generous enough to cover
+// viperblockd's SIGKILL wait for an nbdkit stuck in the storage path, and far
+// short of the unmount's budget because nothing here seals.
 const abandonTimeout = 45 * time.Second
 
-// Abandon tears down this node's export of every volume the instance had open
-// through UnpublishVolume with NoSeal, so none of them is sealed.
+// Abandon tears down this node's export of every volume the instance had open,
+// without sealing any of them.
 //
 // Every volume is attempted and the failures are aggregated, because a volume
 // left exported is one the new owner cannot open — the caller is dropping a
 // superseded instance and needs to know whether it actually let go.
-func (a *volumeMounterAdapter) Abandon(ctx context.Context, instance *vm.VM, reason string) (err error) {
-	ctx, span := otel.Tracer(daemonTracerName).Start(ctx, "ebs.mounter.abandon", trace.WithAttributes(append([]attribute.KeyValue{
-		attribute.String("instance.id", instance.ID),
-		attribute.String("abandon.reason", reason),
-	}, natsmsg.AccountAttrs(instance.AccountID)...)...))
-	defer func() { endOpSpan(span, err) }()
-
+func (a *volumeMounterAdapter) Abandon(ctx context.Context, instance *vm.VM, reason string) error {
 	instance.EBSRequests.Mu.Lock()
 	defer instance.EBSRequests.Mu.Unlock()
 
 	var errs []error
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		if err := a.abandonOne(ctx, instance, ebsRequest.Name, reason); err != nil {
-			errs = append(errs, err)
+		payload, err := json.Marshal(viperblocklegacyv1.VolumeAbandonRequest{Volume: ebsRequest.Name, Reason: reason})
+		if err != nil {
+			errs = append(errs, fmt.Errorf("marshal abandon request for %s: %w", ebsRequest.Name, err))
+			continue
+		}
+
+		msg, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID,
+			viperblocklegacyv1.VolumeAbandonSubject(a.node), payload, abandonTimeout)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("abandon %s: %w", ebsRequest.Name, err))
+			continue
+		}
+
+		var resp viperblocklegacyv1.VolumeAbandonResponse
+		if err := json.Unmarshal(msg.Data, &resp); err != nil {
+			errs = append(errs, fmt.Errorf("unmarshal abandon response for %s: %w", ebsRequest.Name, err))
+			continue
+		}
+		if resp.Error != "" {
+			errs = append(errs, fmt.Errorf("abandon %s: %s", ebsRequest.Name, resp.Error))
+			continue
+		}
+		if resp.Abandoned {
+			slog.WarnContext(ctx, "Gave up a volume export to the node that owns the instance now",
+				"instance", instance.ID, "volume", ebsRequest.Name, "reason", reason)
 		}
 	}
+
 	return errors.Join(errs...)
 }
 
-// abandonOne drops this node's export of volumeID unsealed. A volume the
-// provider does not know is one this node no longer holds.
-func (a *volumeMounterAdapter) abandonOne(ctx context.Context, instance *vm.VM, volumeID, reason string) error {
-	abandonCtx, cancel := context.WithTimeout(ebsprovider.WithAccountID(ctx, instance.AccountID), a.abandonTimeout)
-	defer cancel()
-	unpublished, err := a.provider.UnpublishVolume(abandonCtx, ebsprovider.UnpublishVolumeRequest{
-		Versioned: ebsprovider.NewLatestVersioned(),
-		VolumeID:  volumeID,
-		NodeID:    a.node,
-		NoSeal:    true,
-	})
-	switch {
-	case errors.Is(err, ebsprovider.ErrNotFound):
-		return nil
-	case err != nil:
-		return fmt.Errorf("abandon %s (%s): %w", volumeID, reason, err)
+// MountOne sends ebs.mount for a single request and writes the resolved
+// NBDURI back into req.NBDURI. Used by hot-attach (Manager.AttachVolume).
+func (a *volumeMounterAdapter) MountOne(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
+	payload, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("marshal ebs.mount request: %w", err)
 	}
-	if unpublished.TornDown {
-		slog.WarnContext(ctx, "Gave up a volume export to the node that owns the instance now",
-			"instance", instance.ID, "volume", volumeID, "reason", reason)
+
+	reply, err := ebsRequestWithTrace(ctx, a.nc, accountID, viperblocklegacyv1.MountSubject(a.node), payload, 30*time.Second)
+	if err != nil {
+		return fmt.Errorf("ebs.mount NATS request: %w", err)
 	}
+
+	var resp viperblocklegacyv1.EBSMountResponse
+	if err := json.Unmarshal(reply.Data, &resp); err != nil {
+		return fmt.Errorf("unmarshal ebs.mount response: %w", err)
+	}
+	if resp.Error != "" {
+		return fmt.Errorf("ebs.mount returned error: %s", resp.Error)
+	}
+	if resp.URI == "" {
+		return vm.ErrMountAmbiguous
+	}
+
+	req.NBDURI = resp.URI
 	return nil
 }
 
-// MountOne mounts a single request and writes the resolved export back into
-// req. Used by hot-attach (Manager.AttachVolume).
-func (a *volumeMounterAdapter) MountOne(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
-	return a.publish(ctx, accountID, req)
-}
-
-// UnmountOne unmounts a single request and returns any error. The unmount
-// seals the volume's block map to the backend, so the caller decides whether
-// a failure blocks the volume's available transition.
+// UnmountOne sends ebs.unmount and returns any error. The handler seals the
+// volume's block map to predastore, so the caller decides whether a failure
+// blocks the volume's available transition.
 func (a *volumeMounterAdapter) UnmountOne(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
-	if err := a.unpublish(ctx, accountID, req); err != nil {
+	if err := a.unmountOne(ctx, accountID, req); err != nil {
 		slog.Error("UnmountOne failed", "volume", req.Name, "err", err)
 		return err
 	}
@@ -394,6 +354,42 @@ func (a *volumeMounterAdapter) UnmountOne(ctx context.Context, accountID string,
 	return nil
 }
 
+// unmountOne sends ebs.unmount and returns any error.
+func (a *volumeMounterAdapter) unmountOne(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
+	payload, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("marshal unmount request: %w", err)
+	}
+	msg, err := ebsRequestWithTrace(ctx, a.nc, accountID, viperblocklegacyv1.UnmountSubject(a.node), payload, unmountSealTimeout)
+	if err != nil {
+		return fmt.Errorf("ebs.unmount NATS request: %w", err)
+	}
+	return unmountResponseError(msg.Data)
+}
+
+// unmountResponseError reports a seal/unmount failure from an ebs.unmount
+// response payload: a non-empty Error, or a volume still reported mounted.
+// Returns nil when the unmount and its block-map seal succeeded, or when the
+// volume was already unmounted and sealed (NotFound) — viperblockd only
+// reports NotFound once the seal itself has completed, so a retry landing
+// here is an idempotent success, not a failure.
+func unmountResponseError(data []byte) error {
+	var resp viperblocklegacyv1.EBSUnMountResponse
+	if err := json.Unmarshal(data, &resp); err != nil {
+		return fmt.Errorf("unmarshal unmount response: %w", err)
+	}
+	if resp.NotFound {
+		return nil
+	}
+	if resp.Error != "" {
+		return fmt.Errorf("ebs.unmount returned error: %s", resp.Error)
+	}
+	if resp.Mounted {
+		return fmt.Errorf("volume still mounted after unmount")
+	}
+	return nil
+}
+
 // instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver by projecting
 // EC2 instance type info into the SDK-agnostic vm.InstanceTypeSpec.
 type instanceTypeResolverAdapter struct {
@@ -677,12 +673,13 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 	return vm.Deps{
 		NodeID:             d.node,
 		StateStore:         d.stateStore,
-		VolumeMounter:      newVolumeMounterAdapter(d.ebsProvider, d.node, d.volumeService),
+		VolumeMounter:      newVolumeMounterAdapter(d.natsConn, d.node, d.volumeService),
 		NetworkPlumber:     d.networkPlumber,
 		InstanceTypes:      newInstanceTypeResolverAdapter(d.resourceMgr),
 		Resources:          newResourceControllerAdapter(d.resourceMgr),
 		VolumeStateUpdater: d.volumeService,
 		InstanceCleaner:    newInstanceCleanerAdapter(d),
+		UEFIStores:         d.newUEFIStoreTerminator(),
 		Hooks: vm.ManagerHooks{
 			OnInstanceUp:           d.onInstanceUpHook(),
 			OnInstanceDown:         d.onInstanceDownHook(),

spinifex/runtime/daemon/vm_adapters_contract_test.go

@@ -1,749 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs/natsserve"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/otel"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-)
-
-var mounterTestNode atomic.Int64
-
-// newMounterTestNode returns a node name no other test on the shared NATS
-// server subscribes for, so per-node subjects never cross tests.
-func newMounterTestNode() string {
-	return fmt.Sprintf("node-mounter-%d", mounterTestNode.Add(1))
-}
-
-// newNATSMounter returns a mounter driving node's provider over nc.
-func newNATSMounter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
-	return newVolumeMounterAdapter(ebsprovider.NewNATSProvider(nc, ebsProviderRequestTimeout), node, volState)
-}
-
-func connectMounterTest(t *testing.T) *nats.Conn {
-	t.Helper()
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	t.Cleanup(nc.Close)
-	return nc
-}
-
-// replyJSON answers every request on subject with reply(request body).
-func replyJSON(t *testing.T, nc *nats.Conn, subject string, reply func(data []byte) any) {
-	t.Helper()
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
-		data, err := json.Marshal(reply(msg.Data))
-		if err != nil {
-			panic(err)
-		}
-		_ = msg.Respond(data)
-	})
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-}
-
-// newMemoryProvider returns a memory provider holding volumeIDs.
-func newMemoryProvider(t *testing.T, volumeIDs ...string) *ebsprovider.MemoryProvider {
-	t.Helper()
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	for _, id := range volumeIDs {
-		_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
-			Versioned:     ebsprovider.NewVersioned(),
-			VolumeID:      id,
-			CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 20},
-		})
-		require.NoError(t, err)
-	}
-	return provider
-}
-
-// serveMemoryProvider serves provider as nodeID's local provider.
-func serveMemoryProvider(t *testing.T, nc *nats.Conn, provider *ebsprovider.MemoryProvider, nodeID string) {
-	t.Helper()
-	stop, err := natsserve.Serve(t.Context(), nc, provider, natsserve.Options{NoQueueGroup: true, NodeID: nodeID})
-	require.NoError(t, err)
-	t.Cleanup(stop)
-}
-
-func volumeState(t *testing.T, provider *ebsprovider.MemoryProvider, volumeID string) ebsprovider.VolumeState {
-	t.Helper()
-	volume, err := provider.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})
-	require.NoError(t, err)
-	return volume.State
-}
-
-func mounterTestVM(volumeIDs ...string) *vm.VM {
-	instance := &vm.VM{ID: "i-mounter", AccountID: testAccountID}
-	for _, id := range volumeIDs {
-		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: id})
-	}
-	return instance
-}
-
-// contractCall is one request a fakeNodeProvider or a spy received.
-type contractCall struct {
-	subject string
-	version uint16
-	volume  string
-	noSeal  bool
-	account string
-}
-
-// fakeNodeProvider answers node's publish and unpublish subjects with the
-// replies the test sets, echoing each request's version unless a publish reply
-// sets its own, and records them.
-type fakeNodeProvider struct {
-	mu        sync.Mutex
-	publish   func(volumeID string) ebsprovider.PublishVolumeResponse
-	unpublish func(volumeID string) ebsprovider.UnpublishVolumeResponse
-	calls     []contractCall
-}
-
-// serveFakeNodeProvider starts a fake that publishes every volume as an NBD
-// export and unpublishes it successfully until the test says otherwise.
-func serveFakeNodeProvider(t *testing.T, nc *nats.Conn, node string) *fakeNodeProvider {
-	t.Helper()
-	f := &fakeNodeProvider{
-		publish:   func(volumeID string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://" + volumeID) },
-		unpublish: func(string) ebsprovider.UnpublishVolumeResponse { return ebsprovider.UnpublishVolumeResponse{} },
-	}
-	publishSubject, err := ebsprovider.PublishSubject(node)
-	require.NoError(t, err)
-	replyJSON(t, nc, publishSubject, func(data []byte) any {
-		var req ebsprovider.PublishVolumeRequest
-		_ = json.Unmarshal(data, &req)
-		f.mu.Lock()
-		f.calls = append(f.calls, contractCall{subject: publishSubject, version: req.SchemaVersion, volume: req.VolumeID})
-		reply := f.publish
-		f.mu.Unlock()
-		resp := reply(req.VolumeID)
-		if resp.SchemaVersion == 0 {
-			resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
-		}
-		if resp.Published != nil {
-			published := *resp.Published
-			published.VolumeID, published.NodeID = req.VolumeID, req.NodeID
-			resp.Published = &published
-		}
-		return resp
-	})
-	unpublishSubject, err := ebsprovider.UnpublishSubject(node)
-	require.NoError(t, err)
-	replyJSON(t, nc, unpublishSubject, func(data []byte) any {
-		var req ebsprovider.UnpublishVolumeRequest
-		_ = json.Unmarshal(data, &req)
-		f.mu.Lock()
-		f.calls = append(f.calls, contractCall{subject: unpublishSubject, version: req.SchemaVersion, volume: req.VolumeID, noSeal: req.NoSeal})
-		reply := f.unpublish
-		f.mu.Unlock()
-		resp := reply(req.VolumeID)
-		resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
-		return resp
-	})
-	return f
-}
-
-func (f *fakeNodeProvider) setPublish(reply func(volumeID string) ebsprovider.PublishVolumeResponse) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.publish = reply
-}
-
-func (f *fakeNodeProvider) setUnpublish(reply func(volumeID string) ebsprovider.UnpublishVolumeResponse) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.unpublish = reply
-}
-
-func (f *fakeNodeProvider) snapshot() []contractCall {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]contractCall(nil), f.calls...)
-}
-
-// unpublished returns, in order, the volumes the fake was asked to unpublish.
-func (f *fakeNodeProvider) unpublished() []string {
-	var volumes []string
-	for _, call := range f.snapshot() {
-		if strings.HasSuffix(call.subject, ".unmount") {
-			volumes = append(volumes, call.volume)
-		}
-	}
-	return volumes
-}
-
-func publishedNBD(uri string) ebsprovider.PublishVolumeResponse {
-	return ebsprovider.PublishVolumeResponse{Published: &ebsprovider.PublishedVolume{Export: ebsprovider.NewNBDExport(uri)}}
-}
-
-func providerRefusal(code ebsprovider.ErrorCode, message string) *ebsprovider.ProviderError {
-	return &ebsprovider.ProviderError{Code: code, Message: message}
-}
-
-func publishRefusal(code ebsprovider.ErrorCode) func(string) ebsprovider.PublishVolumeResponse {
-	return func(string) ebsprovider.PublishVolumeResponse {
-		return ebsprovider.PublishVolumeResponse{Error: providerRefusal(code, "refused: "+string(code))}
-	}
-}
-
-func unpublishRefusal(code ebsprovider.ErrorCode, message string) ebsprovider.UnpublishVolumeResponse {
-	return ebsprovider.UnpublishVolumeResponse{Error: providerRefusal(code, message)}
-}
-
-// spyOn records every request sent on subject without answering it, so a
-// request the mounter sends there is seen even when another subscriber
-// answers it.
-func spyOn(t *testing.T, nc *nats.Conn, subject string) func() []contractCall {
-	t.Helper()
-	var (
-		mu    sync.Mutex
-		calls []contractCall
-	)
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
-		var req ebsprovider.UnpublishVolumeRequest
-		_ = json.Unmarshal(msg.Data, &req)
-		mu.Lock()
-		calls = append(calls, contractCall{
-			subject: msg.Subject, version: req.SchemaVersion, volume: req.VolumeID,
-			noSeal: req.NoSeal, account: natsmsg.AccountIDFromMsg(msg),
-		})
-		mu.Unlock()
-	})
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	return func() []contractCall {
-		mu.Lock()
-		defer mu.Unlock()
-		return append([]contractCall(nil), calls...)
-	}
-}
-
-// Every mount, unmount and abandon goes to the node's provider over the
-// contract at the cluster's one version, and none uses a legacy subject.
-func TestVolumeMounterAdapter_EveryRequestAtVersion3OverTheContract(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	provider := newMemoryProvider(t, "vol-a", "vol-b", "vol-hot", "vol-moved")
-	serveMemoryProvider(t, nc, provider, node)
-	contractCalls := spyOn(t, nc, "ebs.provider.v1."+node+".*")
-	var legacySpies []func() []contractCall
-	for _, subject := range []string{"ebs." + node + ".*", "ebs.mount", "ebs.unmount", "ebs.abandon"} {
-		legacySpies = append(legacySpies, spyOn(t, nc, subject))
-	}
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
-
-	instance := mounterTestVM("vol-a", "vol-b")
-	require.NoError(t, adapter.Mount(t.Context(), instance))
-	for _, req := range instance.EBSRequests.Requests {
-		assert.Equal(t, "nbd+unix:///?socket=/memory/"+req.Name+".sock", req.NBDURI, "the published export must reach the request")
-		assert.Equal(t, ebsprovider.VolumeStateInUse, volumeState(t, provider, req.Name))
-	}
-
-	hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
-	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, hot))
-	assert.Equal(t, "nbd+unix:///?socket=/memory/vol-hot.sock", hot.NBDURI)
-	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, *hot))
-	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-hot"))
-
-	require.NoError(t, adapter.Unmount(t.Context(), instance))
-	for _, id := range []string{"vol-a", "vol-b"} {
-		assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, id), "unmount must unpublish %s", id)
-	}
-	assert.ElementsMatch(t, []string{"vol-a:available", "vol-b:available"}, volState.snapshot())
-
-	moved := mounterTestVM("vol-moved")
-	require.NoError(t, adapter.Mount(t.Context(), moved))
-	require.NoError(t, adapter.Abandon(t.Context(), moved, "superseded"))
-	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-moved"), "abandon must give the export up")
-
-	calls := contractCalls()
-	assert.Len(t, calls, 8, "two mounts, a hot attach and detach, two unmounts, a mount and an abandon")
-	for _, call := range calls {
-		assert.Equal(t, ebsprovider.SchemaVersion, call.version, "%s for %s", call.subject, call.volume)
-	}
-	for _, legacyCalls := range legacySpies {
-		assert.Empty(t, legacyCalls(), "no request may use a legacy subject")
-	}
-}
-
-// A relaunch refused as retry-later, or because the volume state is not found
-// yet, backs off as restore expects and mounts once the provider is ready.
-func TestVolumeMounterAdapter_RetryLaterRefusalBacksOffAndRecovers(t *testing.T) {
-	for _, code := range []ebsprovider.ErrorCode{ebsprovider.ErrorCodeUnavailable, ebsprovider.ErrorCodeNotFound} {
-		t.Run(string(code), func(t *testing.T) {
-			nc := connectMounterTest(t)
-			node := newMounterTestNode()
-			fake := serveFakeNodeProvider(t, nc, node)
-			fake.setPublish(publishRefusal(code))
-			adapter := newNATSMounter(nc, node, nil)
-
-			instance := mounterTestVM("vol-relaunch")
-			err := adapter.Mount(t.Context(), instance)
-			require.ErrorIs(t, err, vm.ErrMountRetryable, "restore must back off rather than give up")
-			hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
-			require.ErrorIs(t, adapter.MountOne(t.Context(), testAccountID, hot), vm.ErrMountRetryable)
-
-			fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://ready") })
-			require.NoError(t, adapter.Mount(t.Context(), instance))
-			assert.Equal(t, "nbd://ready", instance.EBSRequests.Requests[0].NBDURI)
-		})
-	}
-}
-
-// A refusal that retrying cannot fix fails the mount outright, so restore
-// does not back off forever against a provider that will never accept it.
-func TestVolumeMounterAdapter_PermanentRefusalIsNotRetryable(t *testing.T) {
-	codes := []ebsprovider.ErrorCode{
-		ebsprovider.ErrorCodeVolumeInUse, ebsprovider.ErrorCodeVolumeHeldElsewhere,
-		ebsprovider.ErrorCodeInternal, ebsprovider.ErrorCodeInvalidArgument,
-	}
-	for _, code := range codes {
-		t.Run(string(code), func(t *testing.T) {
-			nc := connectMounterTest(t)
-			node := newMounterTestNode()
-			fake := serveFakeNodeProvider(t, nc, node)
-			fake.setPublish(publishRefusal(code))
-			adapter := newNATSMounter(nc, node, nil)
-
-			err := adapter.Mount(t.Context(), mounterTestVM("vol-refused"))
-			require.Error(t, err)
-			assert.NotErrorIs(t, err, vm.ErrMountRetryable)
-		})
-	}
-}
-
-// A volume another node holds is refused with the held-elsewhere code, which
-// a provider only sends at version 3, and the refusal reaches Mount's caller
-// as the sentinel that tells it apart from an ordinary volume in use.
-func TestVolumeMounterAdapter_HeldElsewhereReachesTheMountCaller(t *testing.T) {
-	nc := connectMounterTest(t)
-	owner, other := newMounterTestNode(), newMounterTestNode()
-	provider := newMemoryProvider(t, "vol-held")
-	serveMemoryProvider(t, nc, provider, owner)
-	serveMemoryProvider(t, nc, provider, other)
-
-	require.NoError(t, newNATSMounter(nc, owner, nil).Mount(t.Context(), mounterTestVM("vol-held")))
-
-	err := newNATSMounter(nc, other, nil).Mount(t.Context(), mounterTestVM("vol-held"))
-	require.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
-	var providerErr *ebsprovider.ProviderError
-	require.ErrorAs(t, err, &providerErr)
-	assert.Equal(t, ebsprovider.ErrorCodeVolumeHeldElsewhere, providerErr.Code)
-	assert.Equal(t, ebsprovider.VolumeStateInUse, volumeState(t, provider, "vol-held"), "the refused mount must not take the volume from its owner")
-}
-
-// The unpublish budget is the provider's data-movement budget, not the
-// request timeout every other call gets: a seal that outlasts the request
-// timeout still completes, where a publish that slow fails.
-func TestVolumeMounterAdapter_UnpublishOutlastingRequestTimeoutCompletes(t *testing.T) {
-	const requestTimeout = 100 * time.Millisecond
-	const sealTime = 4 * requestTimeout
-
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
-		time.Sleep(sealTime)
-		return ebsprovider.UnpublishVolumeResponse{}
-	})
-	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
-		time.Sleep(sealTime)
-		return publishedNBD("nbd://" + volumeID)
-	})
-
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
-	adapter.provider = ebsprovider.NewNATSProvider(nc, requestTimeout)
-
-	require.NoError(t, adapter.Unmount(t.Context(), mounterTestVM("vol-slow-seal")))
-	assert.Equal(t, []string{"vol-slow-seal:available"}, volState.snapshot())
-	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"}))
-
-	err := adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"})
-	require.ErrorIs(t, err, context.DeadlineExceeded, "the publish must be held to the request timeout")
-}
-
-// A failed volume rolls back the volumes this Mount already published, and a
-// missing volume backs off rather than failing the relaunch.
-func TestVolumeMounterAdapter_MountRollsBackOnFailure(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	provider := newMemoryProvider(t, "vol-present")
-	serveMemoryProvider(t, nc, provider, node)
-	adapter := newNATSMounter(nc, node, nil)
-
-	err := adapter.Mount(t.Context(), mounterTestVM("vol-present", "vol-missing"))
-	require.ErrorIs(t, err, vm.ErrMountRetryable)
-	assert.ErrorIs(t, err, ebsprovider.ErrNotFound)
-	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-present"), "the published volume must be unpublished")
-}
-
-// stubMounterProvider publishes a fixed export without a NATS round trip.
-type stubMounterProvider struct {
-	export ebsprovider.VolumeExport
-}
-
-var _ mounterProvider = (*stubMounterProvider)(nil)
-
-func (s *stubMounterProvider) PublishVolume(_ context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
-	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: s.export}, nil
-}
-
-func (s *stubMounterProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
-	return &ebsprovider.UnpublishedVolume{}, nil
-}
-
-// A block export replaces whatever export the request carried before, so a
-// relaunch never leaves both an NBD URI and a block device set.
-func TestVolumeMounterAdapter_BlockExportReplacesNBDURI(t *testing.T) {
-	adapter := newVolumeMounterAdapter(&stubMounterProvider{export: ebsprovider.VolumeExport{
-		Kind:  ebsprovider.ExportKindBlock,
-		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
-	}}, "node-stub", nil)
-
-	req := &viperblocklegacyv1.EBSRequest{Name: "vol-block", NBDURI: "nbd://stale"}
-	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, req))
-	assert.Empty(t, req.NBDURI)
-	assert.Equal(t, "/dev/sdx", req.BlockDevice)
-	assert.Equal(t, uint64(1<<30), req.BlockSizeBytes)
-
-	adapter.provider = &stubMounterProvider{export: ebsprovider.NewNBDExport("nbd://fresh")}
-	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, req))
-	assert.Equal(t, "nbd://fresh", req.NBDURI)
-	assert.Empty(t, req.BlockDevice)
-	assert.Zero(t, req.BlockSizeBytes)
-}
-
-// A publish whose outcome is unknown may still have started an export: the
-// reply carried no usable export, could not be read, or never came. Hot attach
-// gets ErrMountAmbiguous with the request untouched, and Mount tears it down.
-func TestVolumeMounterAdapter_AmbiguousPublishIsTornDown(t *testing.T) {
-	const requestTimeout = 100 * time.Millisecond
-	replies := map[string]struct {
-		reply   func() ebsprovider.PublishVolumeResponse
-		timeout time.Duration
-		cause   error
-	}{
-		"NoPublishedVolume": {
-			reply: func() ebsprovider.PublishVolumeResponse { return ebsprovider.PublishVolumeResponse{} },
-			cause: ebsprovider.ErrNoPublishedVolume,
-		},
-		"EmptyExport": {
-			reply: func() ebsprovider.PublishVolumeResponse { return publishedNBD("") },
-			cause: ebsprovider.ErrInvalidExport,
-		},
-		"UnsupportedReplyVersion": {
-			reply: func() ebsprovider.PublishVolumeResponse {
-				resp := publishedNBD("nbd://future")
-				resp.SchemaVersion = ebsprovider.SchemaVersion + 1
-				return resp
-			},
-			cause: ebsprovider.ErrUnsupportedVersion,
-		},
-		"Timeout": {
-			reply: func() ebsprovider.PublishVolumeResponse {
-				time.Sleep(4 * requestTimeout)
-				return publishedNBD("nbd://late")
-			},
-			timeout: requestTimeout,
-			cause:   context.DeadlineExceeded,
-		},
-	}
-	for name, tt := range replies {
-		t.Run(name, func(t *testing.T) {
-			nc := connectMounterTest(t)
-			node := newMounterTestNode()
-			fake := serveFakeNodeProvider(t, nc, node)
-			fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return tt.reply() })
-			adapter := newNATSMounter(nc, node, nil)
-			if tt.timeout > 0 {
-				adapter.provider = ebsprovider.NewNATSProvider(nc, tt.timeout)
-			}
-
-			hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot", NBDURI: "nbd://stale"}
-			err := adapter.MountOne(t.Context(), testAccountID, hot)
-			require.ErrorIs(t, err, vm.ErrMountAmbiguous)
-			require.ErrorIs(t, err, tt.cause)
-			assert.NotErrorIs(t, err, vm.ErrMountRetryable)
-			assert.Equal(t, "nbd://stale", hot.NBDURI, "a failed publish must not overwrite the request's export")
-
-			err = adapter.Mount(t.Context(), mounterTestVM("vol-ambiguous"))
-			require.ErrorIs(t, err, vm.ErrMountAmbiguous)
-			assert.Equal(t, []string{"vol-ambiguous"}, fake.unpublished(), "the ambiguous volume must be unpublished")
-		})
-	}
-}
-
-// A caller that gives up during a later volume's publish still gets that volume
-// and the ones already published unpublished, rather than left exported.
-func TestVolumeMounterAdapter_MountRollbackOutlivesCancelledCaller(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	arrived, release := make(chan struct{}), make(chan struct{})
-	t.Cleanup(func() { close(release) })
-	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
-		if volumeID == "vol-slow" {
-			close(arrived)
-			<-release
-		}
-		return publishedNBD("nbd://" + volumeID)
-	})
-	adapter := newNATSMounter(nc, node, nil)
-
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cancel()
-	go func() {
-		<-arrived
-		cancel()
-	}()
-	err := adapter.Mount(ctx, mounterTestVM("vol-first", "vol-slow"))
-	require.ErrorIs(t, err, context.Canceled)
-	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
-	assert.NotErrorIs(t, err, vm.ErrMountRetryable)
-	assert.ElementsMatch(t, []string{"vol-first", "vol-slow"}, fake.unpublished(),
-		"the published volume and the one whose publish the provider received must be unpublished despite the cancelled caller")
-}
-
-// A hot attach whose caller gives up after the provider received the publish
-// is ambiguous, so the attach unpublishes it, and that teardown still reaches
-// the provider under the cancelled context.
-func TestVolumeMounterAdapter_MountOneCancelledAfterSendIsAmbiguous(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	arrived, release := make(chan struct{}), make(chan struct{})
-	t.Cleanup(func() { close(release) })
-	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
-		close(arrived)
-		<-release
-		return publishedNBD("nbd://" + volumeID)
-	})
-	adapter := newNATSMounter(nc, node, nil)
-
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cancel()
-	go func() {
-		<-arrived
-		cancel()
-	}()
-	req := &viperblocklegacyv1.EBSRequest{Name: "vol-hot", NBDURI: "nbd://stale"}
-	err := adapter.MountOne(ctx, testAccountID, req)
-	require.ErrorIs(t, err, context.Canceled)
-	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
-	assert.NotErrorIs(t, err, vm.ErrMountRetryable)
-	assert.Equal(t, "nbd://stale", req.NBDURI, "a failed publish must not overwrite the request's export")
-
-	require.NoError(t, adapter.UnmountOne(ctx, testAccountID, *req))
-	assert.Equal(t, []string{"vol-hot"}, fake.unpublished())
-}
-
-// A caller already gone when a publish would start sends nothing, so neither
-// call is ambiguous and Mount unpublishes only the volumes it published.
-func TestVolumeMounterAdapter_PublishAfterCallerGoneSendsNothing(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
-
-	cancelled, cancel := context.WithCancel(t.Context())
-	cancel()
-	err := adapter.MountOne(cancelled, testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-hot"})
-	require.ErrorIs(t, err, context.Canceled)
-	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous)
-
-	err = adapter.Mount(cancelled, mounterTestVM("vol-never"))
-	require.ErrorIs(t, err, context.Canceled)
-	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous)
-	assert.Empty(t, fake.snapshot(), "no request may reach the provider for a caller already gone")
-}
-
-// Every contract request names the account it acts for on the wire, so the
-// provider's server span is attributable, and an unattributed one names none.
-func TestVolumeMounterAdapter_ContractRequestsCarryTheAccount(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	serveFakeNodeProvider(t, nc, node)
-	calls := spyOn(t, nc, "ebs.provider.v1."+node+".*")
-	adapter := newNATSMounter(nc, node, nil)
-
-	instance := mounterTestVM("vol-a")
-	require.NoError(t, adapter.Mount(t.Context(), instance))
-	require.NoError(t, adapter.Unmount(t.Context(), instance))
-	require.NoError(t, adapter.Abandon(t.Context(), instance, "superseded"))
-	hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
-	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, hot))
-	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, *hot))
-	require.NoError(t, adapter.MountOne(t.Context(), "", &viperblocklegacyv1.EBSRequest{Name: "vol-none"}))
-
-	require.Eventually(t, func() bool { return len(calls()) == 6 }, 5*time.Second, 10*time.Millisecond)
-	for _, call := range calls() {
-		want := testAccountID
-		if call.volume == "vol-none" {
-			want = ""
-		}
-		assert.Equal(t, want, call.account, "%s for %s", call.subject, call.volume)
-	}
-}
-
-// The provider's request span carries the account under the caller's trace,
-// with no mounter span of its own in between.
-func TestVolumeMounterAdapter_RequestSpanCarriesTheAccountUnderTheCaller(t *testing.T) {
-	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
-
-	ctx, caller := otel.Tracer("test").Start(t.Context(), "caller")
-	require.NoError(t, adapter.MountOne(ctx, testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-acct"}))
-	caller.End()
-
-	request := clientSpan(t, sr)
-	assert.Equal(t, "ebs.volume.publish", request.Name())
-	assert.Equal(t, testAccountID, spanAccountOf(request), "the request span must name the account it acted for")
-	assert.Equal(t, caller.SpanContext().SpanID(), request.Parent().SpanID(), "the request span must join the caller directly")
-}
-
-// A detach spends part of its deadline on the unplug, so the seal must not
-// inherit what is left: a caller context already cancelled, or one shorter
-// than the seal, still gets the seal completed.
-func TestVolumeMounterAdapter_UnpublishOutlivesCallerContext(t *testing.T) {
-	const sealTime = 200 * time.Millisecond
-
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	var sealed atomic.Int32
-	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
-		time.Sleep(sealTime)
-		sealed.Add(1)
-		return ebsprovider.UnpublishVolumeResponse{}
-	})
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
-
-	short, cancelShort := context.WithTimeout(t.Context(), sealTime/4)
-	defer cancelShort()
-	require.NoError(t, adapter.UnmountOne(short, testAccountID, viperblocklegacyv1.EBSRequest{Name: "vol-detach"}))
-
-	cancelled, cancel := context.WithCancel(t.Context())
-	cancel()
-	require.NoError(t, adapter.Unmount(cancelled, mounterTestVM("vol-stop")))
-	assert.Equal(t, int32(2), sealed.Load())
-	assert.Equal(t, []string{"vol-stop:available"}, volState.snapshot())
-}
-
-// Abandon drops each export unsealed, at the version that carries NoSeal, and
-// tries every volume: an export torn down is logged, one not held here or
-// unknown to the provider is the ordinary case, and a refusal is returned.
-func TestVolumeMounterAdapter_AbandonUnpublishesWithoutSealing(t *testing.T) {
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	fake := serveFakeNodeProvider(t, nc, node)
-	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
-		switch volumeID {
-		case "vol-torn":
-			return ebsprovider.UnpublishVolumeResponse{TornDown: true}
-		case "vol-unknown":
-			return unpublishRefusal(ebsprovider.ErrorCodeNotFound, "volume not found")
-		case "vol-refused":
-			return unpublishRefusal(ebsprovider.ErrorCodeUnavailable, "export busy")
-		}
-		return ebsprovider.UnpublishVolumeResponse{}
-	})
-	logs := captureSlogForTest(t)
-	adapter := newNATSMounter(nc, node, nil)
-
-	err := adapter.Abandon(t.Context(), mounterTestVM("vol-torn", "vol-absent", "vol-unknown", "vol-refused"), "superseded")
-	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
-	assert.Contains(t, err.Error(), "abandon vol-refused")
-	for _, volume := range []string{"vol-torn", "vol-absent", "vol-unknown"} {
-		assert.NotContains(t, err.Error(), volume)
-	}
-
-	calls := fake.snapshot()
-	require.Len(t, calls, 4, "every volume is attempted")
-	for _, call := range calls {
-		assert.True(t, call.noSeal, "%s must be dropped unsealed", call.volume)
-		assert.Equal(t, ebsprovider.NoSealSchemaVersion, call.version, "%s must be sent at the version carrying NoSeal", call.volume)
-	}
-	assert.Contains(t, err.Error(), "abandon vol-refused (superseded)", "the failure must say why the volume was abandoned")
-	const gaveUp = "Gave up a volume export"
-	assert.Contains(t, logs.String(), "instance=i-mounter volume=vol-torn reason=superseded")
-	assert.Equal(t, 1, strings.Count(logs.String(), gaveUp), "only a torn-down export is reported as given up")
-}
-
-// An abandon that finds no provider has not let go, so the caller is told.
-func TestVolumeMounterAdapter_AbandonNoResponderFails(t *testing.T) {
-	nc := connectMounterTest(t)
-	adapter := newNATSMounter(nc, newMounterTestNode(), nil)
-
-	err := adapter.Abandon(t.Context(), mounterTestVM("vol-1", "vol-2"), "superseded")
-	require.ErrorIs(t, err, nats.ErrNoResponders)
-	assert.Contains(t, err.Error(), "vol-1")
-	assert.Contains(t, err.Error(), "vol-2", "every volume is attempted")
-}
-
-// An abandon that is never answered gives up within its own bound rather
-// than the provider's data-movement budget for an unpublish.
-func TestVolumeMounterAdapter_AbandonGivesUpWithinItsBound(t *testing.T) {
-	const bound = 200 * time.Millisecond
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	spyOn(t, nc, "ebs.provider.v1."+node+".*")
-	adapter := newNATSMounter(nc, node, nil)
-	adapter.abandonTimeout = bound
-
-	done := make(chan error, 1)
-	go func() { done <- adapter.Abandon(t.Context(), mounterTestVM("vol-silent"), "superseded") }()
-	select {
-	case err := <-done:
-		require.ErrorIs(t, err, context.DeadlineExceeded)
-	case <-time.After(20 * bound):
-		t.Fatal("abandon outlasted its bound")
-	}
-}
-
-// The abandon span names the instance, the account and why it was abandoned,
-// and the provider's request spans sit under it.
-func TestVolumeMounterAdapter_AbandonSpanRecordsTheReason(t *testing.T) {
-	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	node := newMounterTestNode()
-	serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
-
-	require.NoError(t, adapter.Abandon(t.Context(), mounterTestVM("vol-moved"), "superseded"))
-
-	request := clientSpan(t, sr)
-	var abandon sdktrace.ReadOnlySpan
-	for _, s := range sr.Ended() {
-		if s.Name() == "ebs.mounter.abandon" {
-			abandon = s
-		}
-	}
-	require.NotNil(t, abandon)
-	assert.Equal(t, "superseded", spanAttribute(abandon, "abandon.reason"))
-	assert.Equal(t, "i-mounter", spanAttribute(abandon, "instance.id"))
-	assert.Equal(t, testAccountID, spanAccountOf(abandon))
-	assert.Equal(t, abandon.SpanContext().SpanID(), request.Parent().SpanID())
-}

spinifex/runtime/daemon/vm_adapters_coverage_test.go

@@ -61,6 +61,16 @@ func closedAdapterTestConn(t *testing.T) *nats.Conn {
 	return nc
 }
 
+// respondJSON subscribes subject on nc and answers every request with reply.
+func respondJSON(t *testing.T, nc *nats.Conn, subject string, reply any) {
+	t.Helper()
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(data) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
 // --- stateStoreAdapter ---
 
 func TestStateStoreAdapter_DelegatesToJetStream(t *testing.T) {
@@ -182,18 +192,15 @@ func TestBuildVMManagerDeps_BackingStoreReady(t *testing.T) {
 
 // --- volumeMounterAdapter ---
 
-// A connection that is down refuses the publish before sending it, so restore
-// backs off and retries, and there is no export to tear down.
-func TestVolumeMounterAdapter_Mount_DisconnectedIsRetryable(t *testing.T) {
-	adapter := newNATSMounter(closedAdapterTestConn(t), "node-closed", nil)
+func TestVolumeMounterAdapter_Mount_ClosedConnIsNotRetryable(t *testing.T) {
+	adapter := newVolumeMounterAdapter(closedAdapterTestConn(t), "node-closed", nil)
 	instance := &vm.VM{ID: "i-mount-closed"}
 	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-closed"}}
 
 	err := adapter.Mount(t.Context(), instance)
 	require.ErrorIs(t, err, nats.ErrConnectionClosed)
-	require.ErrorIs(t, err, vm.ErrMountRetryable, "restore must back off until the connection is back")
-	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous, "a request never sent left nothing to tear down")
-	require.ErrorIs(t, adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}), vm.ErrMountRetryable)
+	assert.NotErrorIs(t, err, vm.ErrMountRetryable,
+		"only a missing responder means viperblockd is still starting")
 }
 
 // recordingVolumeState records UpdateVolumeState calls and returns err.
@@ -212,7 +219,7 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 
 	t.Run("no responder fails the seal and leaves the volume", func(t *testing.T) {
 		volState := &recordingVolumeState{}
-		adapter := newNATSMounter(nc, uniqueAdapterID("node-unmount"), volState)
+		adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-unmount"), volState)
 		instance := &vm.VM{ID: "i-unmount-none"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}, {Name: "vol-b"}}
 
@@ -225,9 +232,9 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 
 	t.Run("state update failure does not fail the unmount", func(t *testing.T) {
 		node := uniqueAdapterID("node-unmount")
-		serveFakeNodeProvider(t, nc, node)
+		respondJSON(t, nc, "ebs."+node+".unmount", viperblocklegacyv1.EBSUnMountResponse{Volume: "vol-a"})
 		volState := &recordingVolumeState{err: errInjected}
-		adapter := newNATSMounter(nc, node, volState)
+		adapter := newVolumeMounterAdapter(nc, node, volState)
 		instance := &vm.VM{ID: "i-unmount-state"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}}
 
@@ -236,6 +243,74 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 	})
 }
 
+func TestVolumeMounterAdapter_Abandon(t *testing.T) {
+	nc := connectAdapterTestNATS(t, sharedNATSURL)
+
+	// abandonResponder answers per volume so one request can mix outcomes.
+	abandonResponder := func(t *testing.T, node string, replies map[string][]byte) *[]viperblocklegacyv1.VolumeAbandonRequest {
+		t.Helper()
+		var (
+			mu   sync.Mutex
+			seen []viperblocklegacyv1.VolumeAbandonRequest
+		)
+		sub, err := nc.Subscribe(viperblocklegacyv1.VolumeAbandonSubject(node), func(msg *nats.Msg) {
+			var req viperblocklegacyv1.VolumeAbandonRequest
+			_ = json.Unmarshal(msg.Data, &req)
+			mu.Lock()
+			seen = append(seen, req)
+			mu.Unlock()
+			_ = msg.Respond(replies[req.Volume])
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+		return &seen
+	}
+	instanceWith := func(volumes ...string) *vm.VM {
+		v := &vm.VM{ID: "i-abandon", AccountID: testAccountID}
+		for _, name := range volumes {
+			v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: name})
+		}
+		return v
+	}
+
+	t.Run("no responder", func(t *testing.T) {
+		adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-abandon"), nil)
+		err := adapter.Abandon(t.Context(), instanceWith("vol-1"), "superseded")
+		assert.ErrorIs(t, err, nats.ErrNoResponders)
+	})
+
+	t.Run("outcomes per volume", func(t *testing.T) {
+		node := uniqueAdapterID("node-abandon")
+		seen := abandonResponder(t, node, map[string][]byte{
+			"vol-abandoned": []byte(`{"abandoned":true}`),
+			"vol-absent":    []byte(`{"abandoned":false}`),
+		})
+		adapter := newVolumeMounterAdapter(nc, node, nil)
+
+		require.NoError(t, adapter.Abandon(t.Context(), instanceWith("vol-abandoned", "vol-absent"), "superseded"),
+			"a volume not exported here is the ordinary case, not a failure")
+		require.Len(t, *seen, 2)
+		assert.Equal(t, viperblocklegacyv1.VolumeAbandonRequest{Volume: "vol-abandoned", Reason: "superseded"}, (*seen)[0])
+	})
+
+	t.Run("failures are joined and do not stop the sweep", func(t *testing.T) {
+		node := uniqueAdapterID("node-abandon")
+		seen := abandonResponder(t, node, map[string][]byte{
+			"vol-garbled": []byte(`not json`),
+			"vol-refused": []byte(`{"error":"export busy"}`),
+			"vol-ok":      []byte(`{"abandoned":true}`),
+		})
+		adapter := newVolumeMounterAdapter(nc, node, nil)
+
+		err := adapter.Abandon(t.Context(), instanceWith("vol-garbled", "vol-refused", "vol-ok"), "fenced")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "unmarshal abandon response for vol-garbled")
+		assert.Contains(t, err.Error(), "abandon vol-refused: export busy")
+		assert.NotContains(t, err.Error(), "vol-ok")
+		assert.Len(t, *seen, 3, "every volume is attempted")
+	})
+}
+
 // --- resourceControllerAdapter ---
 
 func TestResourceControllerAdapter_UnknownAndKnownType(t *testing.T) {

spinifex/runtime/daemon/vm_adapters_invariants_test.go

@@ -1,10 +1,12 @@
 package daemon
 
 import (
+	"encoding/json"
 	"testing"
 
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -21,10 +23,18 @@ import (
 func TestUnmountNeverReleasesBootVolume(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
 
-	// Stand in for the provider: every unpublish succeeds (sealed).
-	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	// Stand in for the ebs daemon: every ebs.<node>.unmount succeeds (sealed).
+	sub, err := daemon.natsConn.Subscribe(viperblocklegacyv1.UnmountSubject(daemon.node), func(msg *nats.Msg) {
+		var req viperblocklegacyv1.EBSRequest
+		require.NoError(t, json.Unmarshal(msg.Data, &req))
+		data, err := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false})
+		require.NoError(t, err)
+		require.NoError(t, msg.Respond(data))
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
 
 	inst := &vm.VM{
 		ID: "i-unmount-boot",

spinifex/runtime/daemon/vm_adapters_test.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"context"
+	"encoding/json"
 	"reflect"
 	"testing"
 
@@ -233,6 +234,118 @@ func TestOnInstanceDownHook_OnlyRemovesTargetedInstance(t *testing.T) {
 	assert.False(t, dropPresent)
 }
 
+func TestVolumeMounterAdapter_MountOne(t *testing.T) {
+	tests := []struct {
+		name       string
+		responder  func(t *testing.T, msg *nats.Msg)
+		skipSub    bool
+		wantErr    bool
+		wantErrSub string
+		wantErrIs  error
+		wantNBDURI string
+		initialURI string
+	}{
+		{
+			name: "HappyPath_UpdatesNBDURI",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-vol"}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantNBDURI: "nbd://mounted-vol",
+		},
+		{
+			name:       "NATSNoResponders_ReturnsError",
+			skipSub:    true,
+			wantErr:    true,
+			wantErrSub: "ebs.mount NATS request",
+		},
+		{
+			name: "UnmarshalFailure_ReturnsError",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				require.NoError(t, msg.Respond([]byte("not json")))
+			},
+			wantErr:    true,
+			wantErrSub: "unmarshal ebs.mount response",
+		},
+		{
+			name: "ResponseError_IncludedInError",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := viperblocklegacyv1.EBSMountResponse{Error: "boom"}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantErr:    true,
+			wantErrSub: "boom",
+		},
+		{
+			name: "EmptyURI_ReturnsErrMountAmbiguous",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := viperblocklegacyv1.EBSMountResponse{URI: ""}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			wantErr:   true,
+			wantErrIs: vm.ErrMountAmbiguous,
+		},
+		{
+			name: "EmptyURI_PreservesInitialNBDURIOnFailure",
+			responder: func(t *testing.T, msg *nats.Msg) {
+				resp := viperblocklegacyv1.EBSMountResponse{URI: ""}
+				data, err := json.Marshal(resp)
+				require.NoError(t, err)
+				require.NoError(t, msg.Respond(data))
+			},
+			initialURI: "nbd://stale",
+			wantErr:    true,
+			wantErrIs:  vm.ErrMountAmbiguous,
+			wantNBDURI: "nbd://stale",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			daemon := createTestDaemon(t, sharedNATSURL)
+			adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+			if !tt.skipSub {
+				sub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+					tt.responder(t, msg)
+				})
+				require.NoError(t, err)
+				defer sub.Unsubscribe()
+			}
+
+			req := &viperblocklegacyv1.EBSRequest{
+				Name:       "vol-mountone",
+				DeviceName: "/dev/sdf",
+				NBDURI:     tt.initialURI,
+			}
+
+			err := adapter.MountOne(t.Context(), "", req)
+			if tt.wantErr {
+				require.Error(t, err)
+				if tt.wantErrSub != "" {
+					assert.Contains(t, err.Error(), tt.wantErrSub)
+				}
+				if tt.wantErrIs != nil {
+					assert.ErrorIs(t, err, tt.wantErrIs)
+				}
+				assert.Equal(t, tt.wantNBDURI, req.NBDURI,
+					"NBDURI must not be overwritten on failure")
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantNBDURI, req.NBDURI,
+				"happy path must write resolved NBDURI back to req")
+		})
+	}
+}
+
 // --- ReleaseGPU ---
 
 // ReleaseGPU is a no-op when the daemon has no GPU manager.
@@ -561,9 +674,56 @@ func TestInstanceCleanerAdapter_ReleaseAttachedENIs_ListInstanceENIsErrorTolerat
 		"an enumeration failure must not surface as a terminate error")
 }
 
+// TestVolumeMounterAdapter_Mount_WrapsErrMountRetryable verifies that a
+// viperblockd ebs.mount response with Retryable=true is wrapped so
+// vm.Manager's recovery-relaunch retry can match it with errors.Is. A
+// Retryable=false response must NOT match, so relaunchAll still fails fast
+// on permanent mount errors.
+func TestVolumeMounterAdapter_Mount_WrapsErrMountRetryable(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	sub, err := daemon.natsConn.Subscribe("ebs."+daemon.node+".mount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSMountResponse{Error: "state not found", Retryable: true}
+		data, _ := json.Marshal(resp)
+		_ = msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instance := &vm.VM{ID: "i-mount-retryable"}
+	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-retryable"}}
+
+	err = adapter.Mount(t.Context(), instance)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, vm.ErrMountRetryable,
+		"a Retryable mount response must wrap vm.ErrMountRetryable so relaunchAll can retry it")
+}
+
+func TestVolumeMounterAdapter_Mount_PermanentErrorNotWrapped(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+
+	sub, err := daemon.natsConn.Subscribe("ebs."+daemon.node+".mount", func(msg *nats.Msg) {
+		resp := viperblocklegacyv1.EBSMountResponse{Error: "volume vol-permanent is already mounted read_only=true on this node"}
+		data, _ := json.Marshal(resp)
+		_ = msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instance := &vm.VM{ID: "i-mount-permanent"}
+	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-permanent"}}
+
+	err = adapter.Mount(t.Context(), instance)
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, vm.ErrMountRetryable,
+		"a non-Retryable mount response must not be mistaken for a transient failure")
+}
+
 func TestVolumeMounterAdapter_Mount_NoResponderIsRetryable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
 
 	// No subscriber on the mount subject: viperblockd is still starting, so
 	// the request returns nats.ErrNoResponders, which must be retryable.

spinifex/runtime/daemon/vm_adapters_trace_test.go

@@ -1,10 +1,12 @@
 package daemon
 
 import (
+	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"testing"
-	"time"
 
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -35,65 +37,70 @@ func withRecordedSpans(t *testing.T) *tracetest.SpanRecorder {
 	return sr
 }
 
-// requestOnceOn answers a single request on subject with an empty JSON body
-// and hands back the headers it carried, so a test can assert on the wire.
-func requestOnceOn(t *testing.T, nc *nats.Conn, subject string) <-chan nats.Header {
-	t.Helper()
-	headerCh := make(chan nats.Header, 1)
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
-		headerCh <- msg.Header
-		require.NoError(t, msg.Respond([]byte("{}")))
-	})
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	return headerCh
-}
-
 // TestEbsRequestWithTrace_HeaderCarriesTraceparent pins the minimum bar: the
-// outbound ebs.* NATS message carries a non-empty traceparent header, so a
-// consumer that calls natsmsg.ExtractTraceContext has something to join.
+// outbound ebs.mount NATS message carries a non-empty traceparent header, so
+// a consumer that calls natsmsg.ExtractTraceContext has something to join.
 func TestEbsRequestWithTrace_HeaderCarriesTraceparent(t *testing.T) {
 	withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	subject := "ebs." + newMounterTestNode() + ".delete"
-	headerCh := requestOnceOn(t, nc, subject)
+	daemon := createTestDaemon(t, sharedNATSURL)
 
-	_, err := ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
+	headerCh := make(chan nats.Header, 1)
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+		headerCh <- msg.Header
+		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://traced-vol"}
+		data, marshalErr := json.Marshal(resp)
+		require.NoError(t, marshalErr)
+		require.NoError(t, msg.Respond(data))
+	})
 	require.NoError(t, err)
+	defer sub.Unsubscribe()
 
-	assert.NotEmpty(t, (<-headerCh).Get("traceparent"), "outbound ebs.* request must carry a traceparent header")
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	req := &viperblocklegacyv1.EBSRequest{Name: "vol-traced", DeviceName: "/dev/sdf"}
+	require.NoError(t, adapter.MountOne(t.Context(), "", req))
+
+	hdr := <-headerCh
+	assert.NotEmpty(t, hdr.Get("traceparent"), "outbound ebs.mount request must carry a traceparent header")
 }
 
 // TestEbsRequestWithTrace_ProducerConsumerLinked proves the actual parent/child
-// linkage: the consumer span opened via natsmsg.StartConsumerSpan (the helper
-// viperblockd's legacy handlers use) shares the producer span's trace ID and
-// names it as its parent, rather than rooting a new trace.
+// linkage: the consumer span opened via natsmsg.StartConsumerSpan (the same
+// helper viperblockd's ebs.mount/ebs.unmount handlers use) shares the producer
+// span's trace ID and names it as its parent, rather than rooting a new trace
+// — the exact gap this fix closes for the 81 mount + 81 unmount rooted spans
+// the trace analysis found.
 func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	subject := "ebs." + newMounterTestNode() + ".delete"
+	daemon := createTestDaemon(t, sharedNATSURL)
 
 	consumerEnded := make(chan struct{})
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
 		defer close(consumerEnded)
+		// Mirrors viperblockd's ebs.unmount handler: open a consumer span
+		// joining whatever trace context the producer injected.
 		_, span := natsmsg.StartConsumerSpan(msg)
 		defer span.End()
-		require.NoError(t, msg.Respond([]byte("{}")))
+
+		resp := viperblocklegacyv1.EBSUnMountResponse{}
+		data, marshalErr := json.Marshal(resp)
+		require.NoError(t, marshalErr)
+		require.NoError(t, msg.Respond(data))
 	})
 	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
 	defer sub.Unsubscribe()
 
-	_, err = ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
-	require.NoError(t, err)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	require.NoError(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-linked"}))
 	// The handler replies before its deferred span.End runs, so the consumer
-	// span may not be recorded yet when the request returns.
+	// span may not be recorded yet when UnmountOne returns.
 	<-consumerEnded
 
 	spans := sr.Ended()
 	require.Len(t, spans, 2, "expected one producer span and one consumer span")
 
+	// Both spans share the "NATS ebs.node-1.unmount" name; distinguish by kind
+	// (producer = client, consumer = server, as set by ebsRequestWithTrace and
+	// natsmsg.StartConsumerSpan respectively).
 	var producer, consumer sdktrace.ReadOnlySpan
 	for _, s := range spans {
 		switch s.SpanKind() {
@@ -112,21 +119,43 @@ func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 		"consumer span's parent must be the producer span")
 }
 
+// mountOnceOn answers a single ebs.mount on subject and hands back the headers
+// the adapter sent, so a test can assert on the wire rather than the span only.
+func mountOnceOn(t *testing.T, nc *nats.Conn, subject string) <-chan nats.Header {
+	t.Helper()
+	headerCh := make(chan nats.Header, 1)
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+		headerCh <- msg.Header
+		data, marshalErr := json.Marshal(viperblocklegacyv1.EBSMountResponse{URI: "nbd://vol"})
+		require.NoError(t, marshalErr)
+		require.NoError(t, msg.Respond(data))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return headerCh
+}
+
 // The ebs hop used to root its own trace from context.Background(), so block
 // storage work appeared as an orphan trace no request could be followed into.
 // Given a caller's context it must extend that trace instead.
 func TestEbsRequestWithTrace_JoinsTheCallersTrace(t *testing.T) {
 	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	subject := "ebs." + newMounterTestNode() + ".delete"
-	requestOnceOn(t, nc, subject)
+	daemon := createTestDaemon(t, sharedNATSURL)
+	mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
 
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
 	ctx, caller := otel.Tracer("test").Start(t.Context(), "caller")
-	_, err := ebsRequestWithTrace(ctx, nc, "000000000042", subject, []byte("{}"), 5*time.Second)
-	require.NoError(t, err)
+	require.NoError(t, adapter.MountOne(ctx, "000000000042", &viperblocklegacyv1.EBSRequest{Name: "vol-ctx"}))
 	caller.End()
 
-	producer := clientSpan(t, sr)
+	var producer sdktrace.ReadOnlySpan
+	for _, s := range sr.Ended() {
+		if s.SpanKind() == trace.SpanKindClient {
+			producer = s
+		}
+	}
+	require.NotNil(t, producer, "producer span must be recorded")
+
 	assert.Equal(t, caller.SpanContext().TraceID(), producer.SpanContext().TraceID(),
 		"the ebs span must join the caller's trace rather than root one of its own")
 	assert.Equal(t, caller.SpanContext().SpanID(), producer.Parent().SpanID(),
@@ -138,16 +167,25 @@ func TestEbsRequestWithTrace_JoinsTheCallersTrace(t *testing.T) {
 // reads the header to attribute its own consumer span.
 func TestEbsRequestWithTrace_CarriesTheAccount(t *testing.T) {
 	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	subject := "ebs." + newMounterTestNode() + ".delete"
-	headerCh := requestOnceOn(t, nc, subject)
+	daemon := createTestDaemon(t, sharedNATSURL)
+	headerCh := mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
 
-	_, err := ebsRequestWithTrace(t.Context(), nc, "000000000042", subject, []byte("{}"), 5*time.Second)
-	require.NoError(t, err)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	instance := &vm.VM{ID: "i-acct", AccountID: "000000000042"}
+	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-acct"}}
+	require.NoError(t, adapter.Mount(t.Context(), instance))
 
 	assert.Equal(t, "000000000042", (<-headerCh).Get(natsmsg.AccountIDHeader),
 		"viperblockd reads the account off the header, so it must be set")
-	assert.Equal(t, "000000000042", spanAccountOf(clientSpan(t, sr)),
+
+	var producer sdktrace.ReadOnlySpan
+	for _, s := range sr.Ended() {
+		if s.SpanKind() == trace.SpanKindClient {
+			producer = s
+		}
+	}
+	require.NotNil(t, producer)
+	assert.Equal(t, "000000000042", spanAccountOf(producer),
 		"the ebs span must name the account it acted for")
 }
 
@@ -155,39 +193,25 @@ func TestEbsRequestWithTrace_CarriesTheAccount(t *testing.T) {
 // would be worse than leaving it unattributed.
 func TestEbsRequestWithTrace_OmitsAnAbsentAccount(t *testing.T) {
 	sr := withRecordedSpans(t)
-	nc := connectMounterTest(t)
-	subject := "ebs." + newMounterTestNode() + ".delete"
-	headerCh := requestOnceOn(t, nc, subject)
+	daemon := createTestDaemon(t, sharedNATSURL)
+	headerCh := mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
 
-	_, err := ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
-	require.NoError(t, err)
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	require.NoError(t, adapter.MountOne(t.Context(), "", &viperblocklegacyv1.EBSRequest{Name: "vol-none"}))
 
 	assert.Empty(t, (<-headerCh).Get(natsmsg.AccountIDHeader),
 		"an unattributed request must not carry a blank account header")
-	assert.Empty(t, spanAccountOf(clientSpan(t, sr)), "an unattributed span must carry no account")
-}
-
-// clientSpan returns the one client span sr recorded.
-func clientSpan(t *testing.T, sr *tracetest.SpanRecorder) sdktrace.ReadOnlySpan {
-	t.Helper()
-	var spans []sdktrace.ReadOnlySpan
 	for _, s := range sr.Ended() {
 		if s.SpanKind() == trace.SpanKindClient {
-			spans = append(spans, s)
+			assert.Empty(t, spanAccountOf(s), "an unattributed span must carry no account")
 		}
 	}
-	require.Len(t, spans, 1)
-	return spans[0]
 }
 
 // spanAccountOf returns the account attribute of span, or "".
 func spanAccountOf(span sdktrace.ReadOnlySpan) string {
-	return spanAttribute(span, natsmsg.AttrAccountID)
-}
-
-func spanAttribute(span sdktrace.ReadOnlySpan, key string) string {
 	for _, attr := range span.Attributes() {
-		if string(attr.Key) == key {
+		if string(attr.Key) == natsmsg.AttrAccountID {
 			return attr.Value.AsString()
 		}
 	}

spinifex/runtime/roles/qemunbdd/provider.go

@@ -533,11 +533,6 @@ func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSna
 		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
 	}
 
-	// qemu-nbd holds a published volume's file locked and keeps writing it, so
-	// no copy of it is point-in-time. The volume must be unpublished first.
-	if pub, ok := p.published[req.VolumeID]; ok && !pub.dead() {
-		return nil, fmt.Errorf("%w: volume %s is published, so it cannot be snapshotted until it is unpublished", ebsprovider.ErrVolumeInUse, req.VolumeID)
-	}
 	if _, err := p.run.Run(ctx, "qemu-img", "convert", "-O", "qcow2", volPath, snapPath); err != nil {
 		return nil, fmt.Errorf("create snapshot %s: %w", req.SnapshotID, err)
 	}

spinifex/runtime/roles/qemunbdd/provider_test.go

@@ -414,30 +414,6 @@ func TestCreateSnapshot_Argv(t *testing.T) {
 	assert.Equal(t, []string{"info", "--output=json", "--force-share", snapPath}, calls[1].args)
 }
 
-// A published volume is refused before qemu-img runs, and an unpublished one
-// snapshots again from its file.
-func TestCreateSnapshot_PublishedVolumeIsRefused(t *testing.T) {
-	p, fr := newTestProvider(t)
-	ctx := context.Background()
-	volPath := p.volumePath("vol-snap-pub")
-	require.NoError(t, os.WriteFile(volPath, nil, 0o640))
-	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-snap-pub", NodeID: "node-1"})
-	require.NoError(t, err)
-	published := len(fr.callsSnapshot())
-
-	_, err = p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-pub", VolumeID: "vol-snap-pub"})
-	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
-	assert.Len(t, fr.callsSnapshot(), published, "a refused snapshot must not run qemu-img")
-	_, statErr := os.Stat(p.snapshotPath("snap-pub"))
-	require.ErrorIs(t, statErr, os.ErrNotExist)
-
-	_, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-snap-pub", NodeID: "node-1"})
-	require.NoError(t, err)
-	snap, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-pub", VolumeID: "vol-snap-pub"})
-	require.NoError(t, err)
-	assert.Equal(t, "vol-snap-pub", snap.SourceVolumeID)
-}
-
 func TestCreateSnapshot_NotFoundSourceVolume(t *testing.T) {
 	p, _ := newTestProvider(t)
 	_, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-orphan", VolumeID: "vol-never-existed"})

spinifex/runtime/roles/qemunbdd/snapshot_published_test.go

@@ -1,104 +0,0 @@
-package qemunbdd_test
-
-import (
-	"bufio"
-	"io"
-	"os/exec"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// attachedClient stands in for a guest's QEMU: a qemu-io holding a connection
-// to the export open, so the snapshot runs against a volume in use.
-type attachedClient struct {
-	cmd   *exec.Cmd
-	stdin io.WriteCloser
-	out   *bufio.Scanner
-}
-
-func attachClient(t *testing.T, uri string) *attachedClient {
-	t.Helper()
-	cmd := exec.Command("qemu-io", "-f", "raw", uri)
-	stdin, err := cmd.StdinPipe()
-	require.NoError(t, err)
-	stdout, err := cmd.StdoutPipe()
-	require.NoError(t, err)
-	cmd.Stderr = cmd.Stdout
-	require.NoError(t, cmd.Start())
-	c := &attachedClient{cmd: cmd, stdin: stdin, out: bufio.NewScanner(stdout)}
-	t.Cleanup(c.detach)
-	return c
-}
-
-// run sends one qemu-io command and returns the line reporting its result.
-func (c *attachedClient) run(t *testing.T, command, want string) {
-	t.Helper()
-	_, err := io.WriteString(c.stdin, command+"\n")
-	require.NoError(t, err)
-	done := make(chan string, 1)
-	go func() {
-		for c.out.Scan() {
-			if line := c.out.Text(); strings.Contains(line, want) || strings.Contains(line, "failed") {
-				done <- line
-				return
-			}
-		}
-		done <- "qemu-io exited"
-	}()
-	select {
-	case line := <-done:
-		require.Contains(t, line, want)
-	case <-time.After(10 * time.Second):
-		t.Fatalf("qemu-io did not answer %q", command)
-	}
-}
-
-func (c *attachedClient) detach() {
-	_, _ = io.WriteString(c.stdin, "quit\n")
-	_ = c.stdin.Close()
-	_ = c.cmd.Wait()
-}
-
-// A snapshot of a volume a guest is attached to is refused and writes nothing,
-// leaving the guest's connection intact. Once the volume is unpublished it
-// snapshots, holding what the guest wrote.
-func TestQEMUNBDSnapshotOfAttachedVolumeIsRefused(t *testing.T) {
-	requireQEMUTools(t)
-	provider := newQEMUProvider(t)
-	ctx := t.Context()
-	const volumeID, snapshotID = "vol-attached-snap", "snap-attached-snap"
-	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 64 << 20}})
-	require.NoError(t, err)
-	pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1"})
-	require.NoError(t, err)
-
-	guest := attachClient(t, pub.Export.NBD.NBDURI)
-	guest.run(t, "write -P 0xab 1M 64k", "wrote 65536/65536")
-
-	_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
-	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
-	listed, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
-	require.NoError(t, err)
-	for _, snap := range listed.Snapshots {
-		assert.NotEqual(t, snapshotID, snap.ID, "a refused snapshot must leave nothing behind")
-	}
-	guest.run(t, "read -P 0xab 1M 64k", "read 65536/65536")
-
-	guest.detach()
-	_, err = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1"})
-	require.NoError(t, err)
-	snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
-	require.NoError(t, err)
-	require.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
-
-	snapPath := strings.TrimPrefix(snap.Handle, "file://")
-	out, err := exec.Command("qemu-io", "-f", "qcow2", "-r", "-c", "read -P 0xab 1M 64k", snapPath).CombinedOutput()
-	require.NoError(t, err, "%s", out)
-	assert.Contains(t, string(out), "read 65536/65536")
-	assert.NotContains(t, string(out), "Pattern verification failed")
-}

tests/e2e/single/createimage_test.go

@@ -72,9 +72,10 @@ func runCreateImage(t *testing.T, fix *Fixture) {
 
 	runBootVerifyAMI(t, fix, customAMIID, "running-path")
 
-	// The offline path: CreateImage against the same instance stopped. Both
-	// states take the single snapshotVolume path, so this proves the provider
-	// snapshots a volume with no connected client as well as a live one.
+	// The offline path: CreateImage against the same instance stopped. The
+	// daemon's IsRunning decision is the instance's actual VM status at call
+	// time (daemon_handlers_image.go), not the NoReboot flag, so stopping the
+	// source instance is what selects snapshotStoppedVolume.
 	harness.Step(t, "stop-instances %s (precondition for stopped-path CreateImage)", instanceID)
 	_, err = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{
 		InstanceIds: []*string{aws.String(instanceID)},