fix(uefi): keep an unrecordable write-back owed and prune off the stop path

A stop whose write-back obligation the store record could not take now
marks the instance record with the owing node (uefi_write_back_owed),
carried by the stop's own record write, and keeps the obligation in
write-back.json beside seeded.json. A start on another node refuses while
the marked node is a cluster member. The reaper records a kept obligation
once the store record answers, lands it by the existing rules, and clears
the marker in the same compare-and-set.

Pruning no longer runs inside a stop: a landed write-back queues the prefix
and the reaper prunes it, keeping the pointer as read at prune time.

docs/DESIGN.md

@@ -111,7 +111,7 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
+2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` and a pointer on the instance record
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -1,126 +0,0 @@
-package instance
-
-import (
-	"bytes"
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// varsTemplateBytes stands in for a firmware VARS template. The real one is
-// 540,672 bytes, but nothing here depends on the size beyond it being exact.
-func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
-
-func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
-	t.Helper()
-	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
-}
-
-// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
-// decouple: the variable store is allocated through the provider with the
-// template as seed bytes, so no control-plane engine is built for it.
-func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
-
-	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
-		"the store must be sized byte-exactly to the VARS template")
-
-	seed, ok := provider.SeedData("vol-efiseed001-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
-}
-
-// A provider that rounds capacity to a GiB boundary produces a VARS region
-// pflash refuses, which would surface as an unbootable guest rather than a
-// failed launch. The mismatch has to fail the launch instead.
-func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
-	provider := &roundingProvider{
-		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
-	}
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
-// the guest wrote on its first boot, so the second call must be a no-op.
-func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	ctx := context.Background()
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-
-	seed, ok := provider.SeedData("vol-efirepeat1-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed)
-}
-
-// A seed above the wire bound must fail the launch with an error naming the
-// limit, not reach NATS and fail as an oversized publish.
-func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
-// against this host's real firmware, including the name the VM launches with.
-func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
-	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
-		t.Skipf("no UEFI firmware on this host: %v", err)
-	}
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	instance := &vm.VM{}
-
-	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
-
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-	require.Len(t, instance.EBSRequests.Requests, 1)
-	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
-	assert.True(t, instance.EBSRequests.Requests[0].EFI)
-	assert.False(t, instance.EBSRequests.Requests[0].Boot)
-
-	seed, ok := provider.SeedData("vol-efiprepare001-efi")
-	require.True(t, ok)
-	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
-}
-
-// roundingProvider reports a GiB-rounded capacity the way a provider with no
-// sub-GiB volume granularity would.
-type roundingProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
-	volume, err := p.EBSProvider.CreateVolume(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	volume.CapacityBytes = 1 << 30
-	return volume, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -9,7 +9,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -246,6 +245,8 @@ type InstanceServiceImpl struct {
 	ipAllocator       PublicIPAllocator
 	dnsBaseDomain     string
 	dnsInternalDomain string
+	uefiRecords       UEFIStoreRecords
+	uefiNode          string
 }
 
 // SetEBSProvider injects the provider boundary used for instance-created
@@ -1247,6 +1248,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 			slog.WarnContext(ctx, "StartInstance: instance not found in manager",
 				"instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+		case errors.Is(err, vm.ErrUEFIWriteBackPending):
+			// Retryable: the owing node lands its write-back once predastore answers.
+			slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+			return errors.New(awserrors.ErrorServerInternal)
 		default:
 			// A volume held elsewhere is the caller's 400, not a fault on this node.
 			if refusal := volumeHeldElsewhereError(err); refusal != nil {
@@ -1613,18 +1618,15 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
-	// guests must not allocate an orphan VARS volume).
-	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
+	// Step 2: Seed the UEFI variable store. A BIOS guest has none.
+	if vm.UsesUEFIVarStore(instance) {
 		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
-		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
-		if err != nil {
+		if err := s.seedUEFIVarStore(ctx, instance, aws.StringValue(input.ImageId), arch); err != nil {
 			return nil, err
 		}
 	}
 
-	// The EFI store is internal, so it is not reported; every volume the
-	// request named is.
+	// Every volume the request named is reported.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1886,81 +1888,6 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
-// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
-// firmware VARS template (pflash requires byte-exact size) and seeded from it.
-// arch is "x86_64" | "arm64".
-func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
-	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
-	if err != nil {
-		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	template, err := os.ReadFile(varsTemplate)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if int64(len(template)) != varsSize {
-		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
-
-	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
-	if s.ebsProvider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
-		return err
-	}
-
-	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
-		Name: efiVolumeName,
-		Boot: false,
-		EFI:  true,
-	})
-	instance.EBSRequests.Mu.Unlock()
-
-	return nil
-}
-
-// createEFIVolumeViaProvider allocates the EFI variable store through the
-// provider, seeding it with this node's firmware VARS template. The bytes
-// travel with the request so the region matches the firmware that will run.
-func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
-	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
-
-	// The store is internal to the launch, never attachable and filtered out of
-	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
-	// does. Its lifecycle is the root volume's DeleteVolume.
-	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
-		Versioned:        ebsprovider.NewVersioned(),
-		VolumeID:         efiVolumeName,
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
-		AvailabilityZone: s.config.AZ,
-		SeedData:         template,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if created == nil {
-		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// pflash rejects a VARS region that is not byte-exact, so a provider that
-	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
-	if created.CapacityBytes != int64(len(template)) {
-		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
-			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
-}
-
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2772,7 +2699,7 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
 	if err := vm.CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
+		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without a variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
 		return errors.New(awserrors.ErrorServerInternal)

spinifex/domains/ec2/instance/uefi_seed.go

@@ -0,0 +1,98 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// UEFIStoreRecords creates the store record that owns a new instance's
+// variable store objects.
+type UEFIStoreRecords interface {
+	Create(ctx context.Context, rec *uefistore.UEFIStoreRecord) (uint64, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// SetUEFIVarStore wires where RunInstances records and writes a new UEFI
+// instance's variable store, and the node named as its writer.
+func (s *InstanceServiceImpl) SetUEFIVarStore(records UEFIStoreRecords, node string) {
+	s.uefiRecords = records
+	s.uefiNode = node
+}
+
+// seedUEFIVarStore gives a new UEFI instance its variable store: the store
+// record, live, then the seed object at a fresh key, then the pointer on the
+// in-memory instance, which the launch's record writes carry.
+func (s *InstanceServiceImpl) seedUEFIVarStore(ctx context.Context, instance *vm.VM, imageID, arch string) error {
+	bucket := ""
+	if s.config != nil {
+		bucket = s.config.Predastore.Bucket
+	}
+	if s.uefiRecords == nil || s.objectStore == nil || bucket == "" || s.uefiNode == "" {
+		slog.ErrorContext(ctx, "UEFI variable store seeding is not wired on this node", "instanceId", instance.ID)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	fw, err := vm.NodeFirmwareFor(arch)
+	if err != nil {
+		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "instanceId", instance.ID, "arch", arch, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	region, source, err := s.uefiSeedRegion(ctx, imageID, fw)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read the UEFI variable store seed", "instanceId", instance.ID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	key, err := uefistore.NewObjectKey(instance.ID)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to name the UEFI variable store object", "instanceId", instance.ID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	// The record first, so terminate can reach every object under the prefix.
+	if _, err := s.uefiRecords.Create(ctx, &uefistore.UEFIStoreRecord{
+		InstanceID: instance.ID,
+		AccountID:  instance.AccountID,
+		State:      uefistore.StateLive,
+	}); err != nil {
+		slog.ErrorContext(ctx, "Failed to create the UEFI store record", "instanceId", instance.ID, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	start := time.Now()
+	if err := uefistore.PutObject(ctx, s.objectStore, bucket, key, region, uefistore.ObjectMeta{
+		OwnerID:    instance.ID,
+		WriterNode: s.uefiNode,
+		Firmware:   fw.Identity,
+		CodePath:   fw.Code,
+		Source:     source,
+	}); err != nil {
+		slog.ErrorContext(ctx, "Failed to write the UEFI variable store seed", "instanceId", instance.ID, "key", key,
+			"put_ms", otelsetup.Millis(time.Since(start)), "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	ref := &vm.UEFIVarStoreRef{Key: key, Firmware: fw.Identity, WriterNode: s.uefiNode, WrittenAt: time.Now().UTC()}
+	s.vmMgr.Inspect(instance, func(v *vm.VM) { v.UEFIVarStore = ref })
+	slog.InfoContext(ctx, "Seeded the UEFI variable store", "instanceId", instance.ID, "key", key, "source", source,
+		"arch", fw.Identity.Arch, "varsSize", fw.Identity.VarsSize, "put_ms", otelsetup.Millis(time.Since(start)))
+	return nil
+}
+
+// uefiSeedRegion returns the region a new instance's store starts from and
+// the source its object records. AMI documents name no variable store yet,
+// so every launch seeds from this node's VARS template.
+func (s *InstanceServiceImpl) uefiSeedRegion(_ context.Context, _ string, fw vm.NodeFirmware) ([]byte, string, error) {
+	region, err := os.ReadFile(fw.VarsTemplate)
+	if err != nil {
+		return nil, "", err
+	}
+	return region, uefistore.SourceTemplate, nil
+}

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -0,0 +1,273 @@
+package instance
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testSeedInstance = "i-0a1b2c3d4e5f60718"
+	testSeedNode     = "node-a"
+	testSeedType     = "t3.nano"
+)
+
+// recordingObjects keeps each put's metadata, which the memory store drops,
+// and can fail store puts or check the store record exists when one lands.
+type recordingObjects struct {
+	*objectstore.MemoryObjectStore
+
+	mu       sync.Mutex
+	metadata map[string]map[string]string
+	putErr   error
+	onPut    func(key string)
+}
+
+var _ objectstore.ObjectStore = (*recordingObjects)(nil)
+
+func (r *recordingObjects) PutObject(ctx context.Context, in *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if r.onPut != nil {
+		r.onPut(aws.StringValue(in.Key))
+	}
+	if r.putErr != nil && strings.HasPrefix(aws.StringValue(in.Key), "uefi/") {
+		return nil, r.putErr
+	}
+	r.mu.Lock()
+	r.metadata[aws.StringValue(in.Key)] = aws.StringValueMap(in.Metadata)
+	r.mu.Unlock()
+	return r.MemoryObjectStore.PutObject(ctx, in)
+}
+
+type seedFixture struct {
+	svc      *InstanceServiceImpl
+	provider *ebsprovider.MemoryProvider
+	objects  *recordingObjects
+	records  *uefistore.Store
+	template []byte
+	codePath string
+	identity uefistore.FirmwareIdentity
+}
+
+func newSeedFixture(t *testing.T) *seedFixture {
+	t.Helper()
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	codeBytes := []byte("ovmf code build 7")
+	template := bytes.Repeat([]byte("VARS"), 1024)
+	require.NoError(t, os.WriteFile(code, codeBytes, 0o600))
+	require.NoError(t, os.WriteFile(vars, template, 0o600))
+	orig := vm.FirmwarePathCandidates
+	vm.FirmwarePathCandidates = map[string][]vm.FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}} //nolint:reassign // exported for tests to swap, restored below
+
+	t.Cleanup(func() { vm.FirmwarePathCandidates = orig }) //nolint:reassign // restoring the candidates captured above
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeSeeding: true, VolumeEnumeration: true})
+	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
+	objects := &recordingObjects{MemoryObjectStore: objectstore.NewMemoryObjectStore(), metadata: map[string]map[string]string{}}
+	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
+	svc.vmMgr = vm.NewManager()
+	svc.instanceTypes = map[string]*ec2.InstanceTypeInfo{testSeedType: {
+		InstanceType:  aws.String(testSeedType),
+		ProcessorInfo: &ec2.ProcessorInfo{SupportedArchitectures: []*string{aws.String("x86_64")}},
+	}}
+	svc.SetUEFIVarStore(records, testSeedNode)
+
+	sum := sha256.Sum256(codeBytes)
+	return &seedFixture{
+		svc: svc, provider: provider, objects: objects, records: records, template: template, codePath: code,
+		identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(template)), CodeSHA256: hex.EncodeToString(sum[:])},
+	}
+}
+
+func (f *seedFixture) instance(bootMode string) *vm.VM {
+	v := &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: testSeedType, BootMode: bootMode, Instance: &ec2.Instance{}}
+	f.svc.vmMgr.Insert(v)
+	return v
+}
+
+func seedRunInput() *ec2.RunInstancesInput {
+	return &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String(testSeedType)}
+}
+
+func (f *seedFixture) objectKeys(t *testing.T) []string {
+	t.Helper()
+	out, err := f.objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(testRootBucket), Prefix: aws.String(uefistore.KeyPrefix(testSeedInstance)),
+	})
+	require.NoError(t, err)
+	keys := make([]string, 0, len(out.Contents))
+	for _, o := range out.Contents {
+		keys = append(keys, aws.StringValue(o.Key))
+	}
+	return keys
+}
+
+func (f *seedFixture) providerVolumeIDs(t *testing.T) []string {
+	t.Helper()
+	out, err := f.provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	ids := make([]string, 0, len(out.Volumes))
+	for _, v := range out.Volumes {
+		ids = append(ids, v.ID)
+	}
+	return ids
+}
+
+// A UEFI launch seeds its store as an object and a pointer: no <root>-efi
+// volume, no EFI request, a live store record, and a gzip of this node's
+// template whose metadata and pointer name the firmware and the writer.
+func TestGenerateVolumes_UEFISeedsTheStoreFromTheTemplate(t *testing.T) {
+	f := newSeedFixture(t)
+	instance := f.instance("uefi")
+
+	infos, err := f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+	require.NoError(t, err)
+
+	require.Len(t, infos, 1, "only the root volume is reported")
+	for _, id := range f.providerVolumeIDs(t) {
+		assert.False(t, strings.HasSuffix(id, "-efi"), "no EFI volume is created, found %s", id)
+	}
+	for _, r := range instance.EBSRequests.Requests {
+		assert.False(t, r.EFI, "no EFI request is appended, found %s", r.Name)
+	}
+
+	ref := instance.UEFIVarStore
+	require.NotNil(t, ref, "the launch's record carries a pointer")
+	assert.Equal(t, []string{ref.Key}, f.objectKeys(t), "the pointer names the one object under the prefix")
+	assert.Equal(t, f.identity, ref.Firmware)
+	assert.Equal(t, testSeedNode, ref.WriterNode)
+	assert.False(t, ref.WrittenAt.IsZero())
+	assert.Equal(t, ref, instance.Record().Status.UEFIVarStore, "the pointer is in the record's status")
+
+	out, err := f.objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testRootBucket), Key: aws.String(ref.Key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	gz, err := gzip.NewReader(out.Body)
+	require.NoError(t, err, "the body is gzip")
+	region, err := io.ReadAll(gz)
+	require.NoError(t, err)
+	assert.Equal(t, f.template, region, "the store starts as this node's VARS template")
+
+	assert.Equal(t, map[string]string{
+		"instance-id":          testSeedInstance,
+		"writer-node":          testSeedNode,
+		"firmware-arch":        "x86_64",
+		"firmware-vars-size":   strconv.Itoa(len(f.template)),
+		"firmware-code-sha256": f.identity.CodeSHA256,
+		"firmware-code-path":   f.codePath,
+		"source":               "template",
+	}, f.objects.metadata[ref.Key])
+
+	rec, _, err := f.records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+	assert.Equal(t, testRootAccount, rec.AccountID)
+	assert.Nil(t, rec.WriteBack)
+}
+
+// The store record exists before the object is put, so terminate can reach
+// every object under the prefix, including one whose put failed midway.
+func TestGenerateVolumes_UEFIRecordsTheStoreBeforeItsObject(t *testing.T) {
+	f := newSeedFixture(t)
+	instance := f.instance("uefi")
+	recordAtPut := errors.New("no put")
+	f.objects.onPut = func(key string) {
+		if strings.HasPrefix(key, uefistore.KeyPrefix(testSeedInstance)) {
+			_, _, recordAtPut = f.records.Get(context.Background(), testSeedInstance)
+		}
+	}
+
+	_, err := f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+
+	require.NoError(t, err)
+	assert.NoError(t, recordAtPut, "the record is live when the object is put")
+}
+
+// A failed put fails the launch with InternalError and sets no pointer, and
+// leaves the record so terminate can still settle the prefix.
+func TestGenerateVolumes_UEFIPutFailureFailsTheLaunch(t *testing.T) {
+	f := newSeedFixture(t)
+	instance := f.instance("uefi")
+	f.objects.putErr = errors.New("predastore unreachable")
+
+	_, err := f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Nil(t, instance.UEFIVarStore)
+	rec, _, getErr := f.records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, getErr)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+}
+
+// A record already under the instance's ID is not this launch's, so the
+// launch writes nothing under its prefix.
+func TestGenerateVolumes_UEFIExistingRecordFailsTheLaunch(t *testing.T) {
+	f := newSeedFixture(t)
+	instance := f.instance("uefi")
+	_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: testSeedInstance, AccountID: "000000000099", State: uefistore.StateLive})
+	require.NoError(t, err)
+
+	_, err = f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Empty(t, f.objectKeys(t))
+	assert.Nil(t, instance.UEFIVarStore)
+}
+
+// A BIOS guest has no variable store: no record, no object, no pointer.
+func TestGenerateVolumes_BIOSSeedsNoStore(t *testing.T) {
+	f := newSeedFixture(t)
+	instance := f.instance("legacy-bios")
+
+	_, err := f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+
+	require.NoError(t, err)
+	assert.Nil(t, instance.UEFIVarStore)
+	assert.Empty(t, f.objectKeys(t))
+	_, _, err = f.records.Get(t.Context(), testSeedInstance)
+	assert.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A node without the store wired cannot seed, so it fails the launch rather
+// than booting a UEFI guest with no store.
+func TestGenerateVolumes_UEFIWithoutStoreWiringFails(t *testing.T) {
+	f := newSeedFixture(t)
+	f.svc.SetUEFIVarStore(nil, "")
+	instance := f.instance("uefi")
+
+	_, err := f.svc.GenerateVolumes(t.Context(), seedRunInput(), instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Nil(t, instance.UEFIVarStore)
+}

spinifex/domains/ec2/instance/uefistore/object.go

@@ -0,0 +1,128 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// SourceTemplate is a store seeded from the launching node's VARS template.
+	SourceTemplate = "template"
+	// SourceWriteBack is a store written back from a node's live copy.
+	SourceWriteBack = "write-back"
+
+	sourceImagePrefix = "image:"
+
+	// PutTimeout bounds one put of a store object. The object is a fixed-size
+	// region, so a put that takes longer is a wedged object store.
+	PutTimeout = 10 * time.Second
+)
+
+// ErrInvalidObject is a store object whose key, region or metadata cannot be
+// written as the owner's store.
+var ErrInvalidObject = errors.New("uefistore: invalid store object")
+
+// SourceImage is a store seeded from imageID's own store.
+func SourceImage(imageID string) string {
+	return sourceImagePrefix + imageID
+}
+
+// ObjectMeta is what a store object records about itself, as x-amz-meta-
+// entries, so its owner, writer and firmware are known from the object alone.
+type ObjectMeta struct {
+	OwnerID    string
+	WriterNode string
+	Firmware   FirmwareIdentity
+	CodePath   string
+	Source     string
+}
+
+// Metadata returns m as object metadata, keyed by the owner's kind.
+func (m ObjectMeta) Metadata() (map[string]*string, error) {
+	var ownerKey string
+	switch {
+	case strings.HasPrefix(m.OwnerID, "i-"):
+		ownerKey = "instance-id"
+	case strings.HasPrefix(m.OwnerID, "ami-"):
+		ownerKey = "image-id"
+	default:
+		return nil, fmt.Errorf("%w: owner %q is neither an instance nor an image", ErrInvalidObject, m.OwnerID)
+	}
+	switch {
+	case m.WriterNode == "":
+		return nil, fmt.Errorf("%w: %s names no writer node", ErrInvalidObject, m.OwnerID)
+	case m.Source == "":
+		return nil, fmt.Errorf("%w: %s names no source", ErrInvalidObject, m.OwnerID)
+	case m.Firmware.Arch == "" || m.Firmware.VarsSize <= 0 || m.Firmware.CodeSHA256 == "":
+		return nil, fmt.Errorf("%w: %s has an incomplete firmware identity", ErrInvalidObject, m.OwnerID)
+	}
+	return map[string]*string{
+		ownerKey:               aws.String(m.OwnerID),
+		"writer-node":          aws.String(m.WriterNode),
+		"firmware-arch":        aws.String(m.Firmware.Arch),
+		"firmware-vars-size":   aws.String(strconv.FormatInt(m.Firmware.VarsSize, 10)),
+		"firmware-code-sha256": aws.String(m.Firmware.CodeSHA256),
+		"firmware-code-path":   aws.String(m.CodePath),
+		"source":               aws.String(m.Source),
+	}, nil
+}
+
+// NewObjectKey returns a fresh key under ownerID's prefix, named by a random
+// version 4 UUID, so no write ever replaces an object a record points at.
+func NewObjectKey(ownerID string) (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate UEFI store key for %s: %w", ownerID, err)
+	}
+	b[6] = b[6]&0x0f | 0x40
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%s%x-%x-%x-%x-%x", KeyPrefix(ownerID), b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
+}
+
+// PutObject writes region gzip-compressed to key with meta, within
+// PutTimeout. The region must be exactly the firmware's VARS size, because
+// pflash refuses any other.
+func PutObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, region []byte, meta ObjectMeta) error {
+	if !ownsKey(meta.OwnerID, key) {
+		return fmt.Errorf("%w: key %q is not under %s", ErrInvalidObject, key, KeyPrefix(meta.OwnerID))
+	}
+	if int64(len(region)) != meta.Firmware.VarsSize {
+		return fmt.Errorf("%w: %s is a %d-byte region under firmware with a %d-byte VARS region",
+			ErrInvalidObject, key, len(region), meta.Firmware.VarsSize)
+	}
+	md, err := meta.Metadata()
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	zw := gzip.NewWriter(&body)
+	if _, err := zw.Write(region); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+	if err := zw.Close(); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+
+	putCtx, cancel := context.WithTimeout(ctx, PutTimeout)
+	defer cancel()
+	if _, err := objects.PutObject(putCtx, &s3.PutObjectInput{
+		Bucket:   aws.String(bucket),
+		Key:      aws.String(key),
+		Body:     bytes.NewReader(body.Bytes()),
+		Metadata: md,
+	}); err != nil {
+		return fmt.Errorf("put UEFI store %s: %w", key, err)
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/object_test.go

@@ -0,0 +1,164 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testBucket = "instances"
+
+func testMeta(owner string) uefistore.ObjectMeta {
+	return uefistore.ObjectMeta{
+		OwnerID:    owner,
+		WriterNode: "node-1",
+		Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 1 << 16, CodeSHA256: "abc123"},
+		CodePath:   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		Source:     uefistore.SourceTemplate,
+	}
+}
+
+var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+// Every key is a fresh version 4 UUID directly under the owner's prefix, so
+// a write never lands on an object a record already points at.
+func TestNewObjectKey_IsAFreshUUIDUnderTheOwnersPrefix(t *testing.T) {
+	seen := map[string]bool{}
+	for range 64 {
+		key, err := uefistore.NewObjectKey(instanceID)
+		require.NoError(t, err)
+		name, ok := bytes.CutPrefix([]byte(key), []byte(uefistore.KeyPrefix(instanceID)))
+		require.True(t, ok, key)
+		assert.Regexp(t, uuidV4, string(name))
+		assert.False(t, seen[key], "key %s repeated", key)
+		seen[key] = true
+	}
+}
+
+func TestObjectMeta_Metadata_NamesTheOwnerByKind(t *testing.T) {
+	md, err := testMeta(instanceID).Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"instance-id":          instanceID,
+		"writer-node":          "node-1",
+		"firmware-arch":        "arm64",
+		"firmware-vars-size":   "65536",
+		"firmware-code-sha256": "abc123",
+		"firmware-code-path":   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		"source":               "template",
+	}, aws.StringValueMap(md))
+
+	img := testMeta("ami-0123456789abcdef0")
+	img.Source = uefistore.SourceImage("ami-0fedcba9876543210")
+	md, err = img.Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, "ami-0123456789abcdef0", aws.StringValue(md["image-id"]))
+	assert.NotContains(t, md, "instance-id")
+	assert.Equal(t, "image:ami-0fedcba9876543210", aws.StringValue(md["source"]))
+}
+
+func TestObjectMeta_Metadata_RefusesAnIncompleteObject(t *testing.T) {
+	tests := map[string]func(*uefistore.ObjectMeta){
+		"unknown owner":  func(m *uefistore.ObjectMeta) { m.OwnerID = "vol-1" },
+		"no writer":      func(m *uefistore.ObjectMeta) { m.WriterNode = "" },
+		"no source":      func(m *uefistore.ObjectMeta) { m.Source = "" },
+		"no arch":        func(m *uefistore.ObjectMeta) { m.Firmware.Arch = "" },
+		"no vars size":   func(m *uefistore.ObjectMeta) { m.Firmware.VarsSize = 0 },
+		"no code sha256": func(m *uefistore.ObjectMeta) { m.Firmware.CodeSHA256 = "" },
+	}
+	for name, mutate := range tests {
+		t.Run(name, func(t *testing.T) {
+			m := testMeta(instanceID)
+			mutate(&m)
+			_, err := m.Metadata()
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+		})
+	}
+}
+
+// The 64 MiB arm64 region is mostly erased flash, so it is stored compressed
+// and decompresses to the exact region.
+func TestPutObject_StoresTheRegionCompressed(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := bytes.Repeat([]byte{0xff}, int(meta.Firmware.VarsSize))
+	copy(region, "BootOrder")
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, key, region, meta))
+
+	out, err := objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	body, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	assert.Less(t, len(body), len(region)/10, "the erased region compresses")
+	gz, err := gzip.NewReader(bytes.NewReader(body))
+	require.NoError(t, err)
+	got, err := io.ReadAll(gz)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+}
+
+// pflash needs a byte-exact region, and a key outside the owner's prefix is
+// one no store record owns; either is refused before anything is put.
+func TestPutObject_RefusesWhatTheOwnerCannotBoot(t *testing.T) {
+	meta := testMeta(instanceID)
+	good, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	other, err := uefistore.NewObjectKey("i-0fedcba9876543210")
+	require.NoError(t, err)
+	tests := []struct {
+		name   string
+		key    string
+		region []byte
+	}{
+		{name: "short region", key: good, region: make([]byte, meta.Firmware.VarsSize-1)},
+		{name: "long region", key: good, region: make([]byte, meta.Firmware.VarsSize+1)},
+		{name: "another owner's key", key: other, region: make([]byte, meta.Firmware.VarsSize)},
+		{name: "nested key", key: uefistore.KeyPrefix(instanceID) + "a/b", region: make([]byte, meta.Firmware.VarsSize)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			objects := objectstore.NewMemoryObjectStore()
+			err := uefistore.PutObject(t.Context(), objects, testBucket, tt.key, tt.region, meta)
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+			out, listErr := objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String(testBucket)})
+			require.NoError(t, listErr)
+			assert.Empty(t, out.Contents)
+		})
+	}
+}
+
+type failingPut struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (failingPut) PutObject(ctx context.Context, _ *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if _, ok := ctx.Deadline(); !ok {
+		return nil, errors.New("put had no deadline")
+	}
+	return nil, errors.New("predastore unreachable")
+}
+
+// A put runs under a deadline, so a wedged object store fails the write
+// rather than holding the caller, and its error reaches the caller.
+func TestPutObject_IsBoundedAndReportsAFailedPut(t *testing.T) {
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	meta := testMeta(instanceID)
+	err = uefistore.PutObject(t.Context(), failingPut{objectstore.NewMemoryObjectStore()}, testBucket, key, make([]byte, meta.Firmware.VarsSize), meta)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "predastore unreachable")
+}

spinifex/domains/ec2/instance/uefistore/prune.go

@@ -0,0 +1,69 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// RetainedVersions is how many of an owner's newest objects a prune keeps,
+	// besides every object it is told to keep.
+	RetainedVersions = 3
+
+	// PruneTimeout bounds one prune: a list of a prefix that holds at most a
+	// handful of objects, and the deletes of the few past retention.
+	PruneTimeout = 10 * time.Second
+)
+
+// Prune deletes ownerID's objects other than the RetainedVersions newest and
+// the keys in keep, returning the keys it deleted. A failed list deletes
+// nothing; a failed delete is reported and the rest still run.
+func Prune(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID string, keep ...string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, PruneTimeout)
+	defer cancel()
+
+	listed, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(KeyPrefix(ownerID)),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list UEFI stores of %s: %w", ownerID, err)
+	}
+	var owned []*s3.Object
+	for _, obj := range listed {
+		if obj != nil && ownsKey(ownerID, aws.StringValue(obj.Key)) {
+			owned = append(owned, obj)
+		}
+	}
+	// Newest first, with the key breaking a tie so every prune of one listing
+	// agrees on which objects are the newest.
+	slices.SortFunc(owned, func(a, b *s3.Object) int {
+		if c := aws.TimeValue(b.LastModified).Compare(aws.TimeValue(a.LastModified)); c != 0 {
+			return c
+		}
+		return strings.Compare(aws.StringValue(b.Key), aws.StringValue(a.Key))
+	})
+
+	var deleted []string
+	var errs []error
+	for i, obj := range owned {
+		key := aws.StringValue(obj.Key)
+		if i < RetainedVersions || slices.Contains(keep, key) {
+			continue
+		}
+		if _, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}); err != nil && !objectstore.IsNoSuchKeyError(err) {
+			errs = append(errs, fmt.Errorf("delete UEFI store %s: %w", key, err))
+			continue
+		}
+		deleted = append(deleted, key)
+	}
+	return deleted, errors.Join(errs...)
+}

spinifex/domains/ec2/instance/uefistore/prune_test.go

@@ -0,0 +1,177 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// clockedStore lists each object at the time the test wrote it, so which
+// objects are the newest does not depend on the wall clock's resolution.
+type clockedStore struct {
+	*objectstore.MemoryObjectStore
+
+	written   map[string]time.Time
+	listErr   error
+	deleteErr map[string]error
+	deleted   []string
+}
+
+func newClockedStore() *clockedStore {
+	return &clockedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), written: map[string]time.Time{}, deleteErr: map[string]error{}}
+}
+
+func (c *clockedStore) putAt(t *testing.T, key string, at time.Time) {
+	t.Helper()
+	_, err := c.PutObject(t.Context(), &s3.PutObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key), Body: bytes.NewReader([]byte("x"))})
+	require.NoError(t, err)
+	c.written[key] = at
+}
+
+func (c *clockedStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	if c.listErr != nil {
+		return nil, c.listErr
+	}
+	out, err := c.MemoryObjectStore.ListObjectsV2(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+	for _, o := range out.Contents {
+		o.LastModified = aws.Time(c.written[aws.StringValue(o.Key)])
+	}
+	return out, nil
+}
+
+func (c *clockedStore) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if err := c.deleteErr[aws.StringValue(in.Key)]; err != nil {
+		return nil, err
+	}
+	c.deleted = append(c.deleted, aws.StringValue(in.Key))
+	return c.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+func (c *clockedStore) keys(t *testing.T, owner string) []string {
+	t.Helper()
+	listed, _, err := objectstore.ListAll(t.Context(), c.MemoryObjectStore, &s3.ListObjectsV2Input{Bucket: aws.String(testBucket), Prefix: aws.String(uefistore.KeyPrefix(owner))})
+	require.NoError(t, err)
+	var keys []string
+	for _, o := range listed {
+		keys = append(keys, aws.StringValue(o.Key))
+	}
+	return keys
+}
+
+var t0 = time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)
+
+func version(n int) string { return uefistore.KeyPrefix(instanceID) + fmt.Sprintf("v%02d", n) }
+
+// The three newest versions and the pointed one are kept; everything else
+// under the owner's prefix goes.
+func TestPrune_KeepsTheThreeNewestAndThePointedVersion(t *testing.T) {
+	store := newClockedStore()
+	for n := 1; n <= 6; n++ {
+		store.putAt(t, version(n), t0.Add(time.Duration(n)*time.Minute))
+	}
+
+	deleted, err := uefistore.Prune(t.Context(), store, testBucket, instanceID, version(1))
+
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{version(2), version(3)}, deleted)
+	assert.ElementsMatch(t, []string{version(1), version(4), version(5), version(6)}, store.keys(t, instanceID))
+}
+
+// pruneS7Clause is quoted by every invariant failure below: prune never
+// deletes the pointed object or a retained version.
+const pruneS7Clause = "ADR-0003:S7: a reaper's judgement is never the decision that deletes customer data; " +
+	"the pointed and the retained versions survive every prune"
+
+// Pruning deletes customer data on its own, so whatever the listing order,
+// times and keep set, it never removes a kept key, one of the three newest,
+// or anything outside the owner's own prefix.
+func TestS7_UEFIPruneNeverDeletesThePointedObjectOrARetainedVersion(t *testing.T) {
+	const other = "i-0fedcba9876543210"
+	rng := rand.New(rand.NewPCG(1, 2))
+	for trial := range 200 {
+		store := newClockedStore()
+		n := 1 + rng.IntN(8)
+		times := map[string]time.Time{}
+		var keys []string
+		for i := range n {
+			key := version(i)
+			at := t0.Add(time.Duration(rng.IntN(5)) * time.Second)
+			store.putAt(t, key, at)
+			times[key] = at
+			keys = append(keys, key)
+		}
+		store.putAt(t, uefistore.KeyPrefix(other)+"newest", t0.Add(time.Hour))
+		store.putAt(t, uefistore.KeyPrefix(instanceID)+"nested/obj", t0)
+		keep := []string{keys[rng.IntN(n)]}
+		if rng.IntN(2) == 0 {
+			keep = append(keep, keys[rng.IntN(n)])
+		}
+
+		deleted, err := uefistore.Prune(t.Context(), store, testBucket, instanceID, keep...)
+		require.NoError(t, err)
+
+		for _, key := range deleted {
+			require.NotContains(t, keep, key, "%s: trial %d: a kept key was deleted", pruneS7Clause, trial)
+			require.Contains(t, keys, key, "%s: trial %d: deleted %s outside the owner's own objects", pruneS7Clause, trial, key)
+			survivors := 0
+			for _, k := range keys {
+				if !slices.Contains(deleted, k) && !times[k].Before(times[key]) {
+					survivors++
+				}
+			}
+			require.GreaterOrEqual(t, survivors, uefistore.RetainedVersions,
+				"%s: trial %d: deleted %s while only %d versions at least as new survive", pruneS7Clause, trial, key, survivors)
+		}
+		remaining := store.keys(t, instanceID)
+		require.Contains(t, remaining, uefistore.KeyPrefix(instanceID)+"nested/obj", pruneS7Clause)
+		require.LessOrEqual(t, len(remaining)-1, uefistore.RetainedVersions+len(keep), "trial %d", trial)
+		require.Len(t, store.keys(t, other), 1, pruneS7Clause)
+	}
+}
+
+// A listing that fails decides nothing, so nothing is deleted on a guess.
+func TestPrune_FailedListDeletesNothing(t *testing.T) {
+	store := newClockedStore()
+	for n := 1; n <= 5; n++ {
+		store.putAt(t, version(n), t0.Add(time.Duration(n)*time.Minute))
+	}
+	store.listErr = errors.New("predastore unavailable")
+
+	deleted, err := uefistore.Prune(t.Context(), store, testBucket, instanceID)
+
+	require.Error(t, err)
+	assert.Empty(t, deleted)
+	assert.Empty(t, store.deleted)
+	assert.Len(t, store.keys(t, instanceID), 5)
+}
+
+// One failed delete is reported and does not stop the others; an object
+// already gone counts as deleted.
+func TestPrune_FailedDeleteIsReportedAndTheRestStillRun(t *testing.T) {
+	store := newClockedStore()
+	for n := 1; n <= 6; n++ {
+		store.putAt(t, version(n), t0.Add(time.Duration(n)*time.Minute))
+	}
+	store.deleteErr[version(1)] = errors.New("500")
+	store.deleteErr[version(2)] = &objectstore.NoSuchKeyError{Key: version(2)}
+
+	deleted, err := uefistore.Prune(t.Context(), store, testBucket, instanceID)
+
+	require.ErrorContains(t, err, version(1))
+	assert.ElementsMatch(t, []string{version(2), version(3)}, deleted)
+}

spinifex/domains/ec2/instance/uefistore/writeback.go

@@ -0,0 +1,109 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// ErrNotLive is a store record that is missing or deleting: its instance is
+// gone or going, so no write-back is owed and nothing may be written under it.
+var ErrNotLive = errors.New("uefistore: store record is not live")
+
+// writeBackAttempts bounds the read-and-CAS loop that records or clears an
+// obligation, so a record under constant contention fails rather than spins.
+const writeBackAttempts = 3
+
+// LiveWriteBack returns the write-back owed on instanceID's live record, or
+// nil when none is. A missing or deleting record is ErrNotLive.
+func (s *Store) LiveWriteBack(ctx context.Context, instanceID string) (*UEFIWriteBack, error) {
+	rec, _, err := s.liveRecord(ctx, instanceID)
+	if err != nil {
+		return nil, err
+	}
+	return rec.WriteBack, nil
+}
+
+// RecordWriteBack sets wb as the write-back owed on instanceID's live record,
+// replacing any other: an instance has at most one open.
+func (s *Store) RecordWriteBack(ctx context.Context, instanceID string, wb UEFIWriteBack) error {
+	return s.mutateLive(ctx, instanceID, func(rec *UEFIStoreRecord) bool {
+		rec.WriteBack = &wb
+		return true
+	})
+}
+
+// ClearWriteBack drops the write-back owed on instanceID's live record. With
+// a key it drops only an obligation recorded for that key, so a retry does
+// not clear one recorded after it read the record.
+func (s *Store) ClearWriteBack(ctx context.Context, instanceID, key string) error {
+	return s.mutateLive(ctx, instanceID, func(rec *UEFIStoreRecord) bool {
+		if rec.WriteBack == nil || (key != "" && rec.WriteBack.Key != key) {
+			return false
+		}
+		rec.WriteBack = nil
+		return true
+	})
+}
+
+// WriteBacksOwedBy returns the instances whose live record owes a write-back
+// from node. A record that breaks its invariants is logged and passed over.
+func (s *Store) WriteBacksOwedBy(ctx context.Context, node string) ([]string, error) {
+	// Records are keyed by instance ID; the prefix passes over the bucket's
+	// schema stamp, which shares the key space.
+	recs, err := s.kv.List(ctx, "i-")
+	if err != nil {
+		return nil, err
+	}
+	var ids []string
+	for i := range recs {
+		rec := &recs[i]
+		if err := rec.validate(rec.InstanceID); err != nil {
+			slog.WarnContext(ctx, "Passing over a UEFI store record that cannot be read", "instanceId", rec.InstanceID, "err", err)
+			continue
+		}
+		if rec.State == StateLive && rec.WriteBack != nil && rec.WriteBack.Node == node {
+			ids = append(ids, rec.InstanceID)
+		}
+	}
+	return ids, nil
+}
+
+// liveRecord reads instanceID's record, reporting a missing or deleting one
+// as ErrNotLive.
+func (s *Store) liveRecord(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.Get(ctx, instanceID)
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil, 0, fmt.Errorf("%w: %s has no store record", ErrNotLive, instanceID)
+	}
+	if err != nil {
+		return nil, 0, err
+	}
+	if rec.State != StateLive {
+		return nil, 0, fmt.Errorf("%w: %s is %s", ErrNotLive, instanceID, rec.State)
+	}
+	return rec, rev, nil
+}
+
+// mutateLive applies change to instanceID's live record under CAS, re-reading
+// on a lost race. change reports whether it changed anything to write.
+func (s *Store) mutateLive(ctx context.Context, instanceID string, change func(*UEFIStoreRecord) bool) error {
+	for range writeBackAttempts {
+		rec, rev, err := s.liveRecord(ctx, instanceID)
+		if err != nil {
+			return err
+		}
+		if !change(rec) {
+			return nil
+		}
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if errors.Is(err, kvstore.ErrConflict) {
+			continue
+		}
+		return err
+	}
+	return fmt.Errorf("%w: %s changed under %d attempts", kvstore.ErrConflict, instanceID, writeBackAttempts)
+}

spinifex/domains/ec2/instance/uefistore/writeback_test.go

@@ -0,0 +1,109 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func markDeleting(t *testing.T, s *uefistore.Store, id string) {
+	t.Helper()
+	_, err := s.MarkDeleting(t.Context(), id)
+	require.NoError(t, err)
+}
+
+// An obligation is recorded on the live record, replacing any other, and
+// read back as recorded.
+func TestRecordWriteBack_SetsTheOneOpenObligation(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	got, err := s.LiveWriteBack(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Nil(t, got)
+
+	first := *writeBack()
+	require.NoError(t, s.RecordWriteBack(t.Context(), instanceID, first))
+	second := first
+	second.Node, second.Key = "node-3", uefistore.KeyPrefix(instanceID)+"c"
+	require.NoError(t, s.RecordWriteBack(t.Context(), instanceID, second))
+
+	got, err = s.LiveWriteBack(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, &second, got)
+}
+
+// A clear for one key leaves an obligation recorded for another, so a retry
+// cannot drop an obligation recorded after it read the record.
+func TestClearWriteBack_ClearsOnlyTheNamedKey(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	wb := *writeBack()
+	require.NoError(t, s.RecordWriteBack(t.Context(), instanceID, wb))
+
+	require.NoError(t, s.ClearWriteBack(t.Context(), instanceID, uefistore.KeyPrefix(instanceID)+"other"))
+	got, err := s.LiveWriteBack(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, &wb, got)
+
+	require.NoError(t, s.ClearWriteBack(t.Context(), instanceID, wb.Key))
+	got, err = s.LiveWriteBack(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Nil(t, got)
+
+	require.NoError(t, s.RecordWriteBack(t.Context(), instanceID, wb))
+	require.NoError(t, s.ClearWriteBack(t.Context(), instanceID, ""), "an empty key clears whatever is owed")
+	got, err = s.LiveWriteBack(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Nil(t, got)
+}
+
+// A store whose record is missing or deleting owes no write-back, and none
+// can be recorded on it.
+func TestWriteBack_RecordThatIsNotLiveIsErrNotLive(t *testing.T) {
+	s, _ := openStore(t)
+
+	_, err := s.LiveWriteBack(t.Context(), instanceID)
+	require.ErrorIs(t, err, uefistore.ErrNotLive)
+	require.ErrorIs(t, s.RecordWriteBack(t.Context(), instanceID, *writeBack()), uefistore.ErrNotLive)
+
+	_, err = s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	markDeleting(t, s, instanceID)
+
+	_, err = s.LiveWriteBack(t.Context(), instanceID)
+	require.ErrorIs(t, err, uefistore.ErrNotLive)
+	require.ErrorIs(t, s.RecordWriteBack(t.Context(), instanceID, *writeBack()), uefistore.ErrNotLive)
+	require.ErrorIs(t, s.ClearWriteBack(t.Context(), instanceID, ""), uefistore.ErrNotLive)
+	rec, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateDeleting, rec.State)
+	assert.Nil(t, rec.WriteBack, "a deleting record is never given an obligation")
+}
+
+// The reaper is handed only the live records whose obligation names its node.
+func TestWriteBacksOwedBy_ListsOnlyThisNodesLiveObligations(t *testing.T) {
+	s, _ := openStore(t)
+	ids := []string{"i-0000000000000000a", "i-0000000000000000b", "i-0000000000000000c", "i-0000000000000000d"}
+	for _, id := range ids {
+		_, err := s.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: id, State: uefistore.StateLive})
+		require.NoError(t, err)
+	}
+	owe := func(id, node string) {
+		wb := *writeBack()
+		wb.Node, wb.Key, wb.SeededFrom = node, uefistore.KeyPrefix(id)+"b", uefistore.KeyPrefix(id)+"a"
+		require.NoError(t, s.RecordWriteBack(t.Context(), id, wb))
+	}
+	owe(ids[0], "node-1")
+	owe(ids[1], "node-2")
+	owe(ids[2], "node-1")
+	markDeleting(t, s, ids[2])
+
+	got, err := s.WriteBacksOwedBy(t.Context(), "node-1")
+	require.NoError(t, err)
+	assert.Equal(t, []string{ids[0]}, got)
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -132,7 +132,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 
 	// Before the reason below is cleared, so a refused start changes nothing.
 	if err := CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
+		slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without a variable store",
 			"instanceId", instance.ID, "err", err)
 		return err
 	}
@@ -269,10 +269,6 @@ func (m *Manager) markAttachedVolumesInUse(instance *VM) error {
 	instance.EBSRequests.Mu.Unlock()
 
 	for _, ebsReq := range requests {
-		// EFI pflash is not a KV-tracked EBS volume; skip it like the unmount gate.
-		if ebsReq.EFI {
-			continue
-		}
 		if m.deps.VolumeStateUpdater == nil {
 			return errors.New("volume state updater not wired")
 		}
@@ -1437,8 +1433,8 @@ func buildDiskBootDrives(requests []EBSRequest, cpuCount int, machineType string
 //
 // The boot volume is a -blockdev node named "os" under a bootindex=1 device
 // that carries werror/rerror; it is never detached, so it keeps fixed names.
-// EFI volumes emit pflash unit=1, as uefiVarsDrive does for a variable store
-// pointer; Config.Execute adds the CODE blob (unit=0).
+// The UEFI variable store is not a volume: uefiVarsDrive adds it as pflash
+// unit=1, and Config.Execute adds the CODE blob (unit=0).
 //
 // Every other (data) volume is given the same named node-name/device-id/
 // iothread-id shape AttachVolume's hot-plug path uses, via a -blockdev entry
@@ -1463,10 +1459,6 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
 		}
 
 		switch {
-		case kind == volumeExportBlock && v.EFI:
-			// The VARS store is a pflash -drive, which has no host_device form.
-			return driveConfig{}, fmt.Errorf("volume %s: block device export is not supported for an EFI volume", v.Name)
-
 		case v.Boot:
 			blockdev, err := exportBlockdev(*v, kind, bootNodeName, bootNBDBlockdev)
 			if err != nil {
@@ -1487,18 +1479,10 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
 			cfg.Blockdevs = append(cfg.Blockdevs, blockdev)
 			cfg.Devices = append(cfg.Devices, dev)
 
-		case v.EFI:
-			cfg.Drives = append(cfg.Drives, Drive{
-				File:   v.NBDURI,
-				Format: "raw",
-				If:     "pflash",
-				Unit:   1,
-			})
-
 		default:
 			// microvm has no PCIe root ports, so its data volumes take none.
 			microvm := IsMicrovm(machineType)
-			// A data (non-boot, non-EFI) volume. Allocate a stable hot-plug
+			// A data (non-boot) volume. Allocate a stable hot-plug
 			// port when the persisted state predates port accounting, or the
 			// volume was attached at RunInstances time without one;
 			// freeHotplugEBSPort re-scans requests as mutated so far in this

spinifex/runtime/compute/vm/lifecycle_test.go

@@ -156,16 +156,6 @@ func TestBuildDrives(t *testing.T) {
 				{Value: `{"driver":"virtio-blk-pci","drive":"os","num-queues":4,"bootindex":1,"iothread-vq-mapping":[{"iothread":"ioth-os-0","vqs":[0,2]},{"iothread":"ioth-os-1","vqs":[1,3]}],"werror":"stop","rerror":"stop"}`},
 			},
 		},
-		{
-			name: "EFI volume emits pflash unit=1",
-			requests: []EBSRequest{
-				{Name: "vol-efi", NBDURI: "nbd:unix:/tmp/efi.sock", EFI: true},
-			},
-			cpuCount: 2,
-			wantDrives: []Drive{
-				{File: "nbd:unix:/tmp/efi.sock", Format: "raw", If: "pflash", Unit: 1},
-			},
-		},
 		{
 			name: "missing NBDURI returns error",
 			requests: []EBSRequest{
@@ -174,14 +164,6 @@ func TestBuildDrives(t *testing.T) {
 			cpuCount: 2,
 			wantErr:  "NBDURI not set for volume vol-bad",
 		},
-		{
-			name: "missing NBDURI on EFI returns error",
-			requests: []EBSRequest{
-				{Name: "vol-efi-bad", EFI: true},
-			},
-			cpuCount: 2,
-			wantErr:  "NBDURI not set for volume vol-efi-bad",
-		},
 		{
 			name: "data volume with a recorded HotplugPort keeps it and emits a named blockdev+device",
 			requests: []EBSRequest{
@@ -249,19 +231,15 @@ func TestBuildDrives(t *testing.T) {
 			wantErr:  "no free EBS hot-plug port for volume vol-overflow",
 		},
 		{
-			name: "mixed boot + EFI + data: boot is a -blockdev node and EFI keeps the pflash -drive",
+			name: "mixed boot + data: boot is a -blockdev node and no volume is a pflash -drive",
 			requests: []EBSRequest{
 				{Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true},
-				{Name: "vol-efi", NBDURI: "nbd:unix:/tmp/efi.sock", EFI: true},
 				{Name: "vol-data-a", NBDURI: "nbd:unix:/tmp/data-a.sock", HotplugPort: 3},
 			},
 			// 2 vCPUs, so the boot disk keeps the legacy scalar shape and this
 			// case still asserts what its name says it does. Data volumes are
 			// unaffected by the pool either way — they keep one IOThread each.
 			cpuCount: 2,
-			wantDrives: []Drive{
-				{File: "nbd:unix:/tmp/efi.sock", Format: "raw", If: "pflash", Unit: 1},
-			},
 			wantIOThreads: []IOThread{
 				{ID: "ioth-os"},
 				{ID: "ioth-vol-data-a"},
@@ -274,7 +252,7 @@ func TestBuildDrives(t *testing.T) {
 				{Value: "virtio-blk-pci,drive=os,iothread=ioth-os,num-queues=2,bootindex=1,werror=stop,rerror=stop"},
 				{Value: "virtio-blk-pci,id=vdisk-vol-data-a,drive=nbd-vol-data-a,iothread=ioth-vol-data-a,serial=voldataa,bus=hotplug-ebs3,werror=stop,rerror=stop"},
 			},
-			wantHotplugPorts: []int{0, 0, 3},
+			wantHotplugPorts: []int{0, 3},
 		},
 		{
 			name: "block data volume emits a raw node over host_device with the same device wiring",
@@ -409,12 +387,6 @@ func TestBuildDrives(t *testing.T) {
 			cpuCount: 2,
 			wantErr:  "parse NBDURI for volume vol-boot",
 		},
-		{
-			name:     "block EFI volume is rejected",
-			requests: []EBSRequest{{Name: "vol-efi", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20, EFI: true}},
-			cpuCount: 2,
-			wantErr:  "volume vol-efi: block device export is not supported for an EFI volume",
-		},
 		{
 			name:     "both NBDURI and BlockDevice set is rejected",
 			requests: []EBSRequest{{Name: "vol-both", NBDURI: "nbd:unix:/tmp/x.sock", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20}},

spinifex/runtime/compute/vm/manager.go

@@ -104,6 +104,13 @@ type Manager struct {
 	// imdsLocks serialises each primary ENI's IMDS datapath attach, repair and
 	// teardown, which run outside mu.
 	imdsLocks keyedMutex
+
+	// uefiLocks serialises a stop's variable store write-back with the
+	// reaper's retry of the same instance's obligation.
+	uefiLocks keyedMutex
+
+	// uefiPrunes is the prefixes landed write-backs left for the reaper to prune.
+	uefiPrunes uefiPruneSet
 }
 
 // NewManager returns a Manager with no collaborators wired. Tests that only

spinifex/runtime/compute/vm/record.go

@@ -107,9 +107,10 @@ type InstanceStatus struct {
 	ENIAvailableSlots []int          `json:"eni_available_slots,omitempty"`
 	ENIAttachedByID   map[string]int `json:"eni_attached_by_id,omitempty"`
 
-	// UEFIVarStore is optional, so records written without it still decode
-	// and the bucket schema does not change.
-	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
+	// The UEFI fields are optional, so records written without them still
+	// decode and the bucket schema does not change.
+	UEFIVarStore      *UEFIVarStoreRef   `json:"uefi_var_store,omitempty"`
+	UEFIWriteBackOwed *UEFIWriteBackOwed `json:"uefi_write_back_owed,omitempty"`
 }
 
 // Record projects a VM onto the persisted record. QMPClient and the request
@@ -171,6 +172,7 @@ func (v *VM) Record() *InstanceRecord {
 			ENIAvailableSlots:               v.ENIRequests.AvailableSlots,
 			ENIAttachedByID:                 v.ENIRequests.AttachedByENIID,
 			UEFIVarStore:                    v.UEFIVarStore,
+			UEFIWriteBackOwed:               v.UEFIWriteBackOwed,
 		},
 	}
 }
@@ -228,6 +230,7 @@ func VMFromRecord(r *InstanceRecord) *VM {
 		ShuttingDownAt:                  r.Status.ShuttingDownAt,
 		FirstLaunch:                     r.Status.FirstLaunch,
 		UEFIVarStore:                    r.Status.UEFIVarStore,
+		UEFIWriteBackOwed:               r.Status.UEFIWriteBackOwed,
 	}
 	v.EBSRequests.Requests = r.Status.EBSRequests
 	v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots

spinifex/runtime/compute/vm/record_test.go

@@ -244,6 +244,7 @@ func populatedVM() *vm.VM {
 			WriterNode: "node-3",
 			WrittenAt:  time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
 		},
+		UEFIWriteBackOwed: &vm.UEFIWriteBackOwed{Node: "node-3", Since: time.Date(2026, 8, 28, 5, 0, 0, 0, time.UTC)},
 
 		Status:                          vm.StateRunning,
 		Instance:                        &ec2.Instance{InstanceId: aws.String("i-round-trip")},

spinifex/runtime/compute/vm/shutdown.go

@@ -100,7 +100,11 @@ func (m *Manager) Stop(id string) error {
 		return ErrInstanceNotFound
 	}
 
-	migrated, stopErr := m.stopOne(instance)
+	migrated, pending, stopErr := m.stopOne(instance)
+	if pending != nil {
+		// A stop that kept the instance in the running set records it there.
+		m.settleUEFIWriteBack(context.Background(), pending, m.writeRunningState())
+	}
 	// A failed seal still ran the whole sequence, so the ownership hand-off
 	// below must still happen; the error is reported after it.
 	if stopErr != nil && !errors.Is(stopErr, ErrVolumeSealFailed) {
@@ -125,17 +129,19 @@ func (m *Manager) Stop(id string) error {
 // Stopping → stopCleanup → release auto-assigned IP → Stopped → migrate to
 // "stopped" KV → OnInstanceDown.
 // The bool reports whether migration removed the instance (caller must persist).
+// A returned write-back is the caller's to settle after it persists the
+// running set, which is the record write a stop that does not migrate makes.
 //
 // A failed volume seal (ErrVolumeSealFailed) does not abort the sequence —
 // QEMU is already down, so the remaining steps must still run — but it is
 // returned so the caller can refuse to advance. A precheck failure returns
 // ErrInvalidTransition with nothing torn down.
-func (m *Manager) stopOne(instance *VM) (bool, error) {
+func (m *Manager) stopOne(instance *VM) (bool, *uefiWriteBack, error) {
 	if err := m.transitionWithPrecheck(instance, StateStopping); err != nil {
-		return false, err
+		return false, nil, err
 	}
 
-	sealErr := m.stopCleanup(instance)
+	pending, sealErr := m.stopCleanup(instance)
 
 	m.UpdateState(instance.ID, func(v *VM) { v.LastNode = m.deps.NodeID })
 
@@ -158,23 +164,25 @@ func (m *Manager) stopOne(instance *VM) (bool, error) {
 		// The address is kept for the same reason. A drain is not a customer
 		// stop, and Restore brings the instance back on the same address rather
 		// than changing it under a guest nobody asked to move.
-		return false, sealErr
+		return false, pending, sealErr
 	}
 
 	if !m.MigrateStoppedToSharedKV(instance) {
+		m.settleUEFIWriteBack(context.Background(), pending, errUEFIRecordWriteFailed)
 		// Either StateStore unavailable / write failed (instance stays in
 		// local map; restoreInstances retries on next boot) OR a concurrent
 		// handler reclaimed the slot (id now resolves to a different live
 		// VM). Either way, do not fire OnInstanceDown — firing it would
 		// unsubscribe the per-id NATS subscriptions of the reclaimed
 		// instance.
-		return false, sealErr
+		return false, nil, sealErr
 	}
+	m.settleUEFIWriteBack(context.Background(), pending, nil)
 
 	if m.deps.Hooks.OnInstanceDown != nil {
 		m.deps.Hooks.OnInstanceDown(instance.ID)
 	}
-	return true, sealErr
+	return true, nil, sealErr
 }
 
 // StopAll fans stopOne across every VM for the coordinated shutdown DRAIN phase.
@@ -191,13 +199,20 @@ func (m *Manager) StopAll() error {
 	var (
 		mu       sync.Mutex
 		sealErrs []error
+		pending  []*uefiWriteBack
 	)
 	var wg sync.WaitGroup
 	for _, instance := range snapshot {
 		wg.Add(1)
 		go func(v *VM) {
 			defer wg.Done()
-			if _, err := m.stopOne(v); err != nil {
+			_, wb, err := m.stopOne(v)
+			if wb != nil {
+				mu.Lock()
+				pending = append(pending, wb)
+				mu.Unlock()
+			}
+			if err != nil {
 				if errors.Is(err, ErrInvalidTransition) {
 					slog.Debug("StopAll: skipping non-running instance",
 						"instanceId", v.ID, "state", string(m.Status(v)))
@@ -213,9 +228,13 @@ func (m *Manager) StopAll() error {
 		}(instance)
 	}
 	wg.Wait()
-	if err := m.writeRunningState(); err != nil {
-		slog.Error("StopAll: failed to persist running state after fan-out", "err", err)
-		sealErrs = append(sealErrs, err)
+	persistErr := m.writeRunningState()
+	for _, wb := range pending {
+		m.settleUEFIWriteBack(context.Background(), wb, persistErr)
+	}
+	if persistErr != nil {
+		slog.Error("StopAll: failed to persist running state after fan-out", "err", persistErr)
+		sealErrs = append(sealErrs, persistErr)
 	}
 	return errors.Join(sealErrs...)
 }
@@ -341,14 +360,17 @@ func (m *Manager) MarkRecoveryFailed(instance *VM, reason string) {
 		"instanceId", instance.ID, "reason", reason)
 
 	m.goroutineWg.Go(func() {
-		if err := m.stopCleanup(instance); err != nil {
+		pending, err := m.stopCleanup(instance)
+		if err != nil {
 			slog.Error("Volume seal failed during recovery cleanup; volume left unsealed for operator action",
 				"instanceId", instance.ID, "err", err)
 		}
 		m.Inspect(instance, func(v *VM) { v.LastNode = m.deps.NodeID })
-		if err := m.writeRunningState(); err != nil {
+		persistErr := m.writeRunningState()
+		m.settleUEFIWriteBack(context.Background(), pending, persistErr)
+		if persistErr != nil {
 			slog.Error("Failed to persist state after recovery failure",
-				"instanceId", instance.ID, "err", err)
+				"instanceId", instance.ID, "err", persistErr)
 		}
 	})
 }
@@ -500,12 +522,14 @@ func (m *Manager) stampOutstandingTeardownFailed(instance *VM) {
 	})
 }
 
-// stopCleanup performs the per-instance teardown shared by Stop and the
-// initial section of Terminate: graceful QMP shutdown, PID-file wait,
-// volume unmount, tap teardown (main + extra ENI + mgmt), resource
-// deallocation. Every step runs; only a failed volume seal is returned.
-func (m *Manager) stopCleanup(instance *VM) error {
-	sealErr := m.shutdownAndUnmount(instance)
+// stopCleanup performs the per-instance teardown of a sealing stop: graceful
+// QMP shutdown, PID-file wait, variable store write-back, volume unmount, tap
+// teardown (main + extra ENI + mgmt), resource deallocation. Every step runs;
+// only a failed volume seal is returned, with any write-back left to settle.
+func (m *Manager) stopCleanup(instance *VM) (*uefiWriteBack, error) {
+	m.shutdownQEMU(instance)
+	pending := m.writeBackUEFIVarStore(context.Background(), instance)
+	sealErr := m.unmountAfterShutdown(instance)
 	m.cleanupTapDevices(instance)
 	if m.deps.InstanceCleaner != nil {
 		if err := m.deps.InstanceCleaner.ReleaseGPU(instance); err != nil {
@@ -522,7 +546,7 @@ func (m *Manager) stopCleanup(instance *VM) error {
 		m.UpdateState(instance.ID, func(v *VM) { v.CapacityReservationId = "" })
 	}
 
-	return sealErr
+	return pending, sealErr
 }
 
 // releaseAutoAssignedPublicIP returns the instance's auto-assigned address to
@@ -620,7 +644,12 @@ func (m *Manager) terminateCleanup(instance *VM) {
 // and telemetry cleanup stay best-effort and are logged only.
 func (m *Manager) shutdownAndUnmount(instance *VM) error {
 	m.shutdownQEMU(instance)
+	return m.unmountAfterShutdown(instance)
+}
 
+// unmountAfterShutdown is shutdownAndUnmount once QEMU is down: the unmount,
+// then the fw_cfg and telemetry cleanup.
+func (m *Manager) unmountAfterShutdown(instance *VM) error {
 	var sealErr error
 	if m.deps.VolumeMounter != nil {
 		if err := m.deps.VolumeMounter.Unmount(context.Background(), instance); err != nil {

spinifex/runtime/compute/vm/shutdown_test.go

@@ -634,7 +634,8 @@ func TestStopCleanup_InvokesReleaseGPU(t *testing.T) {
 	m := NewManagerWithDeps(Deps{InstanceCleaner: cleaner})
 	instance := &VM{ID: "i-stop", GPUAttachments: []gpu.GPUAttachment{{PCIAddress: "0000:01:00.0"}}}
 
-	require.NoError(t, m.stopCleanup(instance))
+	_, err := m.stopCleanup(instance)
+	require.NoError(t, err)
 
 	if got := cleaner.releaseGPU; len(got) != 1 || got[0] != "i-stop" {
 		t.Fatalf("ReleaseGPU on stopCleanup: got %v, want [i-stop]", got)
@@ -656,7 +657,8 @@ func TestStopCleanup_ReservationBoundReleasesAndClears(t *testing.T) {
 	instance := &VM{ID: "i-cr", InstanceType: "t3.micro", CapacityReservationId: "cr-123"}
 	m.Insert(instance)
 
-	require.NoError(t, m.stopCleanup(instance))
+	_, err := m.stopCleanup(instance)
+	require.NoError(t, err)
 
 	if got := rc.releasedToCRID; len(got) != 1 || got[0] != "cr-123" {
 		t.Fatalf("stop must release the slot to the reservation: got %v, want [cr-123]", got)
@@ -694,7 +696,8 @@ func TestCleanup_NoGPU_StillInvokesReleaseGPU(t *testing.T) {
 	m := NewManagerWithDeps(Deps{InstanceCleaner: cleaner})
 	instance := &VM{ID: "i-cpu"}
 
-	require.NoError(t, m.stopCleanup(instance))
+	_, err := m.stopCleanup(instance)
+	require.NoError(t, err)
 	m.terminateCleanup(instance)
 
 	if got := len(cleaner.releaseGPU); got != 2 {
@@ -1551,7 +1554,7 @@ func TestStopCleanup_SealFailure_StillRunsBestEffortSteps(t *testing.T) {
 	instance := &VM{ID: "i-seal-cleanup", InstanceType: "t3.micro"}
 	m.Insert(instance)
 
-	err := m.stopCleanup(instance)
+	_, err := m.stopCleanup(instance)
 
 	require.ErrorIs(t, err, ErrVolumeSealFailed)
 	assert.Equal(t, []string{"i-seal-cleanup"}, cleaner.releaseGPU,

spinifex/runtime/compute/vm/uefi_varstore.go

@@ -11,7 +11,6 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
-	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -33,6 +32,9 @@ const (
 	uefiStoreDirName   = "uefi"
 	uefiVarsFileName   = "vars.fd"
 	uefiSeededFileName = "seeded.json"
+	// uefiOwedFileName holds a write-back this node owes that the store
+	// record could not take.
+	uefiOwedFileName = "write-back.json"
 
 	// uefiStoreGetTimeout bounds the one get a launch makes for a fixed-size
 	// object, so a wedged object store fails the start rather than holding it.
@@ -48,12 +50,12 @@ const (
 )
 
 // ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
-// store and carries no EFI volume, so it has nothing to boot its variables from.
+// store, so it has nothing to boot its variables from.
 var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
 
-// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
-// store and also carries an EFI volume, so either one would be ignored.
-var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
+// ErrUEFIWriteBackPending refuses a start on this node while another node that
+// is still a cluster member owes the instance's variable store write-back.
+var ErrUEFIWriteBackPending = errors.New("another node still owes this instance's UEFI variable store write-back")
 
 // ErrUEFIFirmwareMismatch is a variable store written under firmware whose
 // architecture or VARS size differs from this node's, which pflash cannot load.
@@ -70,6 +72,13 @@ type UEFIVarStoreDeps struct {
 	Objects objectstore.ObjectStore
 	Bucket  string
 	DataDir string
+	// Records and Instances carry write-back obligations and move the
+	// pointer; write-back is off on a node without them.
+	Records   UEFIStoreRecords
+	Instances UEFIInstancePointers
+	// NodeIsMember reports whether node is in the cluster's configuration. A
+	// start waits for a member's owed write-back; nil counts every node a member.
+	NodeIsMember func(node string) bool
 }
 
 // uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
@@ -83,28 +92,17 @@ func isUEFIBootMode(bootMode string) bool {
 	return bootMode == "uefi" || bootMode == "uefi-preferred"
 }
 
-// usesUEFIVarStore reports whether v boots through UEFI and so needs a
+// UsesUEFIVarStore reports whether v boots through UEFI and so needs a
 // variable store. A direct-boot microvm boots qboot and has none.
-func usesUEFIVarStore(v *VM) bool {
+func UsesUEFIVarStore(v *VM) bool {
 	return !v.DirectBoot && isUEFIBootMode(v.BootMode)
 }
 
-// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
-// two, before anything about it is changed. An EFI volume still counts as a
-// store until RunInstances seeds pointers instead. The caller owns v.
+// CheckUEFIVarStore refuses a UEFI instance whose record names no variable
+// store, before anything about it is changed. The caller owns v.
 func CheckUEFIVarStore(v *VM) error {
-	if !usesUEFIVarStore(v) {
-		return nil
-	}
-	v.EBSRequests.Mu.Lock()
-	hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
-	v.EBSRequests.Mu.Unlock()
-
-	switch {
-	case v.UEFIVarStore == nil && !hasEFIVolume:
+	if UsesUEFIVarStore(v) && v.UEFIVarStore == nil {
 		return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
-	case v.UEFIVarStore != nil && hasEFIVolume:
-		return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
 	}
 	return nil
 }
@@ -128,13 +126,16 @@ func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
 }
 
 // uefiVarsDrive is pflash unit 1 from this node's copy of the store the
-// record points at, or nil with no pointer. Launch seeds the copy and a
-// reboot reuses it, so a missing one fails rather than booting without it.
+// record points at, or nil for a guest that boots without UEFI. Launch seeds
+// the copy and a reboot reuses it, so a missing one fails the boot.
 func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
+	if !UsesUEFIVarStore(instance) {
+		return nil, nil
+	}
 	hasPointer := false
 	m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
-	if !hasPointer || !usesUEFIVarStore(instance) {
-		return nil, nil
+	if !hasPointer {
+		return nil, fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, instance.ID)
 	}
 	path, err := m.uefiVarsPath(instance.ID)
 	if err != nil {
@@ -157,7 +158,7 @@ func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
 			ref = &cp
 		}
 	})
-	if ref == nil || !usesUEFIVarStore(instance) {
+	if ref == nil || !UsesUEFIVarStore(instance) {
 		return nil
 	}
 
@@ -183,11 +184,19 @@ func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
 		return err
 	}
 
+	// Before reuse too: this node's copy is as stale as the object when another
+	// node holds the newer variables.
+	if err := m.refuseWhileWriteBackOwedElsewhere(ctx, instance); err != nil {
+		return err
+	}
+
 	dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
 	if err != nil {
 		return err
 	}
+	// The copy booted here supersedes any owed write-back the refusal let by.
 	if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
+		m.Inspect(instance, func(v *VM) { v.UEFIWriteBackOwed = nil })
 		recordUEFIStoreSeed("local")
 		slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
 		return nil
@@ -201,13 +210,11 @@ func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
 	if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
 		return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
 	}
-	seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
-	if err != nil {
-		return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
-	}
-	if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
+	if err := writeUEFISeed(dir, ref.Key); err != nil {
 		return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
 	}
+	m.clearForeignUEFIWriteBack(ctx, instance.ID)
+	m.Inspect(instance, func(v *VM) { v.UEFIWriteBackOwed = nil })
 	recordUEFIStoreSeed("object")
 	slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
 		"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
@@ -354,6 +361,8 @@ var (
 	uefiStoreMetricsOnce   sync.Once
 	uefiStoreSeedCounter   metric.Int64Counter
 	uefiFirmwareMismatches metric.Int64Counter
+	uefiWriteBacks         metric.Int64Counter
+	uefiWriteBackSkips     metric.Int64Counter
 )
 
 func uefiStoreMetrics() {
@@ -372,6 +381,18 @@ func uefiStoreMetrics() {
 		if err != nil {
 			otel.Handle(err)
 		}
+		uefiWriteBacks, err = meter.Int64Counter("mulga.vm.uefi_store.write_backs",
+			metric.WithDescription("UEFI variable store write-backs by outcome: written, deferred to the reaper, retried by it, dropped, or failed."),
+			metric.WithUnit("{write_back}"))
+		if err != nil {
+			otel.Handle(err)
+		}
+		uefiWriteBackSkips, err = meter.Int64Counter("mulga.vm.uefi_store.write_back_skips",
+			metric.WithDescription("Sealing stops that wrote no UEFI variable store back, by the check that refused it."),
+			metric.WithUnit("{stop}"))
+		if err != nil {
+			otel.Handle(err)
+		}
 	})
 }
 

spinifex/runtime/compute/vm/uefi_varstore_test.go

@@ -399,37 +399,28 @@ func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
 	assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
 }
 
-// A UEFI instance with no pointer, or two stores, is refused before the
-// launch clears or changes anything on its record.
-func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
-	efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
+// A UEFI instance with no pointer is refused before the launch clears or
+// changes anything on its record, and an EFI volume is not a store.
+func TestRun_UEFIInstanceWithoutAPointerIsRefused(t *testing.T) {
 	tests := []struct {
-		name    string
-		pointer bool
-		efi     bool
-		want    error
+		name     string
+		requests []EBSRequest
 	}{
-		{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
-		{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
+		{name: "no volumes"},
+		{name: "an EFI volume", requests: []EBSRequest{{Name: "vol-1-efi", NBDURI: "nbd:unix:/tmp/efi.sock", EFI: true}}},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			f := newUEFIFixture(t)
-			key := ""
-			if tt.pointer {
-				key = keyFor("p")
-			}
-			instance := f.instance(key, f.identity)
-			if tt.efi {
-				instance.EBSRequests.Requests = []EBSRequest{efiVolume}
-			}
+			instance := f.instance("", f.identity)
+			instance.EBSRequests.Requests = tt.requests
 			stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
 			instance.Instance.StateReason = stale
 			before := instance.Record()
 
 			err := f.m.Run(t.Context(), instance)
 
-			require.ErrorIs(t, err, tt.want)
+			require.ErrorIs(t, err, ErrUEFIVarStoreMissing)
 			assert.Empty(t, f.mounter.mounted)
 			assert.Equal(t, before, instance.Record(), "the record is unchanged")
 		})
@@ -452,15 +443,13 @@ func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testin
 	assert.Equal(t, StateError, instance.Status)
 }
 
-// Until RunInstances seeds pointers, an instance whose variables are on an
-// EFI volume still launches through the volume, and a BIOS or microvm
-// guest needs no store at all.
+// A UEFI instance with a pointer launches, and a BIOS or microvm guest needs
+// no store at all.
 func TestCheckUEFIVarStore_Accepts(t *testing.T) {
 	tests := []struct {
 		name string
 		v    *VM
 	}{
-		{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
 		{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
 		{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
 		{name: "no boot mode", v: &VM{}},
@@ -495,6 +484,21 @@ func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
 	}, allArgValues(got[0], "-drive"))
 }
 
+// A UEFI guest is never started without its VARS drive: not when the
+// node's copy is missing, and not when the record names no store.
+func TestStartQEMU_UEFIWithoutAPointerIsNotLaunched(t *testing.T) {
+	f := newUEFIFixture(t)
+	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
+	launches := withFakeQEMU(t)
+	instance := f.instance("", f.identity)
+	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
+
+	err := f.m.startQEMU(instance)
+
+	require.ErrorIs(t, err, ErrUEFIVarStoreMissing)
+	assert.Empty(t, launches())
+}
+
 // A reboot reuses the copy the launch seeded; without one there is nothing
 // safe to boot, so QEMU is not started.
 func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {

spinifex/runtime/compute/vm/uefi_writeback.go

@@ -0,0 +1,448 @@
+package vm
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// UEFIStoreRecords is the store-record bucket as a write-back reads and
+// changes it. A record that is missing or deleting is uefistore.ErrNotLive.
+type UEFIStoreRecords interface {
+	LiveWriteBack(ctx context.Context, instanceID string) (*uefistore.UEFIWriteBack, error)
+	RecordWriteBack(ctx context.Context, instanceID string, wb uefistore.UEFIWriteBack) error
+	ClearWriteBack(ctx context.Context, instanceID, key string) error
+	WriteBacksOwedBy(ctx context.Context, node string) ([]string, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// UEFIVarStoreOwner is what an instance record says about who may write its
+// variable store back, read at Revision.
+type UEFIVarStoreOwner struct {
+	Status   InstanceState
+	LastNode string
+	Pointer  *UEFIVarStoreRef
+	Owed     *UEFIWriteBackOwed
+	Revision uint64
+}
+
+// UEFIInstancePointers reads and moves the variable store pointer on the
+// cluster's instance records. Reads are of the current revision, an absent
+// record is found=false, and an unreachable store is an error, never absence.
+type UEFIInstancePointers interface {
+	LoadUEFIVarStoreOwner(instanceID string) (owner UEFIVarStoreOwner, found bool, err error)
+	// CompareAndSetUEFIVarStore points the record at ref and clears its owed
+	// write-back only while it is still at rev, and fails otherwise.
+	CompareAndSetUEFIVarStore(instanceID string, ref *UEFIVarStoreRef, rev uint64) error
+}
+
+// errUEFIRecordWriteFailed is a stop whose record write did not land, so the
+// pointer its write-back set may not be on the record.
+var errUEFIRecordWriteFailed = errors.New("the stop's instance record write did not land")
+
+// uefiWriteBack is a write-back whose put landed and whose pointer is set on
+// the in-memory instance, held under the instance's write-back lock until the
+// stop's record write settles it.
+type uefiWriteBack struct {
+	vm         *VM
+	dir        string
+	key        string
+	seededFrom string
+	since      time.Time
+	previous   *UEFIVarStoreRef
+	put        bool
+	unlock     func()
+}
+
+// writeBackWired reports whether this node can write variable stores back.
+func (d UEFIVarStoreDeps) writeBackWired() bool {
+	return d.Objects != nil && d.Bucket != "" && d.DataDir != "" && d.Records != nil && d.Instances != nil
+}
+
+// writeBackUEFIVarStore writes a sealing stop's live variable store to a key
+// under the instance's prefix, once QEMU is down. The returned write-back is
+// settled after the stop's record write; nil means there is nothing to settle.
+func (m *Manager) writeBackUEFIVarStore(ctx context.Context, instance *VM) *uefiWriteBack {
+	var ref *UEFIVarStoreRef
+	var status InstanceState
+	fenced := false
+	m.Inspect(instance, func(v *VM) {
+		if v.UEFIVarStore != nil {
+			cp := *v.UEFIVarStore
+			ref = &cp
+		}
+		status = v.Status
+		fenced = VolumeFenced(v.Instance)
+	})
+	if ref == nil || !UsesUEFIVarStore(instance) {
+		return nil
+	}
+	switch {
+	case status == StateError:
+		skipUEFIWriteBack(ctx, instance.ID, "error", "the instance is in error")
+		return nil
+	case fenced:
+		skipUEFIWriteBack(ctx, instance.ID, "fenced", "a fence stopped the instance")
+		return nil
+	}
+	deps := m.deps.UEFIVarStore
+	if !deps.writeBackWired() || m.deps.NodeID == "" {
+		slog.ErrorContext(ctx, "UEFI variable store write-back is not wired on this node; the store keeps its last version", "instanceId", instance.ID)
+		recordUEFIWriteBackOutcome("unwired")
+		return nil
+	}
+	dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
+	if err != nil {
+		slog.ErrorContext(ctx, "UEFI variable store write-back has no directory", "instanceId", instance.ID, "err", err)
+		recordUEFIWriteBackOutcome("failed")
+		return nil
+	}
+
+	unlock := m.uefiLocks.lock(instance.ID)
+	wb := &uefiWriteBack{vm: instance, dir: dir, previous: ref, since: time.Now().UTC(), unlock: unlock}
+	if !m.prepareUEFIWriteBack(ctx, wb, ref) {
+		unlock()
+		return nil
+	}
+	return wb
+}
+
+// prepareUEFIWriteBack runs the skip checks, chooses the key and puts the
+// store. It reports false when nothing is left to settle: a skip, or a failure
+// already recorded as an obligation for the reaper.
+func (m *Manager) prepareUEFIWriteBack(ctx context.Context, wb *uefiWriteBack, ref *UEFIVarStoreRef) bool {
+	id := wb.vm.ID
+	deps := m.deps.UEFIVarStore
+	seed, err := readUEFISeed(wb.dir)
+	if err != nil {
+		skipUEFIWriteBack(ctx, id, "unseeded", "this node holds no record of what its copy was seeded from", "err", err)
+		return false
+	}
+	if seed != ref.Key {
+		skipUEFIWriteBack(ctx, id, "reseeded", "the record moved on after this node seeded", "seededFrom", seed, "pointer", ref.Key)
+		return false
+	}
+	wb.seededFrom = seed
+
+	owed, err := deps.Records.LiveWriteBack(ctx, id)
+	if errors.Is(err, uefistore.ErrNotLive) {
+		skipUEFIWriteBack(ctx, id, "store_not_live", "the store record is gone or deleting", "err", err)
+		return false
+	}
+	if err != nil {
+		// Ownership is not checked here: the reaper checks it before any put.
+		if wb.key, err = uefistore.NewObjectKey(id); err != nil {
+			slog.ErrorContext(ctx, "Could not name a UEFI variable store object", "instanceId", id, "err", err)
+			recordUEFIWriteBackOutcome("failed")
+			return false
+		}
+		m.deferUEFIWriteBack(ctx, wb, fmt.Errorf("read store record: %w", err))
+		return false
+	}
+	// A retry of this node's own obligation reuses its key, so failed write-backs
+	// leave one leftover rather than one each. The pointed key is never reused.
+	if owed != nil && owed.Node == m.deps.NodeID && owed.Key != seed {
+		wb.key, wb.since = owed.Key, owed.Since
+	} else if wb.key, err = uefistore.NewObjectKey(id); err != nil {
+		slog.ErrorContext(ctx, "Could not name a UEFI variable store object", "instanceId", id, "err", err)
+		recordUEFIWriteBackOutcome("failed")
+		return false
+	}
+
+	owner, found, err := deps.Instances.LoadUEFIVarStoreOwner(id)
+	switch {
+	case err != nil:
+		m.deferUEFIWriteBack(ctx, wb, fmt.Errorf("read instance record: %w", err))
+		return false
+	case !found:
+		m.deferUEFIWriteBack(ctx, wb, errors.New("no instance record to check ownership against"))
+		return false
+	case owner.LastNode != m.deps.NodeID:
+		skipUEFIWriteBack(ctx, id, "superseded", "the record names another node", "lastNode", owner.LastNode)
+		return false
+	case owner.Pointer == nil || owner.Pointer.Key != seed:
+		var pointer string
+		if owner.Pointer != nil {
+			pointer = owner.Pointer.Key
+		}
+		skipUEFIWriteBack(ctx, id, "reseeded", "the record moved on after this node seeded", "seededFrom", seed, "pointer", pointer)
+		return false
+	}
+
+	start := time.Now()
+	written, err := m.putUEFIVars(ctx, id, wb.dir, wb.key, ref.Firmware.Arch)
+	if err != nil {
+		m.deferUEFIWriteBack(ctx, wb, err)
+		return false
+	}
+	wb.put = true
+	m.Inspect(wb.vm, func(v *VM) {
+		v.UEFIVarStore = written
+		v.UEFIWriteBackOwed = nil
+	})
+	slog.InfoContext(ctx, "Wrote the UEFI variable store back", "instanceId", id, "key", wb.key,
+		"seededFrom", seed, "put_ms", otelsetup.Millis(time.Since(start)))
+	return true
+}
+
+// settleUEFIWriteBack completes wb once the stop's record write is known. A
+// record write that failed reverts the in-memory pointer and leaves the
+// obligation for the reaper, which puts to the same key.
+func (m *Manager) settleUEFIWriteBack(ctx context.Context, wb *uefiWriteBack, recordErr error) {
+	if wb == nil {
+		return
+	}
+	defer wb.unlock()
+	if recordErr != nil {
+		m.Inspect(wb.vm, func(v *VM) {
+			if v.UEFIVarStore != nil && v.UEFIVarStore.Key == wb.key {
+				v.UEFIVarStore = wb.previous
+			}
+		})
+		m.deferUEFIWriteBack(ctx, wb, recordErr)
+		return
+	}
+	if m.finishUEFIWriteBack(ctx, wb.vm.ID, wb.dir, wb.key, wb.seededFrom, "") {
+		recordUEFIWriteBackOutcome("written")
+	}
+}
+
+// deferUEFIWriteBack records wb as owed by this node, on the store record or
+// else in its directory, and marks the in-memory instance so the stop's record
+// write carries it. A store no longer live owes nothing; a put made is deleted.
+func (m *Manager) deferUEFIWriteBack(ctx context.Context, wb *uefiWriteBack, cause error) {
+	id := wb.vm.ID
+	owed := uefistore.UEFIWriteBack{Node: m.deps.NodeID, Key: wb.key, SeededFrom: wb.seededFrom, Since: wb.since}
+	err := m.deps.UEFIVarStore.Records.RecordWriteBack(ctx, id, owed)
+	if errors.Is(err, uefistore.ErrNotLive) {
+		if wb.put {
+			m.deleteUEFIObject(ctx, id, wb.key)
+		}
+		slog.InfoContext(ctx, "UEFI store record is gone or deleting; no write-back is owed", "instanceId", id, "cause", cause, "err", err)
+		recordUEFIWriteBackOutcome("dropped")
+		return
+	}
+	m.Inspect(wb.vm, func(v *VM) { v.UEFIWriteBackOwed = &UEFIWriteBackOwed{Node: owed.Node, Since: owed.Since} })
+	if err != nil {
+		if ferr := writeUEFIOwed(wb.dir, owed); ferr != nil {
+			slog.ErrorContext(ctx, "UEFI variable store write-back failed and could not be recorded anywhere; only a start on this node keeps the changes",
+				"instanceId", id, "key", wb.key, "cause", cause, "err", err, "fileErr", ferr)
+		} else {
+			slog.ErrorContext(ctx, "UEFI variable store write-back failed and the store record is unwritable; kept on this node for the reaper",
+				"instanceId", id, "key", wb.key, "seededFrom", wb.seededFrom, "cause", cause, "err", err)
+		}
+		recordUEFIWriteBackOutcome("unrecorded")
+		return
+	}
+	removeUEFIOwed(ctx, id, wb.dir)
+	slog.WarnContext(ctx, "UEFI variable store write-back failed; recorded for retry", "instanceId", id, "key", wb.key,
+		"seededFrom", wb.seededFrom, "cause", cause)
+	recordUEFIWriteBackOutcome("deferred")
+}
+
+// finishUEFIWriteBack runs once a record points at key: seeded.json names it,
+// the obligation is cleared and the prefix queued for the reaper to prune. It
+// reports whether the store was still live; when not, key is deleted.
+func (m *Manager) finishUEFIWriteBack(ctx context.Context, id, dir, key, seededFrom, clearKey string) bool {
+	deps := m.deps.UEFIVarStore
+	removeUEFIOwed(ctx, id, dir)
+	if err := writeUEFISeed(dir, key); err != nil {
+		slog.WarnContext(ctx, "Could not record the written-back UEFI store locally; the next start here fetches it", "instanceId", id, "key", key, "err", err)
+	}
+	err := deps.Records.ClearWriteBack(ctx, id, clearKey)
+	if errors.Is(err, uefistore.ErrNotLive) {
+		m.deleteUEFIObject(ctx, id, key)
+		slog.InfoContext(ctx, "UEFI store record is gone or deleting; deleted the store just written", "instanceId", id, "key", key, "err", err)
+		recordUEFIWriteBackOutcome("dropped")
+		return false
+	}
+	if err != nil {
+		// An obligation left on the record names a key a retry reuses, so the
+		// prune waits for the next write rather than guess which to keep.
+		slog.WarnContext(ctx, "Could not clear the UEFI write-back obligation; not pruning", "instanceId", id, "err", err)
+		return true
+	}
+	m.uefiPrunes.add(id, uefiPrune{key: key, seededFrom: seededFrom})
+	return true
+}
+
+// putUEFIVars puts this node's vars.fd to key under this node's firmware for
+// arch, returning the pointer that names it.
+func (m *Manager) putUEFIVars(ctx context.Context, id, dir, key, arch string) (*UEFIVarStoreRef, error) {
+	deps := m.deps.UEFIVarStore
+	fw, err := NodeFirmwareFor(arch)
+	if err != nil {
+		return nil, err
+	}
+	region, err := os.ReadFile(filepath.Join(dir, uefiVarsFileName))
+	if err != nil {
+		return nil, fmt.Errorf("read UEFI variable store for %s: %w", id, err)
+	}
+	if err := uefistore.PutObject(ctx, deps.Objects, deps.Bucket, key, region, uefistore.ObjectMeta{
+		OwnerID:    id,
+		WriterNode: m.deps.NodeID,
+		Firmware:   fw.Identity,
+		CodePath:   fw.Code,
+		Source:     uefistore.SourceWriteBack,
+	}); err != nil {
+		return nil, err
+	}
+	return &UEFIVarStoreRef{Key: key, Firmware: fw.Identity, WriterNode: m.deps.NodeID, WrittenAt: time.Now().UTC()}, nil
+}
+
+// deleteUEFIObject deletes one object this node wrote and no record points at.
+func (m *Manager) deleteUEFIObject(ctx context.Context, id, key string) {
+	deps := m.deps.UEFIVarStore
+	ctx, cancel := context.WithTimeout(ctx, uefistore.PutTimeout)
+	defer cancel()
+	_, err := deps.Objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(deps.Bucket), Key: aws.String(key)})
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.WarnContext(ctx, "Could not delete an unpointed UEFI variable store object; a later prune removes it", "instanceId", id, "key", key, "err", err)
+	}
+}
+
+// refuseWhileWriteBackOwedElsewhere fails a start here while a cluster member
+// other than this node owes the write-back, by the instance record's marker
+// or the store record's obligation: its retained copy is newer than the pointer.
+func (m *Manager) refuseWhileWriteBackOwedElsewhere(ctx context.Context, instance *VM) error {
+	id := instance.ID
+	deps := m.deps.UEFIVarStore
+	var marker *UEFIWriteBackOwed
+	m.Inspect(instance, func(v *VM) {
+		if v.UEFIWriteBackOwed != nil {
+			cp := *v.UEFIWriteBackOwed
+			marker = &cp
+		}
+	})
+	if marker != nil && marker.Node != m.deps.NodeID && (deps.NodeIsMember == nil || deps.NodeIsMember(marker.Node)) {
+		slog.WarnContext(ctx, "Refusing to start a UEFI instance while another node owes its variable store write-back",
+			"instanceId", id, "node", marker.Node, "since", marker.Since)
+		return fmt.Errorf("%w: %s owes %s", ErrUEFIWriteBackPending, marker.Node, id)
+	}
+	if deps.Records == nil {
+		return nil
+	}
+	owed, err := deps.Records.LiveWriteBack(ctx, id)
+	if err != nil && !errors.Is(err, uefistore.ErrNotLive) {
+		return fmt.Errorf("read the UEFI store record of %s: %w", id, err)
+	}
+	if owed == nil || owed.Node == m.deps.NodeID {
+		return nil
+	}
+	if deps.NodeIsMember != nil && !deps.NodeIsMember(owed.Node) {
+		return nil
+	}
+	slog.WarnContext(ctx, "Refusing to start a UEFI instance while another node owes its variable store write-back",
+		"instanceId", id, "node", owed.Node, "key", owed.Key, "since", owed.Since)
+	return fmt.Errorf("%w: %s owes %s", ErrUEFIWriteBackPending, owed.Node, id)
+}
+
+// clearForeignUEFIWriteBack drops the obligation of a node that has left the
+// cluster once this node has started the instance from the pointer: that node's
+// changes since launch are lost with it.
+func (m *Manager) clearForeignUEFIWriteBack(ctx context.Context, id string) {
+	records := m.deps.UEFIVarStore.Records
+	if records == nil {
+		return
+	}
+	owed, err := records.LiveWriteBack(ctx, id)
+	if err != nil || owed == nil || owed.Node == m.deps.NodeID {
+		return
+	}
+	if err := records.ClearWriteBack(ctx, id, owed.Key); err != nil {
+		slog.WarnContext(ctx, "Could not clear another node's UEFI write-back obligation", "instanceId", id, "node", owed.Node, "err", err)
+		return
+	}
+	slog.InfoContext(ctx, "Cleared another node's UEFI write-back obligation; started from the pointer", "instanceId", id,
+		"node", owed.Node, "key", owed.Key, "since", owed.Since)
+}
+
+// readUEFIOwed returns the obligation this node kept in dir because the store
+// record could not take it, or nil when none is kept.
+func readUEFIOwed(dir string) (*uefistore.UEFIWriteBack, error) {
+	data, err := os.ReadFile(filepath.Join(dir, uefiOwedFileName))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var owed uefistore.UEFIWriteBack
+	if err := json.Unmarshal(data, &owed); err != nil {
+		return nil, fmt.Errorf("decode %s: %w", uefiOwedFileName, err)
+	}
+	if owed.Key == "" || owed.SeededFrom == "" {
+		return nil, fmt.Errorf("%s names no key", uefiOwedFileName)
+	}
+	return &owed, nil
+}
+
+// writeUEFIOwed keeps owed in dir for this node's reaper.
+func writeUEFIOwed(dir string, owed uefistore.UEFIWriteBack) error {
+	data, err := json.Marshal(owed)
+	if err != nil {
+		return err
+	}
+	return writeFileAtomic(dir, uefiOwedFileName, data)
+}
+
+// removeUEFIOwed drops the obligation kept in dir, once settled or superseded.
+func removeUEFIOwed(ctx context.Context, id, dir string) {
+	if err := os.Remove(filepath.Join(dir, uefiOwedFileName)); err != nil && !errors.Is(err, os.ErrNotExist) {
+		slog.WarnContext(ctx, "Could not remove a settled UEFI write-back kept on this node", "instanceId", id, "err", err)
+	}
+}
+
+// readUEFISeed returns the key seeded.json in dir names.
+func readUEFISeed(dir string) (string, error) {
+	data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
+	if err != nil {
+		return "", err
+	}
+	var seed uefiSeed
+	if err := json.Unmarshal(data, &seed); err != nil {
+		return "", fmt.Errorf("decode %s: %w", uefiSeededFileName, err)
+	}
+	if seed.Key == "" {
+		return "", fmt.Errorf("%s names no key", uefiSeededFileName)
+	}
+	return seed.Key, nil
+}
+
+// writeUEFISeed records that vars.fd in dir holds the store at key.
+func writeUEFISeed(dir, key string) error {
+	seed, err := json.Marshal(uefiSeed{Key: key, SeededAt: time.Now().UTC()})
+	if err != nil {
+		return err
+	}
+	return writeFileAtomic(dir, uefiSeededFileName, seed)
+}
+
+func skipUEFIWriteBack(ctx context.Context, id, reason, why string, attrs ...any) {
+	uefiStoreMetrics()
+	if uefiWriteBackSkips != nil {
+		uefiWriteBackSkips.Add(context.Background(), 1, metric.WithAttributes(attribute.String("reason", reason)))
+	}
+	slog.InfoContext(ctx, "Skipping UEFI variable store write-back: "+why, append([]any{"instanceId", id, "reason", reason}, attrs...)...)
+}
+
+func recordUEFIWriteBackOutcome(outcome string) {
+	uefiStoreMetrics()
+	if uefiWriteBacks != nil {
+		uefiWriteBacks.Add(context.Background(), 1, metric.WithAttributes(attribute.String("outcome", outcome)))
+	}
+}

spinifex/runtime/compute/vm/uefi_writeback_invariants_test.go

@@ -0,0 +1,270 @@
+package vm
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// s7Clause is quoted by every failure here: pruning and the write-back reaper
+// delete customer data on their own, which the clause allows only for an
+// object no record points at.
+const s7Clause = "ADR-0003:S7: a reaper's judgement is never the decision that deletes customer data; " +
+	"the pointed store survives every write-back and reap"
+
+// A copy that is fenced, errored or superseded writes nothing back: no object
+// is put, the pointer every record carries stays where it was, no obligation
+// is recorded, and each skip is counted under its own reason.
+func TestS7_UEFISkippedWriteBackNeverMovesThePointer(t *testing.T) {
+	tests := []struct {
+		name   string
+		reason string
+		setup  func(t *testing.T, c *wbCluster, a *wbNode, v *VM)
+		stop   func(t *testing.T, a *wbNode, v *VM)
+	}{
+		{
+			name: "the record names another node", reason: "superseded",
+			setup: func(_ *testing.T, c *wbCluster, _ *wbNode, v *VM) { c.pointers.run(nodeB, v) },
+		},
+		{
+			name: "seeded.json names an older pointer", reason: "reseeded",
+			setup: func(t *testing.T, _ *wbCluster, a *wbNode, _ *VM) {
+				require.NoError(t, writeUEFISeed(a.dir(), keyFor("older")))
+			},
+		},
+		{
+			name: "seeded.json names an older pointer and the record is unreadable", reason: "reseeded",
+			setup: func(t *testing.T, c *wbCluster, a *wbNode, _ *VM) {
+				require.NoError(t, writeUEFISeed(a.dir(), keyFor("older")))
+				c.pointers.loadErr = errors.New("nats: no responders")
+			},
+		},
+		{
+			name: "the record moved on after this node seeded", reason: "reseeded",
+			setup: func(_ *testing.T, c *wbCluster, _ *wbNode, v *VM) {
+				moved := &VM{ID: v.ID, UEFIVarStore: &UEFIVarStoreRef{Key: keyFor("restored")}}
+				c.pointers.run(nodeA, moved)
+			},
+		},
+		{
+			name: "no record of what this node seeded from", reason: "unseeded",
+			setup: func(t *testing.T, _ *wbCluster, a *wbNode, _ *VM) {
+				require.NoError(t, os.Remove(filepath.Join(a.dir(), uefiSeededFileName)))
+			},
+		},
+		{
+			name: "the store record is deleting", reason: "store_not_live",
+			setup: func(t *testing.T, c *wbCluster, _ *wbNode, _ *VM) {
+				rec, rev, err := c.records.Get(t.Context(), testUEFIInstance)
+				require.NoError(t, err)
+				rec.State = uefistore.StateDeleting
+				_, err = c.records.CompareAndSet(t.Context(), rec, rev)
+				require.NoError(t, err)
+			},
+		},
+		{
+			name: "a fence stopped it", reason: "fenced",
+			setup: func(_ *testing.T, _ *wbCluster, a *wbNode, v *VM) {
+				a.m.Inspect(v, func(v *VM) {
+					v.Instance.StateReason = &ec2.StateReason{Code: aws.String(fenceStateReasonCode), Message: aws.String("lease moved")}
+				})
+			},
+		},
+		{
+			name: "recovery left it in error", reason: "error",
+			stop: func(_ *testing.T, a *wbNode, v *VM) {
+				a.m.MarkRecoveryFailed(v, "relaunch failed")
+				a.m.goroutineWg.Wait()
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newWBCluster(t)
+			a := c.node(t, nodeA)
+			v := a.newInstance(t, region("stale copy"))
+			if tt.setup != nil {
+				tt.setup(t, c, a, v)
+			}
+			before := uefiCounter(t, "mulga.vm.uefi_store.write_back_skips", "reason", tt.reason)
+			puts := c.objects.putCount()
+
+			if tt.stop != nil {
+				tt.stop(t, a, v)
+			} else {
+				a.stop(t, v)
+			}
+
+			assert.Equal(t, puts, c.objects.putCount(), "%s: a skipped copy puts nothing", s7Clause)
+			assert.Equal(t, []string{c.seedKey}, c.keys(t), s7Clause)
+			var pointer string
+			a.m.Inspect(v, func(v *VM) { pointer = v.UEFIVarStore.Key })
+			assert.Equal(t, c.seedKey, pointer, "%s: the in-memory pointer, which every record write carries, is unchanged", s7Clause)
+			if stopped, _ := c.states.LoadStoppedInstance(v.ID); stopped != nil {
+				assert.Equal(t, c.seedKey, stopped.UEFIVarStore.Key, s7Clause)
+			}
+			if tt.reason != "store_not_live" {
+				assert.Nil(t, c.owed(t), "a skip records no obligation")
+			}
+			assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.write_back_skips", "reason", tt.reason))
+		})
+	}
+}
+
+// The reaper deletes only the object its own obligation put, and only once
+// no record points at it: the pointed object and the version it seeded from
+// survive every branch.
+func TestS7_UEFIWriteBackReaperNeverDeletesThePointedObject(t *testing.T) {
+	tests := []struct {
+		name        string
+		pointer     func(c *wbCluster, owed string) string
+		lastNode    string
+		wantDeleted bool
+	}{
+		{name: "the record already points at the obligation's key", pointer: func(_ *wbCluster, owed string) string { return owed }, lastNode: nodeA},
+		{name: "an operator restored an older version", pointer: func(*wbCluster, string) string { return keyFor("restored") }, lastNode: nodeA, wantDeleted: true},
+		{name: "another node owns the record", pointer: func(c *wbCluster, _ string) string { return c.seedKey }, lastNode: nodeB, wantDeleted: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newWBCluster(t)
+			a := c.node(t, nodeA)
+			c.put(t, keyFor("restored"), region("restored"))
+			v := a.newInstance(t, region("written while kv was down"))
+			c.states.stoppedErr = assert.AnError
+			a.stop(t, v)
+			c.states.stoppedErr = nil
+			owed := c.owed(t)
+			require.NotNil(t, owed)
+			require.True(t, c.exists(t, owed.Key), "the put landed before the record write failed")
+
+			pointer := tt.pointer(c, owed.Key)
+			a.m.Delete(v.ID)
+			v.LastNode = tt.lastNode
+			v.UEFIVarStore = &UEFIVarStoreRef{Key: pointer, Firmware: c.identity}
+			require.NoError(t, c.states.WriteStoppedInstance(v.ID, v))
+
+			assert.Equal(t, 1, a.sweep(t))
+
+			assert.Nil(t, c.owed(t))
+			assert.True(t, c.exists(t, pointer), "%s: the pointed object is never deleted", s7Clause)
+			assert.True(t, c.exists(t, c.seedKey), "%s: the version it seeded from is never deleted", s7Clause)
+			assert.Equal(t, !tt.wantDeleted, c.exists(t, owed.Key), "the obligation's own unpointed object is the only one deleted")
+			assert.Equal(t, pointer, c.stopped(t).UEFIVarStore.Key, "%s: a dropped obligation never moves the pointer", s7Clause)
+		})
+	}
+}
+
+// A stop whose obligation the store record cannot take still completes, and
+// the stopped record names the node holding the newer variables. A start on
+// another member waits; this node's reaper lands them after a restart.
+func TestS7_UEFIUnrecordableWriteBackStaysOwedByTheStoppingNode(t *testing.T) {
+	tests := []struct {
+		name string
+		fail func(c *wbCluster)
+	}{
+		{
+			name: "the store record is unreadable",
+			fail: func(c *wbCluster) { c.reach.fail(errors.New("nats: no responders"), nil) },
+		},
+		{
+			name: "the put fails and so does the obligation write",
+			fail: func(c *wbCluster) {
+				c.objects.setPutErr(errPredastoreDown)
+				c.reach.fail(nil, errors.New("nats: timeout"))
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newWBCluster(t)
+			c.members[nodeA], c.members[nodeB] = true, true
+			a, b := c.node(t, nodeA), c.node(t, nodeB)
+			v := a.newInstance(t, region("set on node-a"))
+			tt.fail(c)
+
+			a.stop(t, v)
+
+			rec := c.stopped(t)
+			require.NotNil(t, rec.UEFIWriteBackOwed, "%s: the stopped record must name the node holding newer variables", s7Clause)
+			assert.Equal(t, nodeA, rec.UEFIWriteBackOwed.Node)
+			assert.Equal(t, c.seedKey, rec.UEFIVarStore.Key)
+			kept := filepath.Join(a.dir(), uefiOwedFileName)
+			require.FileExists(t, kept, "the obligation is kept where this node's reaper finds it")
+
+			c.reach.fail(nil, nil)
+			c.objects.setPutErr(nil)
+			require.Nil(t, c.owed(t), "the store record holds no obligation, so only the marker refuses")
+			err := b.m.prepareUEFIVarStore(t.Context(), c.stopped(t))
+			require.ErrorIs(t, err, ErrUEFIWriteBackPending, "%s: a start elsewhere must not boot the older pointer", s7Clause)
+
+			restarted := c.nodeAt(t, nodeA, a.dataDir)
+			assert.Equal(t, 1, restarted.sweep(t))
+
+			rec = c.stopped(t)
+			assert.Nil(t, rec.UEFIWriteBackOwed, "the landed write-back clears the marker")
+			assert.NotEqual(t, c.seedKey, rec.UEFIVarStore.Key)
+			assert.NoFileExists(t, kept)
+			assert.Nil(t, c.owed(t))
+
+			require.NoError(t, c.states.DeleteStoppedInstance(rec.ID))
+			rec.Status, rec.DesiredState = StateRunning, DesiredRunning
+			b.m.Insert(rec)
+			c.pointers.run(nodeB, rec)
+			require.NoError(t, b.m.prepareUEFIVarStore(t.Context(), rec))
+			assert.Equal(t, region("set on node-a"), readFile(t, filepath.Join(b.dir(), uefiVarsFileName)),
+				"node-b boots the variables node-a wrote back")
+		})
+	}
+}
+
+// A marker left by a node that has since left the cluster does not hold a
+// start: the start boots the pointer and clears it.
+func TestUEFIWriteBack_MarkerOfANodeThatLeftDoesNotHoldAStart(t *testing.T) {
+	c := newWBCluster(t)
+	a, b := c.node(t, nodeA), c.node(t, nodeB)
+	v := a.newInstance(t, region("lost with node-a"))
+	c.reach.fail(errors.New("nats: no responders"), nil)
+	a.stop(t, v)
+	c.reach.fail(nil, nil)
+	require.NotNil(t, c.stopped(t).UEFIWriteBackOwed)
+
+	started := b.restart(t, region("template"))
+
+	assert.Equal(t, region("template"), readFile(t, filepath.Join(b.dir(), uefiVarsFileName)))
+	assert.Nil(t, started.UEFIWriteBackOwed, "the start clears the departed node's marker")
+}
+
+// A stop never lists or deletes old versions, however many are retained: the
+// reaper prunes them afterwards, keeping the pointer and the version replaced.
+func TestS8_UEFIStopLeavesPruningToTheReaper(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	for i := range 6 {
+		c.put(t, keyFor(fmt.Sprintf("old-%d", i)), region("old"))
+	}
+	v := a.newInstance(t, region("newest"))
+	lists, deletes := c.objects.listsAndDeletes()
+
+	a.stop(t, v)
+
+	gotLists, gotDeletes := c.objects.listsAndDeletes()
+	assert.Equal(t, lists, gotLists, "ADR-0003:S8: a stop must not list work that scales with retained history")
+	assert.Equal(t, deletes, gotDeletes, "ADR-0003:S8: a stop must not delete retained history")
+	pointer := c.stopped(t).UEFIVarStore.Key
+	require.Len(t, c.keys(t), 8)
+
+	assert.Zero(t, a.sweep(t))
+
+	assert.LessOrEqual(t, len(c.keys(t)), uefistore.RetainedVersions+1, "the reaper prunes to the newest versions: %v", c.keys(t))
+	assert.True(t, c.exists(t, pointer), "%s: the pointed object is never pruned", s7Clause)
+	assert.True(t, c.exists(t, c.seedKey), "%s: the version it replaced is never pruned", s7Clause)
+}

spinifex/runtime/compute/vm/uefi_writeback_reaper.go

@@ -0,0 +1,435 @@
+package vm
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+)
+
+// uefiWriteBackReaper retries this node's owed variable store write-backs
+// from the copy each stop kept, putting to the key its obligation recorded.
+// It moves a pointer only while the instance is still this node's and stopped.
+type uefiWriteBackReaper struct {
+	m *Manager
+}
+
+var _ Reaper = (*uefiWriteBackReaper)(nil)
+
+// NewUEFIWriteBackReaper builds the reaper over this Manager's variable store
+// dependencies.
+func (m *Manager) NewUEFIWriteBackReaper() Reaper {
+	return &uefiWriteBackReaper{m: m}
+}
+
+func (r *uefiWriteBackReaper) Class() string { return "uefi-write-back" }
+
+func (r *uefiWriteBackReaper) Scope() ReaperScope { return ScopeNodeLocal }
+
+// Sweep records the obligations this node kept on disk, settles each it owes,
+// reporting how many it settled, then prunes the prefixes due a prune. One that
+// cannot be decided yet stays for the next sweep.
+func (r *uefiWriteBackReaper) Sweep(ctx context.Context) (int, error) {
+	m := r.m
+	deps := m.deps.UEFIVarStore
+	if !deps.writeBackWired() || m.deps.NodeID == "" {
+		return 0, nil
+	}
+	var errs []error
+	for _, id := range m.keptUEFIWriteBacks(ctx) {
+		if err := m.recordKeptUEFIWriteBack(ctx, id); err != nil {
+			slog.WarnContext(ctx, "Could not record a UEFI write-back kept on this node; retrying next sweep", "instanceId", id, "err", err)
+			errs = append(errs, err)
+		}
+	}
+	settled := 0
+	ids, err := deps.Records.WriteBacksOwedBy(ctx, m.deps.NodeID)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	for _, id := range ids {
+		done, err := m.retryUEFIWriteBack(ctx, id)
+		if err != nil {
+			slog.WarnContext(ctx, "UEFI write-back retry did not settle; retrying next sweep", "instanceId", id, "err", err)
+			errs = append(errs, err)
+		}
+		if done {
+			settled++
+		}
+	}
+	if err := m.pruneDueUEFIStores(ctx); err != nil {
+		errs = append(errs, err)
+	}
+	return settled, errors.Join(errs...)
+}
+
+// keptUEFIWriteBacks lists the instances whose directory here keeps an
+// obligation the store record could not take.
+func (m *Manager) keptUEFIWriteBacks(ctx context.Context) []string {
+	root := filepath.Join(m.deps.UEFIVarStore.DataDir, uefiStoreDirName)
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			slog.WarnContext(ctx, "Could not list this node's UEFI variable stores", "dir", root, "err", err)
+		}
+		return nil
+	}
+	var ids []string
+	for _, e := range entries {
+		if !e.IsDir() || !strings.HasPrefix(e.Name(), "i-") {
+			continue
+		}
+		if _, err := os.Stat(filepath.Join(root, e.Name(), uefiOwedFileName)); err == nil {
+			ids = append(ids, e.Name())
+		}
+	}
+	return ids
+}
+
+// recordKeptUEFIWriteBack moves the obligation kept in id's directory onto the
+// store record, where retryUEFIWriteBack settles it. An obligation the record
+// already holds supersedes it, and a store no longer live owes nothing.
+func (m *Manager) recordKeptUEFIWriteBack(ctx context.Context, id string) error {
+	unlock := m.uefiLocks.lock(id)
+	defer unlock()
+
+	records := m.deps.UEFIVarStore.Records
+	dir, err := uefiInstanceDir(m.deps.UEFIVarStore.DataDir, id)
+	if err != nil {
+		return err
+	}
+	kept, err := readUEFIOwed(dir)
+	if err != nil || kept == nil {
+		return err
+	}
+	cur, err := records.LiveWriteBack(ctx, id)
+	if err == nil && cur == nil {
+		err = records.RecordWriteBack(ctx, id, *kept)
+	}
+	switch {
+	case errors.Is(err, uefistore.ErrNotLive):
+		m.deleteUEFIObject(ctx, id, kept.Key)
+		recordUEFIWriteBackOutcome("dropped")
+	case err != nil:
+		return err
+	case cur != nil:
+		slog.InfoContext(ctx, "A later UEFI write-back obligation supersedes the one kept on this node", "instanceId", id,
+			"key", kept.Key, "owedBy", cur.Node, "owedKey", cur.Key)
+	default:
+		slog.InfoContext(ctx, "Recorded a UEFI write-back kept on this node", "instanceId", id, "key", kept.Key, "since", kept.Since)
+	}
+	removeUEFIOwed(ctx, id, dir)
+	return nil
+}
+
+// retryUEFIWriteBack settles id's obligation if it is still this node's: it is
+// cleared when the record already points at its key, written back while the
+// record is stopped here at the pointer it seeded from, and dropped otherwise.
+func (m *Manager) retryUEFIWriteBack(ctx context.Context, id string) (bool, error) {
+	unlock := m.uefiLocks.lock(id)
+	defer unlock()
+
+	deps := m.deps.UEFIVarStore
+	owed, err := deps.Records.LiveWriteBack(ctx, id)
+	if errors.Is(err, uefistore.ErrNotLive) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if owed == nil || owed.Node != m.deps.NodeID {
+		return false, nil
+	}
+	dir, err := uefiInstanceDir(deps.DataDir, id)
+	if err != nil {
+		return false, err
+	}
+
+	// An instance in this node's running set has its record written from
+	// memory, so its pointer is moved there rather than in KV.
+	if local, ok := m.Get(id); ok {
+		return m.retryLocalUEFIWriteBack(ctx, local, dir, *owed)
+	}
+
+	owner, found, err := deps.Instances.LoadUEFIVarStoreOwner(id)
+	if err != nil {
+		return false, err
+	}
+	var pointer string
+	if found && owner.Pointer != nil {
+		pointer = owner.Pointer.Key
+	}
+	// Settled without a put, the obligation first leaves the record's marker.
+	unmark := func() error {
+		if !found || owner.Owed == nil || owner.Owed.Node != m.deps.NodeID {
+			return nil
+		}
+		return deps.Instances.CompareAndSetUEFIVarStore(id, owner.Pointer, owner.Revision)
+	}
+	switch {
+	case pointer == owed.Key:
+		// An earlier record write landed without its reply.
+		if err := unmark(); err != nil {
+			return false, err
+		}
+		return m.landedUEFIWriteBack(ctx, id, dir, *owed)
+	case !found || owner.Status != StateStopped || owner.LastNode != m.deps.NodeID || pointer != owed.SeededFrom:
+		if err := unmark(); err != nil {
+			return false, err
+		}
+		return m.dropUEFIWriteBack(ctx, id, *owed, pointer, "the record is no longer stopped here at the pointer this node seeded from")
+	}
+	if seed, err := readUEFISeed(dir); err != nil || seed != owed.SeededFrom {
+		if err := unmark(); err != nil {
+			return false, err
+		}
+		return m.dropUEFIWriteBack(ctx, id, *owed, pointer, "this node's copy is no longer the one seeded from the pointer")
+	}
+
+	written, err := m.putUEFIVars(ctx, id, dir, owed.Key, owner.Pointer.Firmware.Arch)
+	if err != nil {
+		return false, err
+	}
+	if settled, err := m.confirmUEFIWriteBackOwed(ctx, id, *owed); !settled {
+		return false, err
+	}
+	if err := deps.Instances.CompareAndSetUEFIVarStore(id, written, owner.Revision); err != nil {
+		return false, err
+	}
+	m.finishUEFIWriteBack(ctx, id, dir, owed.Key, owed.SeededFrom, owed.Key)
+	recordUEFIWriteBackOutcome("retried")
+	slog.InfoContext(ctx, "Wrote an owed UEFI variable store back", "instanceId", id, "key", owed.Key, "since", owed.Since)
+	return true, nil
+}
+
+// retryLocalUEFIWriteBack is retryUEFIWriteBack for an instance this node's
+// running set holds, as a drain-stopped one is until the node restarts.
+func (m *Manager) retryLocalUEFIWriteBack(ctx context.Context, local *VM, dir string, owed uefistore.UEFIWriteBack) (bool, error) {
+	id := local.ID
+	var status InstanceState
+	var previous *UEFIVarStoreRef
+	var marker *UEFIWriteBackOwed
+	m.Inspect(local, func(v *VM) {
+		status = v.Status
+		marker = v.UEFIWriteBackOwed
+		if v.UEFIVarStore != nil {
+			cp := *v.UEFIVarStore
+			previous = &cp
+		}
+	})
+	var pointer string
+	if previous != nil {
+		pointer = previous.Key
+	}
+	switch {
+	case pointer == owed.Key:
+		if err := m.unmarkLocalUEFIWriteBack(local, marker); err != nil {
+			return false, err
+		}
+		return m.landedUEFIWriteBack(ctx, id, dir, owed)
+	case status != StateStopped || pointer != owed.SeededFrom:
+		if err := m.unmarkLocalUEFIWriteBack(local, marker); err != nil {
+			return false, err
+		}
+		return m.dropUEFIWriteBack(ctx, id, owed, pointer, "this node's instance has moved on from the pointer it seeded from")
+	}
+	if seed, err := readUEFISeed(dir); err != nil || seed != owed.SeededFrom {
+		if err := m.unmarkLocalUEFIWriteBack(local, marker); err != nil {
+			return false, err
+		}
+		return m.dropUEFIWriteBack(ctx, id, owed, pointer, "this node's copy is no longer the one seeded from the pointer")
+	}
+
+	written, err := m.putUEFIVars(ctx, id, dir, owed.Key, previous.Firmware.Arch)
+	if err != nil {
+		return false, err
+	}
+	if settled, err := m.confirmUEFIWriteBackOwed(ctx, id, owed); !settled {
+		return false, err
+	}
+	m.Inspect(local, func(v *VM) {
+		v.UEFIVarStore = written
+		v.UEFIWriteBackOwed = nil
+	})
+	if err := m.writeRunningState(); err != nil {
+		m.Inspect(local, func(v *VM) {
+			if v.UEFIVarStore != nil && v.UEFIVarStore.Key == owed.Key {
+				v.UEFIVarStore = previous
+				v.UEFIWriteBackOwed = marker
+			}
+		})
+		return false, err
+	}
+	m.finishUEFIWriteBack(ctx, id, dir, owed.Key, owed.SeededFrom, owed.Key)
+	recordUEFIWriteBackOutcome("retried")
+	slog.InfoContext(ctx, "Wrote an owed UEFI variable store back", "instanceId", id, "key", owed.Key, "since", owed.Since)
+	return true, nil
+}
+
+// confirmUEFIWriteBackOwed re-reads the obligation after a retry's put. A store
+// gone or deleting, or an obligation cleared meanwhile, leaves the put
+// pointed at by nothing, so it is deleted and the retry stops.
+func (m *Manager) confirmUEFIWriteBackOwed(ctx context.Context, id string, owed uefistore.UEFIWriteBack) (bool, error) {
+	cur, err := m.deps.UEFIVarStore.Records.LiveWriteBack(ctx, id)
+	switch {
+	case errors.Is(err, uefistore.ErrNotLive):
+		m.deleteUEFIObject(ctx, id, owed.Key)
+		recordUEFIWriteBackOutcome("dropped")
+		return false, nil
+	case err != nil:
+		return false, err
+	case cur == nil || cur.Key != owed.Key || cur.Node != owed.Node:
+		m.deleteUEFIObject(ctx, id, owed.Key)
+		recordUEFIWriteBackOutcome("dropped")
+		return false, nil
+	}
+	return true, nil
+}
+
+// landedUEFIWriteBack clears an obligation whose key the record already
+// points at, and lets a start here reuse the copy that key holds.
+func (m *Manager) landedUEFIWriteBack(ctx context.Context, id, dir string, owed uefistore.UEFIWriteBack) (bool, error) {
+	if seed, err := readUEFISeed(dir); err != nil || seed != owed.SeededFrom {
+		// The copy here is not the one that was put, so seeded.json keeps
+		// whatever it names and only the obligation is settled.
+		if err := m.deps.UEFIVarStore.Records.ClearWriteBack(ctx, id, owed.Key); err != nil && !errors.Is(err, uefistore.ErrNotLive) {
+			return false, err
+		}
+		recordUEFIWriteBackOutcome("landed")
+		return true, nil
+	}
+	m.finishUEFIWriteBack(ctx, id, dir, owed.Key, owed.SeededFrom, owed.Key)
+	recordUEFIWriteBackOutcome("landed")
+	return true, nil
+}
+
+// dropUEFIWriteBack clears a superseded obligation, then deletes the object
+// at its key, which no record points at. pointer is the record's current key.
+func (m *Manager) dropUEFIWriteBack(ctx context.Context, id string, owed uefistore.UEFIWriteBack, pointer, why string) (bool, error) {
+	err := m.deps.UEFIVarStore.Records.ClearWriteBack(ctx, id, owed.Key)
+	if errors.Is(err, uefistore.ErrNotLive) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if pointer != owed.Key {
+		m.deleteUEFIObject(ctx, id, owed.Key)
+	}
+	recordUEFIWriteBackOutcome("superseded")
+	slog.InfoContext(ctx, "Dropped a superseded UEFI write-back obligation: "+why, "instanceId", id,
+		"key", owed.Key, "seededFrom", owed.SeededFrom, "pointer", pointer)
+	return true, nil
+}
+
+// unmarkLocalUEFIWriteBack clears this node's marker from an instance in the
+// running set and writes the set, restoring the marker if the write fails.
+func (m *Manager) unmarkLocalUEFIWriteBack(local *VM, marker *UEFIWriteBackOwed) error {
+	if marker == nil || marker.Node != m.deps.NodeID {
+		return nil
+	}
+	m.Inspect(local, func(v *VM) { v.UEFIWriteBackOwed = nil })
+	if err := m.writeRunningState(); err != nil {
+		m.Inspect(local, func(v *VM) {
+			if v.UEFIWriteBackOwed == nil {
+				v.UEFIWriteBackOwed = marker
+			}
+		})
+		return err
+	}
+	return nil
+}
+
+// uefiPrune is an instance prefix due a prune after a write-back landed at key.
+type uefiPrune struct {
+	key, seededFrom string
+}
+
+// uefiPruneSet holds the prefixes due a prune off the stop path. It is in
+// memory only: a prune lost to a restart is redone after the next write-back.
+type uefiPruneSet struct {
+	mu  sync.Mutex
+	due map[string]uefiPrune
+}
+
+// add queues id's prefix, replacing an older entry for it.
+func (s *uefiPruneSet) add(id string, p uefiPrune) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.due == nil {
+		s.due = map[string]uefiPrune{}
+	}
+	s.due[id] = p
+}
+
+// requeue puts back a prune that failed unless a newer one was queued since.
+func (s *uefiPruneSet) requeue(id string, p uefiPrune) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if _, ok := s.due[id]; ok {
+		return
+	}
+	if s.due == nil {
+		s.due = map[string]uefiPrune{}
+	}
+	s.due[id] = p
+}
+
+func (s *uefiPruneSet) take() map[string]uefiPrune {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	due := s.due
+	s.due = nil
+	return due
+}
+
+// pruneDueUEFIStores prunes each queued prefix, putting back one that fails.
+func (m *Manager) pruneDueUEFIStores(ctx context.Context) error {
+	var errs []error
+	for id, p := range m.uefiPrunes.take() {
+		if err := m.pruneUEFIStore(ctx, id, p); err != nil {
+			slog.WarnContext(ctx, "Could not prune old UEFI variable store versions; retrying next sweep", "instanceId", id, "err", err)
+			m.uefiPrunes.requeue(id, p)
+			errs = append(errs, err)
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// pruneUEFIStore prunes id's prefix, keeping the key written, the version it
+// replaced and the record's pointer as read now, which a restore may have moved.
+func (m *Manager) pruneUEFIStore(ctx context.Context, id string, p uefiPrune) error {
+	unlock := m.uefiLocks.lock(id)
+	defer unlock()
+
+	deps := m.deps.UEFIVarStore
+	keep := []string{p.key, p.seededFrom}
+	if local, ok := m.Get(id); ok {
+		m.Inspect(local, func(v *VM) {
+			if v.UEFIVarStore != nil {
+				keep = append(keep, v.UEFIVarStore.Key)
+			}
+		})
+	} else {
+		owner, found, err := deps.Instances.LoadUEFIVarStoreOwner(id)
+		if err != nil {
+			return err
+		}
+		if !found {
+			return nil
+		}
+		if owner.Pointer != nil {
+			keep = append(keep, owner.Pointer.Key)
+		}
+	}
+	deleted, err := uefistore.Prune(ctx, deps.Objects, deps.Bucket, id, keep...)
+	if len(deleted) > 0 {
+		slog.InfoContext(ctx, "Pruned old UEFI variable store versions", "instanceId", id, "deleted", deleted, "pointer", p.key)
+	}
+	return err
+}

spinifex/runtime/compute/vm/uefi_writeback_test.go

@@ -0,0 +1,783 @@
+package vm
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	nodeA = "node-a"
+	nodeB = "node-b"
+)
+
+var errPredastoreDown = errors.New("dial tcp: connection refused")
+
+// flakyObjects is the object store with puts that can be failed, the way an
+// unreachable predastore fails them, and a hook run after each put lands.
+type flakyObjects struct {
+	*objectstore.MemoryObjectStore
+
+	mu       sync.Mutex
+	putErr   error
+	afterPut func(key string)
+	puts     int
+	lists    int
+	deletes  int
+	meta     map[string]map[string]string
+}
+
+func (o *flakyObjects) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	o.mu.Lock()
+	o.lists++
+	o.mu.Unlock()
+	return o.MemoryObjectStore.ListObjectsV2(ctx, in)
+}
+
+func (o *flakyObjects) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	o.mu.Lock()
+	o.deletes++
+	o.mu.Unlock()
+	return o.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// listsAndDeletes counts the calls a prune makes.
+func (o *flakyObjects) listsAndDeletes() (int, int) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	return o.lists, o.deletes
+}
+
+func (o *flakyObjects) PutObject(ctx context.Context, in *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	o.mu.Lock()
+	err, hook := o.putErr, o.afterPut
+	o.puts++
+	o.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	out, err := o.MemoryObjectStore.PutObject(ctx, in)
+	if err != nil {
+		return out, err
+	}
+	o.mu.Lock()
+	if o.meta == nil {
+		o.meta = map[string]map[string]string{}
+	}
+	o.meta[aws.StringValue(in.Key)] = aws.StringValueMap(in.Metadata)
+	o.mu.Unlock()
+	if hook != nil {
+		hook(aws.StringValue(in.Key))
+	}
+	return out, err
+}
+
+func (o *flakyObjects) putMetadata(key string) map[string]string {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	return o.meta[key]
+}
+
+func (o *flakyObjects) putCount() int {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	return o.puts
+}
+
+func (o *flakyObjects) setPutErr(err error) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.putErr = err
+}
+
+// flakyRecords is the store-record bucket as a node reaches it: down fails
+// every call, the way an unreachable bucket does, and recordErr fails only the
+// write of an obligation.
+type flakyRecords struct {
+	*uefistore.Store
+
+	mu        sync.Mutex
+	down      error
+	recordErr error
+}
+
+var _ UEFIStoreRecords = (*flakyRecords)(nil)
+
+func (r *flakyRecords) fail(down, recordErr error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.down, r.recordErr = down, recordErr
+}
+
+func (r *flakyRecords) errs() (error, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.down, r.recordErr
+}
+
+func (r *flakyRecords) LiveWriteBack(ctx context.Context, id string) (*uefistore.UEFIWriteBack, error) {
+	if down, _ := r.errs(); down != nil {
+		return nil, down
+	}
+	return r.Store.LiveWriteBack(ctx, id)
+}
+
+func (r *flakyRecords) RecordWriteBack(ctx context.Context, id string, wb uefistore.UEFIWriteBack) error {
+	if down, recordErr := r.errs(); down != nil || recordErr != nil {
+		return errors.Join(down, recordErr)
+	}
+	return r.Store.RecordWriteBack(ctx, id, wb)
+}
+
+func (r *flakyRecords) ClearWriteBack(ctx context.Context, id, key string) error {
+	if down, _ := r.errs(); down != nil {
+		return down
+	}
+	return r.Store.ClearWriteBack(ctx, id, key)
+}
+
+func (r *flakyRecords) WriteBacksOwedBy(ctx context.Context, node string) ([]string, error) {
+	if down, _ := r.errs(); down != nil {
+		return nil, down
+	}
+	return r.Store.WriteBacksOwedBy(ctx, node)
+}
+
+// wbStateStore is fakeStateStore holding the stopped record as KV does, a
+// copy of the VM, and whose write can fail outright or land without a reply.
+type wbStateStore struct {
+	*fakeStateStore
+
+	stoppedErr   error
+	landThenFail bool
+}
+
+func (s *wbStateStore) WriteStoppedInstance(id string, v *VM) error {
+	if s.stoppedErr != nil {
+		return s.stoppedErr
+	}
+	if err := s.fakeStateStore.WriteStoppedInstance(id, VMFromRecord(v.Record())); err != nil {
+		return err
+	}
+	if s.landThenFail {
+		return errors.New("nats: timeout")
+	}
+	return nil
+}
+
+// fakeUEFIPointers answers for the cluster's instance records: a stopped one
+// from the state store, a running one from the node the test says runs it.
+type fakeUEFIPointers struct {
+	mu      sync.Mutex
+	states  *wbStateStore
+	running map[string]*VM
+	nodes   map[string]string
+	revs    map[string]uint64
+	loadErr error
+}
+
+var _ UEFIInstancePointers = (*fakeUEFIPointers)(nil)
+
+func (p *fakeUEFIPointers) run(node string, v *VM) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.running[v.ID], p.nodes[v.ID] = v, node
+}
+
+func (p *fakeUEFIPointers) LoadUEFIVarStoreOwner(id string) (UEFIVarStoreOwner, bool, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.loadErr != nil {
+		return UEFIVarStoreOwner{}, false, p.loadErr
+	}
+	if v, _ := p.states.LoadStoppedInstance(id); v != nil {
+		return UEFIVarStoreOwner{Status: StateStopped, LastNode: v.LastNode, Pointer: v.UEFIVarStore, Owed: v.UEFIWriteBackOwed, Revision: p.revs[id]}, true, nil
+	}
+	if v, ok := p.running[id]; ok {
+		return UEFIVarStoreOwner{Status: StateRunning, LastNode: p.nodes[id], Pointer: v.UEFIVarStore, Owed: v.UEFIWriteBackOwed, Revision: p.revs[id]}, true, nil
+	}
+	return UEFIVarStoreOwner{}, false, nil
+}
+
+func (p *fakeUEFIPointers) CompareAndSetUEFIVarStore(id string, ref *UEFIVarStoreRef, rev uint64) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	v, _ := p.states.LoadStoppedInstance(id)
+	if v == nil || p.revs[id] != rev {
+		return fmt.Errorf("conflict on %s", id)
+	}
+	v.UEFIVarStore, v.UEFIWriteBackOwed = ref, nil
+	p.revs[id]++
+	return nil
+}
+
+// wbCluster is the shared side of a write-back: object store, store records,
+// instance records. Each node has its own manager and data directory.
+type wbCluster struct {
+	objects  *flakyObjects
+	records  *uefistore.Store
+	states   *wbStateStore
+	pointers *fakeUEFIPointers
+	identity uefistore.FirmwareIdentity
+	seedKey  string
+	// reach is the store-record bucket as every node reaches it.
+	reach *flakyRecords
+	// members are the nodes still in the cluster's configuration.
+	members map[string]bool
+}
+
+type wbNode struct {
+	c       *wbCluster
+	name    string
+	m       *Manager
+	dataDir string
+}
+
+func newWBCluster(t *testing.T) *wbCluster {
+	t.Helper()
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	codeBytes := []byte("ovmf code build 1")
+	require.NoError(t, os.WriteFile(code, codeBytes, 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, testUEFIVarsSize), 0o600))
+	orig := FirmwarePathCandidates
+	FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
+	t.Cleanup(func() { FirmwarePathCandidates = orig })
+	sum := sha256.Sum256(codeBytes)
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	states := &wbStateStore{fakeStateStore: newFakeStateStore()}
+	c := &wbCluster{
+		objects:  &flakyObjects{MemoryObjectStore: objectstore.NewMemoryObjectStore()},
+		records:  records,
+		reach:    &flakyRecords{Store: records},
+		states:   states,
+		pointers: &fakeUEFIPointers{states: states, running: map[string]*VM{}, nodes: map[string]string{}, revs: map[string]uint64{}},
+		identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
+		seedKey:  keyFor("seed"),
+		members:  map[string]bool{},
+	}
+	_, err := records.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: testUEFIInstance, AccountID: "111122223333", State: uefistore.StateLive})
+	require.NoError(t, err)
+	c.put(t, c.seedKey, region("template"))
+	return c
+}
+
+func (c *wbCluster) node(t *testing.T, name string) *wbNode {
+	t.Helper()
+	return c.nodeAt(t, name, t.TempDir())
+}
+
+// nodeAt builds name's manager over dataDir; over an existing node's directory
+// it is that node after a daemon restart.
+func (c *wbCluster) nodeAt(t *testing.T, name, dataDir string) *wbNode {
+	t.Helper()
+	n := &wbNode{c: c, name: name, dataDir: dataDir}
+	n.m = NewManagerWithDeps(Deps{
+		NodeID:        name,
+		StateStore:    c.states,
+		VolumeMounter: &fakeVolumeMounter{},
+		InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore: UEFIVarStoreDeps{
+			Objects: c.objects, Bucket: testUEFIBucket, DataDir: n.dataDir,
+			Records: c.reach, Instances: c.pointers,
+			NodeIsMember: func(node string) bool { return c.members[node] },
+		},
+	})
+	return n
+}
+
+func (c *wbCluster) put(t *testing.T, key string, vars []byte) {
+	t.Helper()
+	_, err := c.objects.MemoryObjectStore.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(testUEFIBucket), Key: aws.String(key), Body: bytes.NewReader(gzipped(t, vars)),
+	})
+	require.NoError(t, err)
+}
+
+// keys lists every object under the instance's prefix.
+func (c *wbCluster) keys(t *testing.T) []string {
+	t.Helper()
+	listed, _, err := objectstore.ListAll(t.Context(), c.objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(testUEFIBucket), Prefix: aws.String(uefistore.KeyPrefix(testUEFIInstance)),
+	})
+	require.NoError(t, err)
+	var keys []string
+	for _, o := range listed {
+		keys = append(keys, aws.StringValue(o.Key))
+	}
+	return keys
+}
+
+func (c *wbCluster) exists(t *testing.T, key string) bool {
+	t.Helper()
+	return slices.Contains(c.keys(t), key)
+}
+
+// region returns the decompressed region stored at key.
+func (c *wbCluster) region(t *testing.T, key string) []byte {
+	t.Helper()
+	out, err := c.objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testUEFIBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	zr, err := gzip.NewReader(out.Body)
+	require.NoError(t, err)
+	b, err := io.ReadAll(zr)
+	require.NoError(t, err)
+	return b
+}
+
+func (c *wbCluster) owed(t *testing.T) *uefistore.UEFIWriteBack {
+	t.Helper()
+	wb, err := c.records.LiveWriteBack(t.Context(), testUEFIInstance)
+	require.NoError(t, err)
+	return wb
+}
+
+func (c *wbCluster) stopped(t *testing.T) *VM {
+	t.Helper()
+	v, err := c.states.LoadStoppedInstance(testUEFIInstance)
+	require.NoError(t, err)
+	require.NotNil(t, v, "the stop must have written the stopped record")
+	return v
+}
+
+// launch starts the instance on n from pointer ref, as a start does: the
+// store is seeded or reused, then the guest writes vars.
+func (n *wbNode) launch(t *testing.T, v *VM, vars []byte) *VM {
+	t.Helper()
+	v.Status, v.DesiredState = StateRunning, DesiredRunning
+	n.m.Insert(v)
+	n.c.pointers.run(n.name, v)
+	require.NoError(t, n.m.prepareUEFIVarStore(t.Context(), v))
+	n.guestWrites(t, vars)
+	return v
+}
+
+func (n *wbNode) newInstance(t *testing.T, vars []byte) *VM {
+	t.Helper()
+	v := &VM{
+		ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Instance: &ec2.Instance{},
+		UEFIVarStore: &UEFIVarStoreRef{Key: n.c.seedKey, Firmware: n.c.identity, WriterNode: n.name, WrittenAt: time.Now().UTC()},
+	}
+	return n.launch(t, v, vars)
+}
+
+// restart takes the stopped record and starts it on n again.
+func (n *wbNode) restart(t *testing.T, vars []byte) *VM {
+	t.Helper()
+	v := n.c.stopped(t)
+	require.NoError(t, n.c.states.DeleteStoppedInstance(v.ID))
+	return n.launch(t, v, vars)
+}
+
+func (n *wbNode) guestWrites(t *testing.T, vars []byte) {
+	t.Helper()
+	require.NoError(t, os.WriteFile(filepath.Join(n.dir(), uefiVarsFileName), vars, 0o600))
+}
+
+func (n *wbNode) dir() string { return filepath.Join(n.dataDir, "uefi", testUEFIInstance) }
+
+func (n *wbNode) seeded(t *testing.T) string {
+	t.Helper()
+	key, err := readUEFISeed(n.dir())
+	require.NoError(t, err)
+	return key
+}
+
+func (n *wbNode) stop(t *testing.T, v *VM) {
+	t.Helper()
+	n.m.UpdateState(v.ID, func(v *VM) { v.DesiredState = DesiredStopped })
+	require.NoError(t, n.m.Stop(v.ID))
+}
+
+func (n *wbNode) sweep(t *testing.T) int {
+	t.Helper()
+	settled, err := n.m.NewUEFIWriteBackReaper().Sweep(t.Context())
+	require.NoError(t, err)
+	return settled
+}
+
+// Variables the guest wrote are in a new object the stopped record points
+// at, and a start on the same node boots this node's copy without a get.
+func TestUEFIWriteBack_StopWritesTheStoreBackAndAStartHereReusesIt(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("BootOrder=0001"))
+	before := uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "written")
+
+	a.stop(t, v)
+
+	rec := c.stopped(t)
+	require.NotNil(t, rec.UEFIVarStore)
+	key := rec.UEFIVarStore.Key
+	assert.NotEqual(t, c.seedKey, key, "a write-back puts to a fresh key, never over the pointed one")
+	assert.Equal(t, region("BootOrder=0001"), c.region(t, key))
+	assert.Equal(t, nodeA, rec.UEFIVarStore.WriterNode)
+	assert.Equal(t, c.identity, rec.UEFIVarStore.Firmware)
+	assert.Equal(t, key, a.seeded(t), "seeded.json names the written key so a start here reuses the copy")
+	assert.Nil(t, c.owed(t))
+	assert.True(t, c.exists(t, c.seedKey), "the version it replaced is retained")
+	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "written"))
+
+	reused := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
+	a.restart(t, region("BootOrder=0001"))
+	assert.Equal(t, reused+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
+}
+
+// The written object carries the write-back source and its writer, so an
+// overwrite by a superseded node is detectable from the object alone.
+func TestUEFIWriteBack_ObjectRecordsItsSourceAndWriter(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	a.stop(t, a.newInstance(t, region("v1")))
+
+	want, err := uefistore.ObjectMeta{OwnerID: testUEFIInstance, WriterNode: nodeA, Firmware: c.identity,
+		CodePath: FirmwarePathCandidates["x86_64"][0].Code, Source: uefistore.SourceWriteBack}.Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, aws.StringValueMap(want), c.objects.putMetadata(c.stopped(t).UEFIVarStore.Key))
+}
+
+// A drain stop keeps the instance in the running set, so its pointer is set
+// in memory and every later running-set write carries it.
+func TestUEFIWriteBack_DrainStopPointerSurvivesTheNextRunningSetWrite(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("drained"))
+
+	require.NoError(t, a.m.StopAll())
+
+	var key string
+	a.m.Inspect(v, func(v *VM) { key = v.UEFIVarStore.Key })
+	assert.NotEqual(t, c.seedKey, key)
+	assert.Equal(t, region("drained"), c.region(t, key))
+	assert.Equal(t, key, a.seeded(t))
+
+	require.NoError(t, a.m.writeRunningState())
+	saved, _, err := c.states.LoadRunningState(nodeA)
+	require.NoError(t, err)
+	require.Contains(t, saved, v.ID)
+	assert.Equal(t, key, saved[v.ID].UEFIVarStore.Key, "the running-set write must carry the pointer the stop set")
+}
+
+// With predastore unreachable a stop completes, keeps the record on the
+// version it booted from and records the obligation; the reaper converges.
+func TestS8_UEFIStopCompletesWithPredastoreUnreachableAndTheReaperConverges(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("changed while down"))
+	c.objects.setPutErr(errPredastoreDown)
+	deferred := uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "deferred")
+
+	start := time.Now()
+	a.stop(t, v)
+	assert.Less(t, time.Since(start), uefistore.PutTimeout, "the stop must not wait out more than one bounded put")
+
+	assert.Equal(t, c.seedKey, c.stopped(t).UEFIVarStore.Key, "the record keeps the pointer it booted from")
+	owed := c.owed(t)
+	require.NotNil(t, owed, "the obligation is recorded on the store record")
+	assert.Equal(t, nodeA, owed.Node)
+	assert.Equal(t, c.seedKey, owed.SeededFrom)
+	assert.Equal(t, region("changed while down"), readFile(t, filepath.Join(a.dir(), uefiVarsFileName)), "vars.fd is kept")
+	assert.Equal(t, c.seedKey, a.seeded(t))
+	assert.Equal(t, deferred+1, uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "deferred"))
+
+	_, err := a.m.NewUEFIWriteBackReaper().Sweep(t.Context())
+	require.Error(t, err, "a sweep while predastore is down settles nothing")
+	require.NotNil(t, c.owed(t))
+
+	c.objects.setPutErr(nil)
+	assert.Equal(t, 1, a.sweep(t))
+
+	rec := c.stopped(t)
+	assert.Equal(t, owed.Key, rec.UEFIVarStore.Key, "the reaper puts to the recorded key and points the record at it")
+	assert.Equal(t, region("changed while down"), c.region(t, owed.Key))
+	assert.Nil(t, c.owed(t))
+	assert.Equal(t, owed.Key, a.seeded(t))
+	assert.Zero(t, a.sweep(t), "a settled obligation is not retried")
+}
+
+// A start on another node before the reaper runs boots the pointer, clears
+// the obligation, and the first node's reaper then writes nothing.
+// A start elsewhere waits while the node owing the write-back is a member: its
+// copy is newer than both the pointer and any copy the starting node kept.
+func TestUEFIWriteBack_StartElsewhereWaitsForAMembersOwedWriteBack(t *testing.T) {
+	for name, keepsACopy := range map[string]bool{"no local copy": false, "a stale local copy": true} {
+		t.Run(name, func(t *testing.T) {
+			c := newWBCluster(t)
+			c.members[nodeA], c.members[nodeB] = true, true
+			a, b := c.node(t, nodeA), c.node(t, nodeB)
+			first, second := a, b
+			if keepsACopy {
+				first, second = b, a
+			}
+			v := first.newInstance(t, region("v1"))
+			if keepsACopy {
+				first.stop(t, v)
+				v = second.restart(t, region("set on node-a"))
+			} else {
+				first.guestWrites(t, region("set on node-a"))
+			}
+			c.objects.setPutErr(errPredastoreDown)
+			a.stop(t, v)
+			require.NotNil(t, c.owed(t))
+			pointer := c.stopped(t).UEFIVarStore.Key
+
+			err := b.m.prepareUEFIVarStore(t.Context(), c.stopped(t))
+			require.ErrorIs(t, err, ErrUEFIWriteBackPending)
+			require.NotNil(t, c.owed(t), "the refusal keeps node-a's obligation")
+			assert.Equal(t, pointer, c.stopped(t).UEFIVarStore.Key)
+
+			c.objects.setPutErr(nil)
+			a.sweep(t)
+			require.Nil(t, c.owed(t), "node-a's reaper lands the write-back")
+			v = c.stopped(t)
+			require.NoError(t, c.states.DeleteStoppedInstance(v.ID))
+			v.Status, v.DesiredState = StateRunning, DesiredRunning
+			b.m.Insert(v)
+			c.pointers.run(nodeB, v)
+			require.NoError(t, b.m.prepareUEFIVarStore(t.Context(), v))
+			assert.Equal(t, region("set on node-a"), readFile(t, filepath.Join(b.dir(), uefiVarsFileName)),
+				"node-b boots the variables node-a wrote back")
+		})
+	}
+}
+
+// A node that has left the cluster takes its unwritten variables with it.
+func TestUEFIWriteBack_StartElsewhereClearsTheObligationAndBootsThePointer(t *testing.T) {
+	c := newWBCluster(t)
+	a, b := c.node(t, nodeA), c.node(t, nodeB)
+	v := a.newInstance(t, region("lost with node-a"))
+	c.objects.setPutErr(errPredastoreDown)
+	a.stop(t, v)
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+	c.objects.setPutErr(nil)
+
+	c.stopped(t).LastNode = nodeB
+	started := b.restart(t, region("template"))
+
+	assert.Equal(t, region("template"), readFile(t, filepath.Join(b.dir(), uefiVarsFileName)), "node-b boots the pointed object")
+	assert.Nil(t, c.owed(t), "the start clears node-a's obligation")
+	puts := c.objects.putCount()
+	assert.Zero(t, a.sweep(t))
+	assert.Equal(t, puts, c.objects.putCount(), "node-a's reaper has nothing left to write")
+	assert.Equal(t, c.seedKey, started.UEFIVarStore.Key)
+}
+
+// A put that lands while the record write fails is retried at the same key,
+// so repeated failures add one object rather than one each.
+func TestUEFIWriteBack_RepeatedFailedWriteBacksHoldAtMostFiveObjects(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("v0"))
+	type step struct {
+		putFails, recordFails bool
+	}
+	steps := []step{{}, {}, {}, {recordFails: true}, {recordFails: true}, {recordFails: true}, {putFails: true}, {},
+		{recordFails: true}, {}, {}, {putFails: true}, {recordFails: true}, {recordFails: true}, {recordFails: true}, {}, {}}
+	for i, s := range steps {
+		if i > 0 {
+			v = a.restart(t, region(fmt.Sprintf("v%d", i)))
+		}
+		if s.putFails {
+			c.objects.setPutErr(errPredastoreDown)
+		}
+		if s.recordFails {
+			c.states.stoppedErr = errors.New("kv unavailable")
+		}
+		a.stop(t, v)
+		require.NoError(t, a.m.pruneDueUEFIStores(t.Context()))
+		c.objects.setPutErr(nil)
+		if s.recordFails {
+			// The instance is still in the running set, as a failed stopped
+			// write leaves it, so the next stop drives it from there.
+			c.states.stoppedErr = nil
+			require.NoError(t, c.states.fakeStateStore.WriteStoppedInstance(v.ID, v))
+			a.m.Delete(v.ID)
+		}
+		assert.LessOrEqual(t, len(c.keys(t)), 5, "step %d: %v", i, c.keys(t))
+		assert.True(t, c.exists(t, c.stopped(t).UEFIVarStore.Key), "step %d: the pointed object exists", i)
+	}
+	assert.LessOrEqual(t, len(c.keys(t)), 4, "a successful write-back prunes to the newest versions")
+}
+
+func readFile(t *testing.T, path string) []byte {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return b
+}
+
+func (c *wbCluster) markDeleting(t *testing.T) {
+	t.Helper()
+	rec, rev, err := c.records.Get(t.Context(), testUEFIInstance)
+	require.NoError(t, err)
+	rec.State, rec.WriteBack = uefistore.StateDeleting, nil
+	_, err = c.records.CompareAndSet(t.Context(), rec, rev)
+	require.NoError(t, err)
+}
+
+// A record write that landed without its reply leaves an obligation for a key
+// the record already points at; the reaper clears it without another put.
+func TestUEFIWriteBackReaper_ClearsAnObligationWhoseRecordWriteLanded(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("landed"))
+	c.states.landThenFail = true
+	a.stop(t, v)
+	c.states.landThenFail = false
+	a.m.Delete(v.ID)
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+	require.Equal(t, owed.Key, c.stopped(t).UEFIVarStore.Key)
+	landed := uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "landed")
+	puts := c.objects.putCount()
+
+	assert.Equal(t, 1, a.sweep(t))
+
+	assert.Nil(t, c.owed(t))
+	assert.Equal(t, puts, c.objects.putCount(), "the object is already there")
+	assert.Equal(t, owed.Key, a.seeded(t), "a start here reuses the copy the landed key holds")
+	assert.Equal(t, landed+1, uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "landed"))
+}
+
+// A terminate that marks the store deleting while a stop is writing back owes
+// nothing more, so the stop deletes the object it just wrote.
+func TestUEFIWriteBack_StoreDeletingAfterThePutDeletesTheObjectJustWritten(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("terminating"))
+	var written string
+	c.objects.afterPut = func(key string) {
+		written = key
+		c.markDeleting(t)
+	}
+	dropped := uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "dropped")
+
+	a.stop(t, v)
+
+	require.NotEmpty(t, written)
+	assert.False(t, c.exists(t, written), "the object written under a deleting store is deleted again")
+	assert.Equal(t, dropped+1, uefiCounter(t, "mulga.vm.uefi_store.write_backs", "outcome", "dropped"))
+	_, err := c.records.LiveWriteBack(t.Context(), testUEFIInstance)
+	require.ErrorIs(t, err, uefistore.ErrNotLive)
+}
+
+// The reaper's put races a terminate the same way, and the same rule holds:
+// the object it put is deleted and the pointer is left alone.
+func TestUEFIWriteBackReaper_StoreDeletingDuringTheRetryDeletesItsPut(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("terminating"))
+	c.objects.setPutErr(errPredastoreDown)
+	a.stop(t, v)
+	c.objects.setPutErr(nil)
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+	c.objects.afterPut = func(string) { c.markDeleting(t) }
+
+	assert.Zero(t, a.sweep(t))
+
+	assert.False(t, c.exists(t, owed.Key))
+	assert.Equal(t, c.seedKey, c.stopped(t).UEFIVarStore.Key)
+}
+
+// A drain-stopped instance stays in the running set, so the reaper moves its
+// pointer in memory, where the next running-set write carries it.
+func TestUEFIWriteBackReaper_DrainStoppedInstanceGetsItsPointerInMemory(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("drained while down"))
+	c.objects.setPutErr(errPredastoreDown)
+	require.NoError(t, a.m.StopAll())
+	c.objects.setPutErr(nil)
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+
+	assert.Equal(t, 1, a.sweep(t))
+
+	var key string
+	a.m.Inspect(v, func(v *VM) { key = v.UEFIVarStore.Key })
+	assert.Equal(t, owed.Key, key)
+	assert.Equal(t, region("drained while down"), c.region(t, key))
+	saved, _, err := c.states.LoadRunningState(nodeA)
+	require.NoError(t, err)
+	assert.Equal(t, owed.Key, saved[v.ID].UEFIVarStore.Key, "the reaper's pointer is in the running set it wrote")
+	require.NoError(t, a.m.writeRunningState())
+	saved, _, err = c.states.LoadRunningState(nodeA)
+	require.NoError(t, err)
+	assert.Equal(t, owed.Key, saved[v.ID].UEFIVarStore.Key, "and survives the next running-set write")
+	assert.Nil(t, c.owed(t))
+}
+
+// Once this node has started the instance again from its own copy, the next
+// stop writes that copy back, so the older obligation is dropped.
+func TestUEFIWriteBackReaper_DropsAnObligationOnceTheInstanceRunsAgain(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("v1"))
+	c.states.stoppedErr = assert.AnError
+	a.stop(t, v)
+	c.states.stoppedErr = nil
+	require.NoError(t, c.states.fakeStateStore.WriteStoppedInstance(v.ID, v))
+	a.m.Delete(v.ID)
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+	v = a.restart(t, region("v2"))
+
+	assert.Equal(t, 1, a.sweep(t))
+
+	assert.Nil(t, c.owed(t))
+	assert.False(t, c.exists(t, owed.Key))
+	a.stop(t, v)
+	key := c.stopped(t).UEFIVarStore.Key
+	assert.NotEqual(t, owed.Key, key)
+	assert.Equal(t, region("v2"), c.region(t, key))
+}
+
+// A stop that cannot read the instance record cannot check ownership, so it
+// puts nothing and leaves the decision to the reaper, which checks it.
+func TestUEFIWriteBack_UnreadableInstanceRecordDefersWithoutAPut(t *testing.T) {
+	c := newWBCluster(t)
+	a := c.node(t, nodeA)
+	v := a.newInstance(t, region("unverified"))
+	c.pointers.loadErr = errors.New("nats: no responders")
+	puts := c.objects.putCount()
+
+	a.stop(t, v)
+
+	assert.Equal(t, puts, c.objects.putCount())
+	owed := c.owed(t)
+	require.NotNil(t, owed)
+	assert.Equal(t, c.seedKey, owed.SeededFrom)
+
+	c.pointers.loadErr = nil
+	assert.Equal(t, 1, a.sweep(t))
+	assert.Equal(t, owed.Key, c.stopped(t).UEFIVarStore.Key)
+	assert.Equal(t, region("unverified"), c.region(t, owed.Key))
+}

spinifex/runtime/compute/vm/vm.go

@@ -317,6 +317,17 @@ type VM struct {
 	// UEFIVarStore points at the object holding this instance's UEFI variable
 	// store. Status, so written only by the node; nil for a BIOS instance.
 	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
+	// UEFIWriteBackOwed names the node holding variables newer than the
+	// pointer, set by a stop whose write-back did not land. A start elsewhere
+	// waits while that node is a cluster member.
+	UEFIWriteBackOwed *UEFIWriteBackOwed `json:"uefi_write_back_owed,omitempty"`
+}
+
+// UEFIWriteBackOwed is the node owing an instance's variable store write-back
+// and when its stop failed to land it.
+type UEFIWriteBackOwed struct {
+	Node  string    `json:"node"`
+	Since time.Time `json:"since"`
 }
 
 // UEFIVarStoreRef names the store object an instance launches from, with the
@@ -417,7 +428,7 @@ type IOThread struct {
 // monitor-owned node addressable by node-name, which is what makes
 // blockdev-del able to remove it later. Used for boot and data volumes, so a
 // cold-booted data volume has the same detachable block-graph shape as a
-// hot-attached one; EFI volumes keep the pflash -drive shape.
+// hot-attached one.
 type Blockdev struct {
 	Value string `json:"value"`
 }
@@ -590,7 +601,7 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		}
 		// pflash CODE must come before VARS so QEMU loads firmware into the
 		// readonly slot first; startQEMU emits the VARS drive (unit 1) from
-		// the node's copy of the variable store or the EFI volume.
+		// the node's copy of the variable store.
 		drives = append([]Drive{{If: "pflash", Format: "raw", Unit: 0, ReadOnly: true, File: codePath}}, drives...)
 	}
 

spinifex/runtime/daemon/daemon.go

@@ -2222,6 +2222,9 @@ func (d *Daemon) startCluster() error {
 	}
 	d.instanceService.SetVolumeCreator(d.volumeService)
 	d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, ipAllocator)
+	if d.jsManager != nil && d.jsManager.UEFIStores() != nil {
+		d.instanceService.SetUEFIVarStore(d.jsManager.UEFIStores(), d.node)
+	}
 
 	if d.gpuManager != nil {
 		d.instanceService.SetGPUClaimer(&daemonGPUClaimer{d: d})
@@ -2580,6 +2583,7 @@ func (d *Daemon) startCluster() error {
 			d.vmMgr.NewTerminatedTeardownReaper(),
 			d.vmMgr.NewOrphanQEMUReaper(),
 			d.vmMgr.NewStuckTerminateReaper(),
+			d.vmMgr.NewUEFIWriteBackReaper(),
 			d.vmMgr.NewLocalConsoleLogReaper(),
 		}
 		if eniRec := d.newENIReconciler(); eniRec != nil {

spinifex/runtime/daemon/daemon_test.go

@@ -3386,8 +3386,8 @@ func TestVolumeMounterAdapter_Unmount_NotFoundIsASealedOutcome(t *testing.T) {
 		AccountID: "000000000042",
 		EBSRequests: vm.EBSRequests{
 			Requests: []viperblocklegacyv1.EBSRequest{
-				{Name: "vol-boot-retry", Boot: true, EFI: false},
-				{Name: "vol-data-retry", Boot: false, EFI: false},
+				{Name: "vol-boot-retry", Boot: true},
+				{Name: "vol-data-retry", Boot: false},
 			},
 		},
 	}

spinifex/runtime/daemon/instance_records.go

@@ -219,6 +219,49 @@ func loadRecord[T any](store *kvstore.Store[T], key string) (*T, error) {
 	return record, nil
 }
 
+var _ vm.UEFIInstancePointers = (*JetStreamManager)(nil)
+
+// LoadUEFIVarStoreOwner reads the fields of instanceID's record a variable
+// store write-back is decided on, at the revision read. An absent record is
+// found=false; an unreachable bucket is an error.
+func (m *JetStreamManager) LoadUEFIVarStoreOwner(instanceID string) (vm.UEFIVarStoreOwner, bool, error) {
+	if m.records == nil {
+		return vm.UEFIVarStoreOwner{}, false, errors.New("KV bucket not initialized")
+	}
+	record, rev, err := m.records.Get(context.Background(), instanceRecordKey(instanceID))
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return vm.UEFIVarStoreOwner{}, false, nil
+	}
+	if err != nil {
+		return vm.UEFIVarStoreOwner{}, false, err
+	}
+	return vm.UEFIVarStoreOwner{
+		Status:   record.Status.Status,
+		LastNode: record.Status.LastNode,
+		Pointer:  record.Status.UEFIVarStore,
+		Owed:     record.Status.UEFIWriteBackOwed,
+		Revision: rev,
+	}, true, nil
+}
+
+// CompareAndSetUEFIVarStore points instanceID's record at ref and clears its
+// owed write-back only while it is still at rev, changing nothing else, and
+// reports kvstore.ErrConflict once anything has written the record since.
+func (m *JetStreamManager) CompareAndSetUEFIVarStore(instanceID string, ref *vm.UEFIVarStoreRef, rev uint64) error {
+	if m.records == nil {
+		return errors.New("KV bucket not initialized")
+	}
+	key := instanceRecordKey(instanceID)
+	record, _, err := m.records.Get(context.Background(), key)
+	if err != nil {
+		return err
+	}
+	record.Status.UEFIVarStore = ref
+	record.Status.UEFIWriteBackOwed = nil
+	_, err = m.records.CompareAndSet(context.Background(), key, record, rev)
+	return err
+}
+
 // TerminateDecided reports whether instanceID has no live instance record, or
 // one that is terminating, read from the KV store at call time. The live record
 // wins over a terminated one, and a read error is returned rather than guessed.

spinifex/runtime/daemon/instance_records_uefi_test.go

@@ -0,0 +1,77 @@
+package daemon_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func uefiRef(key string) *vm.UEFIVarStoreRef {
+	return &vm.UEFIVarStoreRef{
+		Key:        uefistore.KeyPrefix("i-1") + key,
+		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "abc"},
+		WriterNode: "node-1",
+		WrittenAt:  time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC),
+	}
+}
+
+// The owner read names the fields a write-back decides on, and an absent
+// record is reported as absent rather than as an error.
+func TestLoadUEFIVarStoreOwner_ReadsStatusOwnerAndPointer(t *testing.T) {
+	m := newRecordManager(t)
+
+	_, found, err := m.LoadUEFIVarStoreOwner("i-1")
+	require.NoError(t, err)
+	assert.False(t, found)
+
+	rec := testRecord("i-1")
+	rec.Status.UEFIVarStore = uefiRef("a")
+	rec.Status.UEFIWriteBackOwed = owedBy("node-1")
+	require.NoError(t, m.WriteInstanceRecord("i-1", rec))
+
+	owner, found, err := m.LoadUEFIVarStoreOwner("i-1")
+	require.NoError(t, err)
+	require.True(t, found)
+	assert.Equal(t, vm.StateStopped, owner.Status)
+	assert.Equal(t, "node-1", owner.LastNode)
+	assert.Equal(t, uefiRef("a"), owner.Pointer)
+	assert.Equal(t, owedBy("node-1"), owner.Owed)
+	assert.NotZero(t, owner.Revision)
+}
+
+func owedBy(node string) *vm.UEFIWriteBackOwed {
+	return &vm.UEFIWriteBackOwed{Node: node, Since: time.Date(2026, 11, 4, 0, 0, 0, 0, time.UTC)}
+}
+
+// The pointer moves only at the revision the decision was made against, and
+// the move clears the owed write-back it lands and changes nothing else.
+func TestCompareAndSetUEFIVarStore_MovesOnlyThePointerAtTheRevisionRead(t *testing.T) {
+	m := newRecordManager(t)
+	rec := testRecord("i-1")
+	rec.Status.UEFIVarStore = uefiRef("a")
+	rec.Status.UEFIWriteBackOwed = owedBy("node-1")
+	require.NoError(t, m.WriteInstanceRecord("i-1", rec))
+	owner, _, err := m.LoadUEFIVarStoreOwner("i-1")
+	require.NoError(t, err)
+
+	require.NoError(t, m.CompareAndSetUEFIVarStore("i-1", uefiRef("b"), owner.Revision))
+
+	got, err := m.LoadInstanceRecord("i-1")
+	require.NoError(t, err)
+	want := testRecord("i-1")
+	want.Status.UEFIVarStore = uefiRef("b")
+	assert.Equal(t, want, got)
+
+	err = m.CompareAndSetUEFIVarStore("i-1", uefiRef("c"), owner.Revision)
+	require.ErrorIs(t, err, kvstore.ErrConflict, "a record written since the read must refuse the move")
+	got, err = m.LoadInstanceRecord("i-1")
+	require.NoError(t, err)
+	assert.Equal(t, uefiRef("b"), got.Status.UEFIVarStore)
+
+	require.ErrorIs(t, m.CompareAndSetUEFIVarStore("i-2", uefiRef("c"), 1), kvstore.ErrNotFound)
+}

spinifex/runtime/daemon/vm_adapters.go

@@ -687,14 +687,39 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 		DeviceDeletedTimeout:       d.deviceDeletedTimeout,
 		ConsumeCleanShutdownMarker: d.consumeCleanShutdownMarker(),
 		BackingStoreReady:          func() bool { return d.checkPredastoreReady() && d.checkViperblockReady() },
-		UEFIVarStore: vm.UEFIVarStoreDeps{
-			Objects: d.instanceObjects,
-			Bucket:  d.config.Predastore.Bucket,
-			DataDir: DataDirOrDefault(d.config.DataDir),
-		},
+		UEFIVarStore:               d.uefiVarStoreDeps(),
 	}
 }
 
+// uefiVarStoreDeps is where the node reads, keeps and writes back variable
+// stores. Records and Instances are left nil, not typed nil, without
+// JetStream, so write-back reads as unwired rather than panicking.
+func (d *Daemon) uefiVarStoreDeps() vm.UEFIVarStoreDeps {
+	deps := vm.UEFIVarStoreDeps{
+		Objects:      d.instanceObjects,
+		Bucket:       d.config.Predastore.Bucket,
+		DataDir:      DataDirOrDefault(d.config.DataDir),
+		NodeIsMember: d.isClusterMember,
+	}
+	if d.jsManager != nil {
+		deps.Instances = d.jsManager
+		if stores := d.jsManager.UEFIStores(); stores != nil {
+			deps.Records = stores
+		}
+	}
+	return deps
+}
+
+// isClusterMember reports whether node is in this cluster's configuration. A
+// node without a cluster view counts every node a member, which only waits.
+func (d *Daemon) isClusterMember(node string) bool {
+	if d.clusterConfig == nil {
+		return true
+	}
+	_, ok := d.clusterConfig.Nodes[node]
+	return ok
+}
+
 // instanceCleanerAdapter satisfies vm.InstanceCleaner by delegating to the
 // daemon's existing volume/VPC/EIP/placement-group services. The manager
 // owns the QMP and tap teardown directly; this adapter covers the

spinifex/runtime/daemon/vm_adapters_coverage_test.go

@@ -196,6 +196,28 @@ func TestBuildVMManagerDeps_UEFIVarStore(t *testing.T) {
 
 	d.config.DataDir = ""
 	assert.Equal(t, DefaultDataDir, d.buildVMManagerDeps().UEFIVarStore.DataDir)
+	var unwired vm.UEFIStoreRecords
+	assert.Equal(t, unwired, got.Records, "without JetStream the records stay a nil interface, so write-back reads as unwired")
+	var noInstances vm.UEFIInstancePointers
+	assert.Equal(t, noInstances, got.Instances)
+}
+
+// With JetStream the write-back reads and changes the store records and the
+// instance records through the daemon's own handles.
+func TestBuildVMManagerDeps_UEFIWriteBackUsesTheStoreRecordsAndInstanceRecords(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js, err := NewJetStreamManager(nc)
+	require.NoError(t, err)
+	d := &Daemon{config: &config.Config{}, vmMgr: vm.NewManager(), jsManager: js}
+
+	got := d.buildVMManagerDeps().UEFIVarStore
+	var unwired vm.UEFIStoreRecords
+	assert.Equal(t, unwired, got.Records, "an unopened store-record bucket is not handed out as a typed nil")
+	assert.Same(t, js, got.Instances)
+
+	require.NoError(t, js.InitUEFIStoreBucket())
+	got = d.buildVMManagerDeps().UEFIVarStore
+	assert.Same(t, js.UEFIStores(), got.Records)
 }
 
 // --- volumeMounterAdapter ---

spinifex/runtime/daemon/vm_adapters_invariants_test.go

@@ -28,15 +28,14 @@ func TestUnmountNeverReleasesBootVolume(t *testing.T) {
 		ID: "i-unmount-boot",
 		EBSRequests: vm.EBSRequests{
 			Requests: []viperblocklegacyv1.EBSRequest{
-				{Name: "vol-bios-root", Boot: true, EFI: false},
-				{Name: "vol-efi-root", Boot: true, EFI: true},
-				{Name: "vol-data", Boot: false, EFI: false},
+				{Name: "vol-root", Boot: true},
+				{Name: "vol-data", Boot: false},
 			},
 		},
 	}
 	require.NoError(t, adapter.Unmount(t.Context(), inst))
 
-	// Boot, EFI and data volumes alike are sealed and left attached: stop and
-	// crash unmount never release any attached volume.
-	assert.ElementsMatch(t, []string{"vol-bios-root", "vol-efi-root", "vol-data"}, fake.unpublished())
+	// Boot and data volumes alike are sealed and left attached: stop and crash
+	// unmount never release any attached volume.
+	assert.ElementsMatch(t, []string{"vol-root", "vol-data"}, fake.unpublished())
 }