test(uefi): separate the embedded store from the injected error in the failing-get fake

A launch whose record points at a variable store keeps the live copy at
<DataDir>/uefi/<instance-id>/vars.fd and opens it as pflash unit 1 with
format=raw. It reuses the copy when seeded.json beside it names the
pointer, and otherwise gets the object, decompresses it, checks it is
exactly the node's VARS size and writes vars.fd then seeded.json by
temporary file and rename. A failed get is retryable like a mount; a
missing or malformed object is not.

A store whose firmware arch or VARS size differs from the node's fails
the launch before any volume is mounted, with a Server.InternalError
state reason and a log naming both firmwares. A different CODE hash
boots, and is logged and counted.

A UEFI instance with no pointer and no EFI volume, or with both, is
refused with InternalError before its record is changed, in launch,
Start and the claimed-instance start. An instance whose variables are on
an EFI volume still launches through it.

spinifex/domains/ec2/instance/service_impl.go

@@ -2689,6 +2689,15 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 // start returns it to the stopped store, a recovery returns it to the node it
 // was taken from.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
+	// Before anything is stamped or claimed, so the record handed back is the
+	// one that was claimed.
+	if err := vm.CheckUEFIVarStore(instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
+			"instanceId", instance.ID, "err", err)
+		undo()
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
 	instance.ResetNodeLocalState()
 
 	// AWS restamps LaunchTime on every start. The pending watchdog times a

spinifex/domains/ec2/instance/service_impl_test.go

@@ -2605,6 +2605,34 @@ func TestStartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
 	assert.Equal(t, vm.StateStopped, store.WroteStopped[id].Status)
 }
 
+// A UEFI instance with no variable store pointer is refused before anything is
+// stamped or claimed, so the record handed back is the one that was claimed.
+func TestStartStoppedInstance_UEFIInstanceWithoutAPointerIsRefused(t *testing.T) {
+	id := "i-uefi-nopointer"
+	itype := "t3.micro"
+	launched := time.Unix(1700000000, 0).UTC()
+	stopped := &vm.VM{
+		ID: id, Status: vm.StateStopped, AccountID: "acc", InstanceType: itype, BootMode: "uefi",
+		Instance: &ec2.Instance{LaunchTime: aws.Time(launched)},
+	}
+	before := stopped.Record()
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stopped}}
+	prov := &fakeResourceCapacityProvider{instanceTypes: map[string]*ec2.InstanceTypeInfo{
+		itype: {InstanceType: aws.String(itype)},
+	}}
+	svc := &InstanceServiceImpl{stoppedStore: store, resourceMgr: prov, vmMgr: vm.NewManager()}
+
+	_, err := svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, "acc")
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Empty(t, prov.allocated, "nothing is allocated for a start that cannot boot")
+	require.NotNil(t, store.WroteStopped[id], "the claimed instance is handed back")
+	assert.Equal(t, before, store.WroteStopped[id].Record(), "the record is unchanged")
+	_, inManager := svc.vmMgr.Get(id)
+	assert.False(t, inManager)
+}
+
 func TestStartStoppedInstance_AllocateFails(t *testing.T) {
 	id := "i-alloc-fail"
 	itype := "t3.micro"

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,185 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	objectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return objectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/operator/upgrade/VERSIONS.md

@@ -23,6 +23,7 @@ Single source of truth for the current schema version of every config file Spini
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
 | `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
+| `spinifex-uefi-stores` | `1` | `uefistore.BucketVersion` |
 
 ## Registered migrations
 

spinifex/runtime/compute/vm/firmware.go

@@ -1,14 +1,21 @@
 package vm
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"io"
 	"os"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 )
 
 // FirmwareCandidate is a (CODE, VARS template) pair shipped together by a
-// firmware package. CODE is mounted readonly as pflash unit 0; VARS is copied
-// into a per-VM viperblock volume on first boot and mounted as pflash unit 1
-// so EFI variables (BootOrder, BootNext, secure-boot state) survive reboots.
+// firmware package. CODE is mounted readonly as pflash unit 0; VARS seeds the
+// instance's variable store, which is mounted as pflash unit 1 so EFI
+// variables (BootOrder, BootNext, secure-boot state) survive reboots.
 type FirmwareCandidate struct {
 	Code         string
 	VarsTemplate string
@@ -48,3 +55,67 @@ func FirmwarePaths(arch string) (code, varsTemplate string, varsSize int64, err
 	}
 	return "", "", 0, fmt.Errorf("no UEFI firmware found for architecture %q; install ovmf (x86_64) or qemu-efi-aarch64 (arm64)", arch)
 }
+
+// NodeFirmware is the UEFI firmware this node runs for one architecture: the
+// candidate FirmwarePaths matched and the identity a variable store is
+// checked against.
+type NodeFirmware struct {
+	Code         string
+	VarsTemplate string
+	Identity     uefistore.FirmwareIdentity
+}
+
+// codeHashKey identifies one version of a CODE blob on disk, so a package
+// update that replaces the file is hashed again.
+type codeHashKey struct {
+	path    string
+	size    int64
+	modTime time.Time
+}
+
+// codeHashes caches CODE blob hashes by codeHashKey. The arm64 blob is 64 MiB,
+// and every UEFI launch needs its hash.
+var codeHashes sync.Map
+
+// NodeFirmwareFor returns the firmware this node runs for arch, with the
+// SHA-256 of its CODE blob.
+func NodeFirmwareFor(arch string) (NodeFirmware, error) {
+	code, vars, varsSize, err := FirmwarePaths(arch)
+	if err != nil {
+		return NodeFirmware{}, err
+	}
+	sum, err := codeSHA256(code)
+	if err != nil {
+		return NodeFirmware{}, err
+	}
+	return NodeFirmware{
+		Code:         code,
+		VarsTemplate: vars,
+		Identity:     uefistore.FirmwareIdentity{Arch: arch, VarsSize: varsSize, CodeSHA256: sum},
+	}, nil
+}
+
+func codeSHA256(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", fmt.Errorf("open UEFI CODE %s: %w", path, err)
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return "", fmt.Errorf("stat UEFI CODE %s: %w", path, err)
+	}
+	key := codeHashKey{path: path, size: info.Size(), modTime: info.ModTime()}
+	if cached, ok := codeHashes.Load(key); ok {
+		if sum, ok := cached.(string); ok {
+			return sum, nil
+		}
+	}
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("hash UEFI CODE %s: %w", path, err)
+	}
+	sum := hex.EncodeToString(h.Sum(nil))
+	codeHashes.Store(key, sum)
+	return sum, nil
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -92,6 +92,9 @@ func (m *Manager) Start(ctx context.Context, id string) error {
 	if !ok {
 		return fmt.Errorf("%w: %s", ErrInstanceNotFound, id)
 	}
+	if err := CheckUEFIVarStore(instance); err != nil {
+		return err
+	}
 	// StateError is rejected by launchStillValid; move to Pending first so
 	// launch proceeds (resource re-allocation is the caller's responsibility).
 	if m.Status(instance) == StateError && m.deps.TransitionState != nil {
@@ -127,6 +130,13 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		return nil
 	}
 
+	// Before the reason below is cleared, so a refused start changes nothing.
+	if err := CheckUEFIVarStore(instance); err != nil {
+		slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
+			"instanceId", instance.ID, "err", err)
+		return err
+	}
+
 	// A reason left by the stop, or by a restore that found this node could not
 	// schedule the instance, describes a state the instance is now leaving.
 	// Clearing it here keeps the node the only writer of the field.
@@ -148,6 +158,16 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		}
 	}
 
+	// Before the mount, so a store this node cannot boot fails the launch
+	// with nothing to roll back.
+	uefiCtx, uefiSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.uefi_var_store")
+	uefiErr := m.prepareUEFIVarStore(uefiCtx, instance)
+	endSpanWithError(uefiSpan, uefiErr)
+	if uefiErr != nil {
+		slog.ErrorContext(ctx, "Failed to prepare the UEFI variable store", "instanceId", instance.ID, "err", uefiErr)
+		return uefiErr
+	}
+
 	_, mountSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.mount_volumes")
 	mountErr := m.deps.VolumeMounter.Mount(ctx, instance)
 	endSpanWithError(mountSpan, mountErr)
@@ -324,6 +344,14 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 		instance.Config.IOThreads = append(instance.Config.IOThreads, driveCfg.IOThreads...)
 		instance.Config.Devices = append(instance.Config.Devices, driveCfg.Devices...)
 		instance.Config.Blockdevs = append(instance.Config.Blockdevs, driveCfg.Blockdevs...)
+
+		varsDrive, err := m.uefiVarsDrive(instance)
+		if err != nil {
+			return err
+		}
+		if varsDrive != nil {
+			instance.Config.Drives = append(instance.Config.Drives, *varsDrive)
+		}
 	}
 
 	if instance.DirectBoot {
@@ -1322,7 +1350,7 @@ func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serial
 		CPUCount:       vCPUs,
 		Architecture:   architecture,
 		InstanceType:   instanceType,
-		UseUEFI:        bootMode == "uefi" || bootMode == "uefi-preferred",
+		UseUEFI:        isUEFIBootMode(bootMode),
 
 		// Present EC2-shaped DMI so stock cloud-init activates the Ec2 datasource.
 		SMBIOSUUID:         ec2SMBIOSUUID(instanceID),
@@ -1415,7 +1443,8 @@ func buildDiskBootDrives(requests []EBSRequest, cpuCount int, machineType string
 //
 // The boot volume is a -blockdev node named "os" under a bootindex=1 device
 // that carries werror/rerror; it is never detached, so it keeps fixed names.
-// EFI volumes emit pflash unit=1; Config.Execute adds the CODE blob (unit=0).
+// EFI volumes emit pflash unit=1, as uefiVarsDrive does for a variable store
+// pointer; Config.Execute adds the CODE blob (unit=0).
 //
 // Every other (data) volume is given the same named node-name/device-id/
 // iothread-id shape AttachVolume's hot-plug path uses, via a -blockdev entry

spinifex/runtime/compute/vm/manager.go

@@ -87,6 +87,9 @@ type Deps struct {
 	// (predastore + viperblock) is confirmed up. Nil is treated as ready,
 	// preserving behaviour for tests and nodes with storage unwired.
 	BackingStoreReady func() bool
+
+	// UEFIVarStore is where UEFI variable stores are read from and kept.
+	UEFIVarStore UEFIVarStoreDeps
 }
 
 // Manager owns the in-memory map of running VMs on this node and every

spinifex/runtime/compute/vm/record.go

@@ -106,6 +106,10 @@ type InstanceStatus struct {
 	EBSRequests       []EBSRequest   `json:"ebs_requests,omitempty"`
 	ENIAvailableSlots []int          `json:"eni_available_slots,omitempty"`
 	ENIAttachedByID   map[string]int `json:"eni_attached_by_id,omitempty"`
+
+	// UEFIVarStore is optional, so records written without it still decode
+	// and the bucket schema does not change.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
 }
 
 // Record projects a VM onto the persisted record. QMPClient and the request
@@ -166,6 +170,7 @@ func (v *VM) Record() *InstanceRecord {
 			EBSRequests:                     v.EBSRequests.Requests,
 			ENIAvailableSlots:               v.ENIRequests.AvailableSlots,
 			ENIAttachedByID:                 v.ENIRequests.AttachedByENIID,
+			UEFIVarStore:                    v.UEFIVarStore,
 		},
 	}
 }
@@ -222,6 +227,7 @@ func VMFromRecord(r *InstanceRecord) *VM {
 		TerminatedAt:                    r.Status.TerminatedAt,
 		ShuttingDownAt:                  r.Status.ShuttingDownAt,
 		FirstLaunch:                     r.Status.FirstLaunch,
+		UEFIVarStore:                    r.Status.UEFIVarStore,
 	}
 	v.EBSRequests.Requests = r.Status.EBSRequests
 	v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots

spinifex/runtime/compute/vm/record_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -114,6 +115,37 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
 	assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
 }
 
+// The pointer is optional: a record written before it existed decodes with
+// none, and a record without one writes no key, so the bucket schema holds.
+func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
+	var decoded vm.InstanceRecord
+	require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
+	assert.Nil(t, decoded.Status.UEFIVarStore)
+
+	encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
+	require.NoError(t, err)
+	assert.NotContains(t, string(encoded), "uefi_var_store")
+}
+
+// The pointer's wire names are what the record carries across releases, so
+// they are pinned rather than left to follow the Go field names.
+func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
+	encoded, err := json.Marshal(populatedVM().Record())
+	require.NoError(t, err)
+
+	var wire struct {
+		Status struct {
+			UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
+		} `json:"status"`
+	}
+	require.NoError(t, json.Unmarshal(encoded, &wire))
+	ref := wire.Status.UEFIVarStore
+	assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
+	assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
+	assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
+	assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
+}
+
 // The mutexes are dropped on the way out, so the rebuilt instance must arrive
 // with usable zero-value locks rather than ones somebody else left held.
 func TestVMFromRecord_RequestWrappersAreUsable(t *testing.T) {
@@ -206,6 +238,12 @@ func populatedVM() *vm.VM {
 		CapacityReservationId: "cr-1",
 		InstanceLifecycle:     "spot",
 		SpotInstanceRequestId: "sir-1",
+		UEFIVarStore: &vm.UEFIVarStoreRef{
+			Key:        "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
+			Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
+			WriterNode: "node-3",
+			WrittenAt:  time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
+		},
 
 		Status:                          vm.StateRunning,
 		Instance:                        &ec2.Instance{InstanceId: aws.String("i-round-trip")},

spinifex/runtime/compute/vm/uefi_varstore.go

@@ -0,0 +1,390 @@
+package vm
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	// uefiStoreDirName is the directory under DataDir that holds one
+	// subdirectory per instance with its live variable store.
+	uefiStoreDirName   = "uefi"
+	uefiVarsFileName   = "vars.fd"
+	uefiSeededFileName = "seeded.json"
+
+	// uefiStoreGetTimeout bounds the one get a launch makes for a fixed-size
+	// object, so a wedged object store fails the start rather than holding it.
+	uefiStoreGetTimeout = 10 * time.Second
+
+	// uefiStoreBodySlack is how far a compressed body may exceed its region
+	// before it is refused; gzip grows incompressible data by far less.
+	uefiStoreBodySlack = 1 << 20
+
+	// uefiStateReasonCode is the StateReason a launch refused over its
+	// variable store records, as EC2 reports an internal launch failure.
+	uefiStateReasonCode = "Server.InternalError"
+)
+
+// ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
+// store and carries no EFI volume, so it has nothing to boot its variables from.
+var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
+
+// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
+// store and also carries an EFI volume, so either one would be ignored.
+var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
+
+// ErrUEFIFirmwareMismatch is a variable store written under firmware whose
+// architecture or VARS size differs from this node's, which pflash cannot load.
+var ErrUEFIFirmwareMismatch = errors.New("UEFI variable store firmware does not match this node")
+
+// ErrUEFIVarStoreUnusable is a variable store object that is missing or does
+// not decode to a region of this node's VARS size. Retrying does not help.
+var ErrUEFIVarStoreUnusable = errors.New("UEFI variable store object is unusable")
+
+// UEFIVarStoreDeps is where a node reads instance variable stores from and
+// keeps the live copy QEMU opens. DataDir is the daemon's persistent data
+// directory, so the copy survives a daemon restart and a host reboot.
+type UEFIVarStoreDeps struct {
+	Objects objectstore.ObjectStore
+	Bucket  string
+	DataDir string
+}
+
+// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
+type uefiSeed struct {
+	Key      string    `json:"key"`
+	SeededAt time.Time `json:"seeded_at"`
+}
+
+// isUEFIBootMode reports whether bootMode boots the guest through UEFI firmware.
+func isUEFIBootMode(bootMode string) bool {
+	return bootMode == "uefi" || bootMode == "uefi-preferred"
+}
+
+// usesUEFIVarStore reports whether v boots through UEFI and so needs a
+// variable store. A direct-boot microvm boots qboot and has none.
+func usesUEFIVarStore(v *VM) bool {
+	return !v.DirectBoot && isUEFIBootMode(v.BootMode)
+}
+
+// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
+// two, before anything about it is changed. An EFI volume still counts as a
+// store until RunInstances seeds pointers instead. The caller owns v.
+func CheckUEFIVarStore(v *VM) error {
+	if !usesUEFIVarStore(v) {
+		return nil
+	}
+	v.EBSRequests.Mu.Lock()
+	hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
+	v.EBSRequests.Mu.Unlock()
+
+	switch {
+	case v.UEFIVarStore == nil && !hasEFIVolume:
+		return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
+	case v.UEFIVarStore != nil && hasEFIVolume:
+		return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
+	}
+	return nil
+}
+
+// uefiInstanceDir is the directory holding instanceID's vars.fd and
+// seeded.json. An ID that is not a single path segment is refused.
+func uefiInstanceDir(dataDir, instanceID string) (string, error) {
+	if !strings.HasPrefix(instanceID, "i-") || strings.ContainsAny(instanceID, `/\`) || instanceID != filepath.Clean(instanceID) {
+		return "", fmt.Errorf("instance ID %q cannot name a UEFI store directory", instanceID)
+	}
+	return filepath.Join(dataDir, uefiStoreDirName, instanceID), nil
+}
+
+// uefiVarsPath is the live variable store QEMU opens as pflash unit 1.
+func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
+	dir, err := uefiInstanceDir(m.deps.UEFIVarStore.DataDir, instanceID)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, uefiVarsFileName), nil
+}
+
+// uefiVarsDrive is pflash unit 1 from this node's copy of the store the
+// record points at, or nil with no pointer. Launch seeds the copy and a
+// reboot reuses it, so a missing one fails rather than booting without it.
+func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
+	hasPointer := false
+	m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
+	if !hasPointer || !usesUEFIVarStore(instance) {
+		return nil, nil
+	}
+	path, err := m.uefiVarsPath(instance.ID)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(path); err != nil {
+		return nil, fmt.Errorf("UEFI variable store for %s: %w", instance.ID, err)
+	}
+	return &Drive{File: path, Format: "raw", If: "pflash", Unit: 1}, nil
+}
+
+// prepareUEFIVarStore leaves vars.fd holding the store the record points at,
+// reusing this node's copy when seeded.json names the pointer and fetching
+// the object otherwise. The object and the record's pointer are only read.
+func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
+	var ref *UEFIVarStoreRef
+	m.Inspect(instance, func(v *VM) {
+		if v.UEFIVarStore != nil {
+			cp := *v.UEFIVarStore
+			ref = &cp
+		}
+	})
+	if ref == nil || !usesUEFIVarStore(instance) {
+		return nil
+	}
+
+	deps := m.deps.UEFIVarStore
+	if deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" {
+		return errors.New("UEFI variable store access is not wired on this node")
+	}
+	if name, ok := strings.CutPrefix(ref.Key, uefistore.KeyPrefix(instance.ID)); !ok || name == "" || strings.Contains(name, "/") {
+		return fmt.Errorf("%w: pointer %q is not under %s", ErrUEFIVarStoreUnusable, ref.Key, uefistore.KeyPrefix(instance.ID))
+	}
+	if m.deps.InstanceTypes == nil {
+		return errors.New("InstanceTypeResolver not wired")
+	}
+	spec, ok := m.deps.InstanceTypes.Resolve(instance.InstanceType)
+	if !ok {
+		return fmt.Errorf("instance type %s not found", instance.InstanceType)
+	}
+	fw, err := NodeFirmwareFor(spec.Architecture)
+	if err != nil {
+		return err
+	}
+	if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
+		return err
+	}
+
+	dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
+	if err != nil {
+		return err
+	}
+	if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
+		recordUEFIStoreSeed("local")
+		slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
+		return nil
+	}
+
+	start := time.Now()
+	region, err := fetchUEFIVars(ctx, deps, ref.Key, fw.Identity.VarsSize)
+	if err != nil {
+		return err
+	}
+	if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
+		return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
+	}
+	seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
+	if err != nil {
+		return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
+	}
+	if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
+		return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
+	}
+	recordUEFIStoreSeed("object")
+	slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
+		"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
+	return nil
+}
+
+// checkUEFIFirmware fails a store whose arch or VARS size differs from this
+// node's, naming both firmwares in the log and the StateReason. A different
+// CODE build boots, because a package update changes it on every node.
+func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeFirmware) error {
+	stored, node := ref.Firmware, fw.Identity
+	var kind string
+	switch {
+	case stored.Arch != node.Arch:
+		kind = "arch"
+	case stored.VarsSize != node.VarsSize:
+		kind = "vars_size"
+	case stored.CodeSHA256 != node.CodeSHA256:
+		recordUEFIFirmwareMismatch("code")
+		slog.Warn("UEFI variable store was written under a different CODE build; booting it",
+			"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode,
+			"storeCodeSha256", stored.CodeSHA256, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
+		return nil
+	default:
+		return nil
+	}
+
+	recordUEFIFirmwareMismatch(kind)
+	msg := fmt.Sprintf("%s: the UEFI variable store %s was written under firmware arch=%s vars_size=%d code_sha256=%s, and this node runs arch=%s vars_size=%d code_sha256=%s (%s)",
+		uefiStateReasonCode, ref.Key, stored.Arch, stored.VarsSize, stored.CodeSHA256,
+		node.Arch, node.VarsSize, node.CodeSHA256, fw.Code)
+	m.Inspect(instance, func(v *VM) {
+		if v.Instance != nil {
+			v.Instance.StateReason = &ec2.StateReason{Code: aws.String(uefiStateReasonCode), Message: aws.String(msg)}
+		}
+	})
+	slog.Error("UEFI variable store firmware does not match this node; refusing the launch",
+		"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode, "mismatch", kind,
+		"storeArch", stored.Arch, "storeVarsSize", stored.VarsSize, "storeCodeSha256", stored.CodeSHA256,
+		"nodeArch", node.Arch, "nodeVarsSize", node.VarsSize, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
+	return fmt.Errorf("%w: %s", ErrUEFIFirmwareMismatch, msg)
+}
+
+// reuseUEFIVars reports whether dir already holds the store key names: a
+// seeded.json naming key beside a vars.fd of the region's size.
+func reuseUEFIVars(dir, key string, varsSize int64) bool {
+	data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
+	if err != nil {
+		return false
+	}
+	var seed uefiSeed
+	if err := json.Unmarshal(data, &seed); err != nil || seed.Key != key {
+		return false
+	}
+	info, err := os.Stat(filepath.Join(dir, uefiVarsFileName))
+	if err != nil || info.Size() != varsSize {
+		slog.Warn("UEFI variable store seeded here has no usable vars.fd; fetching it again", "dir", dir, "key", key)
+		return false
+	}
+	return true
+}
+
+// fetchUEFIVars gets key and returns its decompressed region, which must be
+// exactly varsSize bytes. A failed get is retryable; a missing or malformed
+// object is not.
+func fetchUEFIVars(ctx context.Context, deps UEFIVarStoreDeps, key string, varsSize int64) ([]byte, error) {
+	getCtx, cancel := context.WithTimeout(ctx, uefiStoreGetTimeout)
+	defer cancel()
+
+	out, err := deps.Objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(deps.Bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, fmt.Errorf("%w: %s does not exist", ErrUEFIVarStoreUnusable, key)
+		}
+		return nil, fmt.Errorf("get UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
+	}
+	defer out.Body.Close()
+
+	limit := varsSize + uefiStoreBodySlack
+	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
+	if err != nil {
+		return nil, fmt.Errorf("read UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
+	}
+	if int64(len(raw)) > limit {
+		return nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrUEFIVarStoreUnusable, key, limit)
+	}
+
+	gz, err := gzip.NewReader(bytes.NewReader(raw))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s is not gzip: %w", ErrUEFIVarStoreUnusable, key, err)
+	}
+	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s does not decompress: %w", ErrUEFIVarStoreUnusable, key, err)
+	}
+	if int64(len(region)) != varsSize {
+		return nil, fmt.Errorf("%w: %s holds a %d-byte region, and this node's VARS template is %d bytes",
+			ErrUEFIVarStoreUnusable, key, len(region), varsSize)
+	}
+	return region, nil
+}
+
+// writeFileAtomic replaces dir/name with data by temporary file and rename,
+// syncing both, so a crash leaves the old file or the new one.
+func writeFileAtomic(dir, name string, data []byte) error {
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(dir, "."+name+".*")
+	if err != nil {
+		return err
+	}
+	tmpName := tmp.Name()
+	if _, err := tmp.Write(data); err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(tmpName)
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(tmpName)
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		_ = os.Remove(tmpName)
+		return err
+	}
+	if err := os.Rename(tmpName, filepath.Join(dir, name)); err != nil {
+		_ = os.Remove(tmpName)
+		return err
+	}
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	return d.Sync()
+}
+
+// uefiStoreMeterName is the instrumentation scope for variable store launches.
+const uefiStoreMeterName = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/uefistore"
+
+var (
+	uefiStoreMetricsOnce   sync.Once
+	uefiStoreSeedCounter   metric.Int64Counter
+	uefiFirmwareMismatches metric.Int64Counter
+)
+
+func uefiStoreMetrics() {
+	uefiStoreMetricsOnce.Do(func() {
+		meter := otel.Meter(uefiStoreMeterName)
+		var err error
+		uefiStoreSeedCounter, err = meter.Int64Counter("mulga.vm.uefi_store.seeds",
+			metric.WithDescription("UEFI launches by where their variable store came from: this node's copy or the object store."),
+			metric.WithUnit("{launch}"))
+		if err != nil {
+			otel.Handle(err)
+		}
+		uefiFirmwareMismatches, err = meter.Int64Counter("mulga.vm.uefi_store.firmware_mismatches",
+			metric.WithDescription("UEFI launches whose variable store was written under different firmware, by what differed. Only kind=code boots."),
+			metric.WithUnit("{launch}"))
+		if err != nil {
+			otel.Handle(err)
+		}
+	})
+}
+
+func recordUEFIStoreSeed(source string) {
+	uefiStoreMetrics()
+	if uefiStoreSeedCounter != nil {
+		uefiStoreSeedCounter.Add(context.Background(), 1, metric.WithAttributes(attribute.String("source", source)))
+	}
+}
+
+func recordUEFIFirmwareMismatch(kind string) {
+	uefiStoreMetrics()
+	if uefiFirmwareMismatches != nil {
+		uefiFirmwareMismatches.Add(context.Background(), 1, metric.WithAttributes(attribute.String("kind", kind)))
+	}
+}

spinifex/runtime/compute/vm/uefi_varstore_test.go

@@ -0,0 +1,539 @@
+package vm
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+const (
+	testUEFIBucket   = "instances"
+	testUEFIVarsSize = 4096
+	testUEFIInstance = "i-0123456789abcdef0"
+)
+
+var (
+	uefiMetricsReaderOnce sync.Once
+	uefiMetricsReader     *sdkmetric.ManualReader
+)
+
+// uefiCounter returns the sum of the named counter's points whose attribute
+// key has value.
+func uefiCounter(t *testing.T, name, key, value string) int64 {
+	t.Helper()
+	uefiMetricsReaderOnce.Do(func() {
+		uefiMetricsReader = sdkmetric.NewManualReader()
+		otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(uefiMetricsReader)))
+	})
+	var rm metricdata.ResourceMetrics
+	require.NoError(t, uefiMetricsReader.Collect(context.Background(), &rm))
+	var total int64
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			sum, ok := m.Data.(metricdata.Sum[int64])
+			if m.Name != name || !ok {
+				continue
+			}
+			for _, dp := range sum.DataPoints {
+				if v, ok := dp.Attributes.Value(attribute.Key(key)); ok && v.AsString() == value {
+					total += dp.Value
+				}
+			}
+		}
+	}
+	return total
+}
+
+// uefiFixture is a node with one x86_64 firmware candidate, an in-memory
+// object store and a data directory.
+type uefiFixture struct {
+	m        *Manager
+	mounter  *fakeVolumeMounter
+	objects  *objectstore.MemoryObjectStore
+	dataDir  string
+	codePath string
+	identity uefistore.FirmwareIdentity
+}
+
+func newUEFIFixture(t *testing.T) *uefiFixture {
+	t.Helper()
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	codeBytes := []byte("ovmf code build 1")
+	require.NoError(t, os.WriteFile(code, codeBytes, 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, testUEFIVarsSize), 0o600))
+	orig := FirmwarePathCandidates
+	FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
+	t.Cleanup(func() { FirmwarePathCandidates = orig })
+
+	sum := sha256.Sum256(codeBytes)
+	f := &uefiFixture{
+		mounter:  &fakeVolumeMounter{},
+		objects:  objectstore.NewMemoryObjectStore(),
+		dataDir:  t.TempDir(),
+		codePath: code,
+		identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
+	}
+	f.m = NewManagerWithDeps(Deps{
+		VolumeMounter: f.mounter,
+		InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore:  UEFIVarStoreDeps{Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir},
+	})
+	return f
+}
+
+// region is a VARS region of the fixture's size whose bytes say which store it is.
+func region(tag string) []byte {
+	b := make([]byte, testUEFIVarsSize)
+	copy(b, tag)
+	return b
+}
+
+func gzipped(t *testing.T, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	_, err := zw.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func (f *uefiFixture) putObject(t *testing.T, key string, body []byte) {
+	t.Helper()
+	_, err := f.objects.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(testUEFIBucket), Key: aws.String(key), Body: bytes.NewReader(body),
+	})
+	require.NoError(t, err)
+}
+
+func (f *uefiFixture) objectBody(t *testing.T, key string) []byte {
+	t.Helper()
+	out, err := f.objects.GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(testUEFIBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	b, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	return b
+}
+
+func (f *uefiFixture) instanceDir() string {
+	return filepath.Join(f.dataDir, "uefi", testUEFIInstance)
+}
+
+func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, vars []byte) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "vars.fd"), vars, 0o600))
+	seed, err := json.Marshal(uefiSeed{Key: seededKey, SeededAt: time.Now()})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), seed, 0o600))
+}
+
+func (f *uefiFixture) localVars(t *testing.T) []byte {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "vars.fd"))
+	require.NoError(t, err)
+	return b
+}
+
+func (f *uefiFixture) seededKey(t *testing.T) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "seeded.json"))
+	require.NoError(t, err)
+	var seed uefiSeed
+	require.NoError(t, json.Unmarshal(b, &seed))
+	return seed.Key
+}
+
+func (f *uefiFixture) instance(key string, fw uefistore.FirmwareIdentity) *VM {
+	v := &VM{
+		ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Status: StateStopped,
+		Instance: &ec2.Instance{},
+	}
+	if key != "" {
+		v.UEFIVarStore = &UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-a", WrittenAt: time.Unix(1700000000, 0).UTC()}
+	}
+	f.m.Insert(v)
+	return v
+}
+
+func keyFor(name string) string { return uefistore.KeyPrefix(testUEFIInstance) + name }
+
+func TestPrepareUEFIVarStore_FetchesTheObjectAndRecordsTheSeed(t *testing.T) {
+	f := newUEFIFixture(t)
+	key := keyFor("11111111-1111-1111-1111-111111111111")
+	f.putObject(t, key, gzipped(t, region("converted")))
+	instance := f.instance(key, f.identity)
+	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object")
+
+	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
+
+	assert.Equal(t, region("converted"), f.localVars(t), "vars.fd must hold the decompressed region")
+	assert.Equal(t, key, f.seededKey(t))
+	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object"))
+	assert.Equal(t, key, instance.UEFIVarStore.Key, "seeding only reads the pointer")
+}
+
+// A start on the node that seeded the pointer boots that node's copy, which
+// can be newer than the object, and makes no get at all.
+func TestPrepareUEFIVarStore_ReusesTheLocalCopyWhenSeededNamesThePointer(t *testing.T) {
+	f := newUEFIFixture(t)
+	key := keyFor("22222222-2222-2222-2222-222222222222")
+	f.writeLocal(t, key, region("guest wrote this"))
+	instance := f.instance(key, f.identity)
+	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
+
+	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance), "no object exists, so a get would fail")
+
+	assert.Equal(t, region("guest wrote this"), f.localVars(t))
+	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
+}
+
+// Once the pointer has moved past what this node seeded, the local copy is
+// stale and the pointed object replaces it.
+func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing.T) {
+	tests := []struct {
+		name  string
+		local func(f *uefiFixture, t *testing.T)
+	}{
+		{name: "seeded from an older pointer", local: func(f *uefiFixture, t *testing.T) {
+			f.writeLocal(t, keyFor("older"), region("stale"))
+		}},
+		{name: "vars.fd of the wrong size", local: func(f *uefiFixture, t *testing.T) {
+			f.writeLocal(t, keyFor("current"), []byte("short"))
+		}},
+		{name: "seeded.json unreadable", local: func(f *uefiFixture, t *testing.T) {
+			f.writeLocal(t, keyFor("current"), region("stale"))
+			require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte("{"), 0o600))
+		}},
+		{name: "no vars.fd", local: func(f *uefiFixture, t *testing.T) {
+			f.writeLocal(t, keyFor("current"), region("stale"))
+			require.NoError(t, os.Remove(filepath.Join(f.instanceDir(), "vars.fd")))
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newUEFIFixture(t)
+			key := keyFor("current")
+			tt.local(f, t)
+			f.putObject(t, key, gzipped(t, region("restored")))
+			instance := f.instance(key, f.identity)
+
+			require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
+
+			assert.Equal(t, region("restored"), f.localVars(t))
+			assert.Equal(t, key, f.seededKey(t))
+		})
+	}
+}
+
+// A store pflash cannot load fails the launch before any volume is mounted,
+// names both firmwares, and leaves the object, the pointer and the local
+// copy as they were.
+func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name   string
+		kind   string
+		mutate func(*uefistore.FirmwareIdentity)
+		want   []string
+	}{
+		{name: "arch", kind: "arch", mutate: func(fw *uefistore.FirmwareIdentity) { fw.Arch = "arm64" },
+			want: []string{"arch=arm64", "arch=x86_64"}},
+		{name: "vars size", kind: "vars_size", mutate: func(fw *uefistore.FirmwareIdentity) { fw.VarsSize = 67108864 },
+			want: []string{"vars_size=67108864", "vars_size=4096"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newUEFIFixture(t)
+			key := keyFor("mismatch")
+			body := gzipped(t, region("other firmware"))
+			f.putObject(t, key, body)
+			f.writeLocal(t, keyFor("older"), region("local"))
+			stored := f.identity
+			tt.mutate(&stored)
+			instance := f.instance(key, stored)
+			wantRef := *instance.UEFIVarStore
+			before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind)
+
+			err := f.m.Run(t.Context(), instance)
+
+			require.ErrorIs(t, err, ErrUEFIFirmwareMismatch)
+			require.NotErrorIs(t, err, ErrMountRetryable, "a mismatch does not clear with time")
+			assert.Empty(t, f.mounter.mounted, "nothing is mounted for a store that cannot boot")
+			require.NotNil(t, instance.Instance.StateReason)
+			assert.Equal(t, "Server.InternalError", aws.StringValue(instance.Instance.StateReason.Code))
+			for _, s := range append(tt.want, f.identity.CodeSHA256, f.codePath, key) {
+				assert.Contains(t, aws.StringValue(instance.Instance.StateReason.Message), s)
+			}
+			assert.Equal(t, wantRef, *instance.UEFIVarStore, "the pointer is unchanged")
+			assert.Equal(t, body, f.objectBody(t, key), "the object is unchanged")
+			assert.Equal(t, region("local"), f.localVars(t), "the local copy is unchanged")
+			assert.Equal(t, keyFor("older"), f.seededKey(t))
+			assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind))
+		})
+	}
+}
+
+// An ovmf package update changes the CODE hash on every node, so a store
+// written under another build boots and the difference is counted.
+func TestPrepareUEFIVarStore_DifferentCodeBuildBootsAndIsCounted(t *testing.T) {
+	f := newUEFIFixture(t)
+	key := keyFor("code")
+	f.putObject(t, key, gzipped(t, region("older ovmf")))
+	stored := f.identity
+	stored.CodeSHA256 = strings.Repeat("0", 64)
+	instance := f.instance(key, stored)
+	before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code")
+
+	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
+
+	assert.Equal(t, region("older ovmf"), f.localVars(t))
+	assert.Nil(t, instance.Instance.StateReason)
+	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code"))
+}
+
+// failingGetStore fails every get, standing in for an object store that
+// does not answer.
+type failingGetStore struct {
+	*objectstore.MemoryObjectStore
+
+	err error
+}
+
+func (s *failingGetStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	return nil, s.err
+}
+
+func TestPrepareUEFIVarStore_FailedGetIsRetryable(t *testing.T) {
+	f := newUEFIFixture(t)
+	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("connection refused")}
+	instance := f.instance(keyFor("down"), f.identity)
+
+	err := f.m.prepareUEFIVarStore(t.Context(), instance)
+
+	require.ErrorIs(t, err, ErrMountRetryable, "restore backs off on a get as it does on a mount")
+	assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
+}
+
+// An object that is missing or does not decode to this node's region is
+// refused for good, and nothing is written locally.
+func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
+	tests := []struct {
+		name string
+		body []byte
+	}{
+		{name: "missing"},
+		{name: "not gzip", body: region("raw, uncompressed")},
+		{name: "short region", body: gzipped(t, make([]byte, testUEFIVarsSize-1))},
+		{name: "long region", body: gzipped(t, make([]byte, testUEFIVarsSize+1))},
+		{name: "truncated gzip", body: gzipped(t, region("cut"))[:20]},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newUEFIFixture(t)
+			key := keyFor("bad")
+			if tt.body != nil {
+				f.putObject(t, key, tt.body)
+			}
+			instance := f.instance(key, f.identity)
+
+			err := f.m.prepareUEFIVarStore(t.Context(), instance)
+
+			require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
+			require.NotErrorIs(t, err, ErrMountRetryable)
+			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "vars.fd"))
+			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
+		})
+	}
+}
+
+func TestPrepareUEFIVarStore_PointerOutsideTheInstancePrefixIsRefused(t *testing.T) {
+	for _, key := range []string{"uefi/i-0aaaaaaaaaaaaaaaa/other", keyFor("a/b"), uefistore.KeyPrefix(testUEFIInstance)} {
+		t.Run(key, func(t *testing.T) {
+			f := newUEFIFixture(t)
+			f.putObject(t, key, gzipped(t, region("x")))
+			instance := f.instance(key, f.identity)
+
+			require.ErrorIs(t, f.m.prepareUEFIVarStore(t.Context(), instance), ErrUEFIVarStoreUnusable)
+		})
+	}
+}
+
+// A BIOS instance, a microvm and a legacy instance with no pointer touch
+// neither the object store nor the data directory.
+func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
+	f := newUEFIFixture(t)
+	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("must not be called")}
+	bios := f.instance(keyFor("bios"), f.identity)
+	bios.BootMode = ""
+	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), bios))
+
+	legacy := f.instance("", f.identity)
+	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), legacy))
+	assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
+}
+
+// A UEFI instance with no pointer, or two stores, is refused before the
+// launch clears or changes anything on its record.
+func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
+	efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
+	tests := []struct {
+		name    string
+		pointer bool
+		efi     bool
+		want    error
+	}{
+		{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
+		{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newUEFIFixture(t)
+			key := ""
+			if tt.pointer {
+				key = keyFor("p")
+			}
+			instance := f.instance(key, f.identity)
+			if tt.efi {
+				instance.EBSRequests.Requests = []EBSRequest{efiVolume}
+			}
+			stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
+			instance.Instance.StateReason = stale
+			before := instance.Record()
+
+			err := f.m.Run(t.Context(), instance)
+
+			require.ErrorIs(t, err, tt.want)
+			assert.Empty(t, f.mounter.mounted)
+			assert.Equal(t, before, instance.Record(), "the record is unchanged")
+		})
+	}
+}
+
+// Start from StateError transitions to pending before launching; a refused
+// UEFI instance must not take that transition either.
+func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testing.T) {
+	f := newUEFIFixture(t)
+	transitions := 0
+	f.m.deps.TransitionState = func(v *VM, s InstanceState) error { transitions++; v.Status = s; return nil }
+	instance := f.instance("", f.identity)
+	instance.Status = StateError
+
+	err := f.m.Start(t.Context(), instance.ID)
+
+	require.ErrorIs(t, err, ErrUEFIVarStoreMissing)
+	assert.Equal(t, 0, transitions)
+	assert.Equal(t, StateError, instance.Status)
+}
+
+// Until RunInstances seeds pointers, an instance whose variables are on an
+// EFI volume still launches through the volume, and a BIOS or microvm
+// guest needs no store at all.
+func TestCheckUEFIVarStore_Accepts(t *testing.T) {
+	tests := []struct {
+		name string
+		v    *VM
+	}{
+		{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
+		{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
+		{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
+		{name: "no boot mode", v: &VM{}},
+		{name: "microvm", v: &VM{BootMode: "uefi", DirectBoot: true}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.NoError(t, CheckUEFIVarStore(tt.v))
+		})
+	}
+}
+
+// A launch from a pointer opens this node's copy as pflash unit 1 with
+// format=raw, after the read-only CODE blob.
+func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
+	f := newUEFIFixture(t)
+	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
+	launches := withFakeQEMU(t)
+	key := keyFor("boot")
+	f.writeLocal(t, key, region("seeded"))
+	instance := f.instance(key, f.identity)
+	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
+
+	require.NoError(t, f.m.startQEMU(instance, false))
+
+	got := launches()
+	require.Len(t, got, 1)
+	vars := filepath.Join(f.instanceDir(), "vars.fd")
+	assert.Equal(t, []string{
+		"file=" + f.codePath + ",format=raw,if=pflash,unit=0,readonly=on",
+		"file=" + vars + ",format=raw,if=pflash,unit=1",
+	}, allArgValues(got[0], "-drive"))
+}
+
+// A reboot reuses the copy the launch seeded; without one there is nothing
+// safe to boot, so QEMU is not started.
+func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
+	f := newUEFIFixture(t)
+	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
+	launches := withFakeQEMU(t)
+	instance := f.instance(keyFor("boot"), f.identity)
+	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
+
+	err := f.m.startQEMU(instance, false)
+
+	require.ErrorIs(t, err, os.ErrNotExist)
+	assert.Empty(t, launches())
+}
+
+func TestUEFIInstanceDir_RefusesIDsThatAreNotOnePathSegment(t *testing.T) {
+	for _, id := range []string{"", "..", "i-../x", "i-a/b", `i-a\b`, "vol-1"} {
+		_, err := uefiInstanceDir("/data", id)
+		assert.Error(t, err, id)
+	}
+	dir, err := uefiInstanceDir("/data", testUEFIInstance)
+	require.NoError(t, err)
+	assert.Equal(t, "/data/uefi/"+testUEFIInstance, dir)
+}
+
+// The CODE hash is what a store records; a package update that replaces the
+// blob must be hashed again rather than served from the cache.
+func TestNodeFirmwareFor_HashesTheCodeBlobAgainAfterItChanges(t *testing.T) {
+	f := newUEFIFixture(t)
+	first, err := NodeFirmwareFor("x86_64")
+	require.NoError(t, err)
+	assert.Equal(t, f.identity, first.Identity)
+
+	updated := []byte("ovmf code build 2, a longer blob")
+	require.NoError(t, os.WriteFile(f.codePath, updated, 0o600))
+	second, err := NodeFirmwareFor("x86_64")
+	require.NoError(t, err)
+
+	sum := sha256.Sum256(updated)
+	assert.Equal(t, hex.EncodeToString(sum[:]), second.Identity.CodeSHA256)
+	assert.Equal(t, f.codePath, second.Code)
+}

spinifex/runtime/compute/vm/vm.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
@@ -308,6 +309,20 @@ type VM struct {
 	// tolerated by migrate.
 	InstanceLifecycle     string `json:"instance_lifecycle,omitempty"`
 	SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
+
+	// UEFIVarStore points at the object holding this instance's UEFI variable
+	// store. Status, so written only by the node; nil for a BIOS instance.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
+}
+
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string                     `json:"key"`
+	Firmware   uefistore.FirmwareIdentity `json:"firmware"`
+	WriterNode string                     `json:"writer_node"`
+	WrittenAt  time.Time                  `json:"written_at"`
 }
 
 // UnmarshalJSON folds the fields of legacy records that mixed a request with an
@@ -372,7 +387,7 @@ type Device struct {
 }
 
 // Drive is one QEMU -drive entry, used for the pflash firmware drives (the
-// CODE blob and the EFI volume). Each set field becomes a key=value option.
+// CODE blob and the variable store). Each set field becomes a key=value option.
 type Drive struct {
 	File   string `json:"file"`
 	Format string `json:"format"`
@@ -381,7 +396,7 @@ type Drive struct {
 	ID     string `json:"id"`
 	Cache  string `json:"cache,omitempty"`
 	// Unit selects the pflash slot when If=="pflash": 0 is the CODE blob,
-	// 1 is the per-VM VARS volume. Ignored for non-pflash drives.
+	// 1 is the per-VM variable store. Ignored for non-pflash drives.
 	Unit int `json:"unit,omitempty"`
 	// ReadOnly emits readonly=on; used for the pflash CODE blob.
 	ReadOnly bool `json:"readonly,omitempty"`
@@ -573,8 +588,8 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 			return nil, fwErr
 		}
 		// pflash CODE must come before VARS so QEMU loads firmware into the
-		// readonly slot first; buildDrives emits the VARS drive (unit 1) from
-		// the per-VM EFI viperblock volume.
+		// readonly slot first; startQEMU emits the VARS drive (unit 1) from
+		// the node's copy of the variable store or the EFI volume.
 		drives = append([]Drive{{If: "pflash", Format: "raw", Unit: 0, ReadOnly: true, File: codePath}}, drives...)
 	}
 

spinifex/runtime/daemon/daemon.go

@@ -152,6 +152,7 @@ type Daemon struct {
 	natsConn          *nats.Conn
 	resourceMgr       *ResourceManager
 	instanceService   *ec2instance.InstanceServiceImpl
+	instanceObjects   objectstore.ObjectStore
 	dnsWriter         *dns.Writer
 	dnsReconciler     *dns.Reconciler
 	dnsBaseDomain     string
@@ -2017,6 +2018,7 @@ func (d *Daemon) startCluster() error {
 
 	// Create services before loading/launching instances, since LaunchInstance depends on them
 	store := objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(d.config.Predastore.Host), d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey)
+	d.instanceObjects = store
 	d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, d.resourceMgr.instanceTypes, d.natsConn, store, d.vmMgr, d.resourceMgr, d.jsManager)
 	d.dnsWriter = dns.NewWriter(d.config, d.clusterConfig, d.natsConn)
 	d.dnsReconciler = dns.NewReconciler(d.config, d.clusterConfig, d.natsConn, d.dnsWriter, d.dnsDesiredSet, d.dnsWatchSources()...)
@@ -2754,6 +2756,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitClusterConfigBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitUEFIStoreBucket()
+		}
+
 		if err == nil {
 			d.jsManager.SetSyncObserver(d)
 			// Replicas cannot be undeclared here: bucket creation refuses

spinifex/runtime/daemon/jetstream.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
@@ -88,6 +89,7 @@ type JetStreamManager struct {
 	termRecords   *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
 	clusterKV     jetstream.KeyValue                // spinifex-cluster-state
 	clusterConfig *kvstore.Bucket                   // spinifex-cluster-config
+	uefiStores    *uefistore.Store                  // spinifex-uefi-stores
 	obs           KVSyncObserver
 	running       runningSetState
 }
@@ -232,6 +234,18 @@ func (m *JetStreamManager) InitClusterConfigBucket() error {
 	return err
 }
 
+// InitUEFIStoreBucket opens the UEFI store-record bucket, creating it if it
+// doesn't exist, so a bucket that cannot be created fails startup.
+func (m *JetStreamManager) InitUEFIStoreBucket() error {
+	m.uefiStores = uefistore.NewStore(m.js)
+	return m.uefiStores.Open(context.Background())
+}
+
+// UEFIStores returns the store-record bucket opened by InitUEFIStoreBucket.
+func (m *JetStreamManager) UEFIStores() *uefistore.Store {
+	return m.uefiStores
+}
+
 // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
 // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
 func (m *JetStreamManager) InitTerminatedInstanceBucket() error {

spinifex/runtime/daemon/local_state.go

@@ -42,10 +42,16 @@ type LocalState struct {
 // LocalStatePath returns the absolute path to the per-node instance state file.
 // Empty dataDir falls back to DefaultDataDir.
 func LocalStatePath(dataDir string) string {
+	return filepath.Join(DataDirOrDefault(dataDir), DefaultLocalStateDir, LocalStateFileName)
+}
+
+// DataDirOrDefault is the daemon's persistent data directory: dataDir, or
+// DefaultDataDir when it is empty.
+func DataDirOrDefault(dataDir string) string {
 	if dataDir == "" {
-		dataDir = DefaultDataDir
+		return DefaultDataDir
 	}
-	return filepath.Join(dataDir, DefaultLocalStateDir, LocalStateFileName)
+	return dataDir
 }
 
 // MarshalLocalState produces the JSON wire form of vms with the lowest schema

spinifex/runtime/daemon/uefi_store_bucket_test.go

@@ -0,0 +1,36 @@
+package daemon_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The daemon creates the store-record bucket at startup and hands out the
+// store it opened, so later readers share one handle.
+func TestInitUEFIStoreBucket_CreatesBucketAndServesTheStore(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	m, err := daemon.NewJetStreamManager(nc)
+	require.NoError(t, err)
+
+	require.NoError(t, m.InitUEFIStoreBucket())
+
+	_, err = js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err, "init must create the bucket rather than leave it to first use")
+
+	rec := &uefistore.UEFIStoreRecord{InstanceID: "i-0123456789abcdef0", State: uefistore.StateLive}
+	rev, err := m.UEFIStores().Create(t.Context(), rec)
+	require.NoError(t, err)
+	got, gotRev, err := m.UEFIStores().Get(t.Context(), rec.InstanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, rec, got)
+
+	_, err = m.UEFIStores().Create(t.Context(), rec)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+}

spinifex/runtime/daemon/vm_adapters.go

@@ -696,6 +696,11 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 		DeviceDeletedTimeout:       d.deviceDeletedTimeout,
 		ConsumeCleanShutdownMarker: d.consumeCleanShutdownMarker(),
 		BackingStoreReady:          func() bool { return d.checkPredastoreReady() && d.checkViperblockReady() },
+		UEFIVarStore: vm.UEFIVarStoreDeps{
+			Objects: d.instanceObjects,
+			Bucket:  d.config.Predastore.Bucket,
+			DataDir: DataDirOrDefault(d.config.DataDir),
+		},
 	}
 }
 

spinifex/runtime/daemon/vm_adapters_coverage_test.go

@@ -190,6 +190,24 @@ func TestBuildVMManagerDeps_BackingStoreReady(t *testing.T) {
 		"no predastore configured and JetStream answering is ready")
 }
 
+// Variable stores live in the instance domain's bucket, read through the
+// same object store, with the live copy under the persistent data directory.
+func TestBuildVMManagerDeps_UEFIVarStore(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	d := &Daemon{
+		config:          &config.Config{DataDir: "/srv/spinifex", Predastore: config.PredastoreConfig{Bucket: "spinifex-instances"}},
+		vmMgr:           vm.NewManager(),
+		instanceObjects: objects,
+	}
+	got := d.buildVMManagerDeps().UEFIVarStore
+	assert.Same(t, objects, got.Objects)
+	assert.Equal(t, "spinifex-instances", got.Bucket)
+	assert.Equal(t, "/srv/spinifex", got.DataDir)
+
+	d.config.DataDir = ""
+	assert.Equal(t, DefaultDataDir, d.buildVMManagerDeps().UEFIVarStore.DataDir)
+}
+
 // --- volumeMounterAdapter ---
 
 func TestVolumeMounterAdapter_Mount_ClosedConnIsNotRetryable(t *testing.T) {