Compare commits

...

5 Commits

Author SHA1 Message Date
brynmailer-mdc c826cbd158 fix: confirm a stale or absent UEFI holder heartbeat with the stream leader before takeover 2026-11-05 17:22:21 +10:00
brynmailer-mdc 82b3b55d70 fix: check UEFI firmware before the claim and never overwrite a copy the claim guards
A start checks the pointer's firmware before it touches the claim, reuses
vars.fd whenever seeded.json names the claim's generation, and refuses as
retryable, keeping the file and the claim, when seeded.json cannot be read.
A rollback releases the claim only when QEMU never started and no local
copy may owe a write-back. An EFI-volume instance reads no store record,
and a pointer move to the current key is still confirmed by CAS.
2026-11-05 17:22:20 +10:00
brynmailer-mdc c864bfbc84 fix: keep a UEFI claim guarding a local copy and refuse on an undated heartbeat
A launch that fails after adopting its claim keeps it while seeded.json
names the claim's generation, or cannot be read. A heartbeat that exists but
cannot be dated refuses the takeover. A stamp at the record's own
generation is written by CAS, so a follower's stale pointer is never booted.
2026-11-05 17:22:20 +10:00
brynmailer-mdc c9c6dae176 fix: refuse a start whose UEFI store a live node holds as InsufficientInstanceCapacity 2026-11-05 17:22:20 +10:00
brynmailer-mdc c99ff34d53 feat: claim the UEFI variable store and fence its pointer by generation
The store record now carries the boot pointer and the claim generation that
may move it. A launch claims the store through kvsync, takes it over only once
the holder's heartbeat is LivenessWindow old, stamps the record with its
generation, and boots from the pointer it read after the stamp. A record above
the claim's generation refuses the start as retryable and changes nothing.
The store reaper removes the claim before it closes the record.
2026-11-05 17:22:20 +10:00
27 changed files with 1787 additions and 381 deletions
+25 -1
View File
@@ -1,6 +1,9 @@
package clusterv1
import "time"
import (
"encoding/json"
"time"
)
// ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
// other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
return "heartbeat." + node
}
// LivenessWindow is how recently a node must have heartbeated for a reader
// deciding on a takeover to count it live. It is six heartbeats, compared
// across node clocks, where NTP skew is small against it.
const LivenessWindow = 60 * time.Second
// LiveAt reports whether a heartbeat written at written still vouches for its
// node at now: it does while it is under LivenessWindow old.
func LiveAt(written, now time.Time) bool {
return now.Sub(written) < LivenessWindow
}
// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
// or dated vouches for nothing, which is an answer rather than a fault.
func HeartbeatWritten(data []byte) (time.Time, bool) {
var beat HeartbeatTimestamp
if json.Unmarshal(data, &beat) != nil {
return time.Time{}, false
}
return beat.Time()
}
// HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
// decodes only the timestamp, so a reader does not depend on the capacity
// figures the daemon writes beside it.
+20
View File
@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
assert.False(t, ok, "timestamp %q", ts)
}
}
// A beat is live strictly under the window, so one exactly the window old is
// the first a takeover may treat as stopped.
func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
}
func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
require.True(t, ok)
assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
for _, raw := range []string{"not json", `{"node":"node-a"}`} {
_, ok := clusterv1.HeartbeatWritten([]byte(raw))
assert.False(t, ok, raw)
}
}
@@ -1265,6 +1265,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
return refusal
}
if errors.Is(err, vm.ErrUEFIClaimHeld) {
slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
return errInsufficientCapacity
}
slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
return errors.New(awserrors.ErrorServerInternal)
}
@@ -2760,7 +2764,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
// Before anything is stamped or claimed, so the record handed back is the
// one that was claimed.
if err := vm.CheckUEFIVarStore(instance); err != nil {
if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
"instanceId", instance.ID, "err", err)
undo()
@@ -2893,6 +2897,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
if refusal := volumeHeldElsewhereError(err); refusal != nil {
return refusal
}
// A live node holds the variable store, so another node may take it.
if errors.Is(err, vm.ErrUEFIClaimHeld) {
return s.capacityRefusal(ctx, instance)
}
return errors.New(awserrors.ErrorServerInternal)
}
@@ -25,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
return s.kv.Exists(ctx, instanceID)
}
// MarkDeleting records that instanceID's store is to be deleted, and drops any
// write-back still owed. It reports false with no error when the instance has
// no store record. Only a terminate or an operator's forget may call it.
// MarkDeleting records that instanceID's store is to be deleted. It reports
// false with no error when the instance has no store record. Only a terminate
// or an operator's forget may call it.
func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
for range markAttempts {
rec, rev, err := s.Get(ctx, instanceID)
@@ -41,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
return true, nil
}
rec.State = StateDeleting
rec.WriteBack = nil
_, err = s.CompareAndSet(ctx, rec, rev)
if err == nil {
return true, nil
@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
}
// Deletion is final: a writer holding the revision of a deleting record, such
// as a write-back retry, must not bring the store back to live.
// as a late pointer move, must not bring the store back to live.
func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
s, _ := openStore(t)
rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
rev, err = s.CompareAndSet(t.Context(), deleting, rev)
require.NoError(t, err)
revived := liveRecord()
revived.WriteBack = writeBack()
revived := pointedRecord()
_, err = s.CompareAndSet(t.Context(), revived, rev)
require.ErrorIs(t, err, uefistore.ErrDeleting)
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
assert.Equal(t, uefistore.StateDeleting, got.State)
}
// A write-back owed on a deleting store would put an object behind the delete.
func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
s, _ := openStore(t)
rev, err := s.Create(t.Context(), liveRecord())
require.NoError(t, err)
rec := liveRecord()
rec.State = uefistore.StateDeleting
rec.WriteBack = writeBack()
_, err = s.CompareAndSet(t.Context(), rec, rev)
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
}
func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
s, _ := openStore(t)
rec := liveRecord()
rec.WriteBack = writeBack()
_, err := s.Create(t.Context(), rec)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
got, rev, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, uefistore.StateDeleting, got.State)
assert.Nil(t, got.WriteBack)
assert.Equal(t, "111122223333", got.AccountID)
assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
has, err = s.MarkDeleting(t.Context(), instanceID)
require.NoError(t, err)
@@ -0,0 +1,93 @@
package uefistore
import (
"context"
"errors"
"fmt"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
)
// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
// is ClaimPrefix+instanceID, and names the node that may hold variable changes
// no object has yet.
const ClaimPrefix = "uefi."
// NewClaims returns node's group of store claims over store. The daemon builds
// it once per process, as kvsync requires of every prefix.
func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
return kvsync.NewClaimGroup(store, ClaimPrefix, node)
}
// ErrFenced is a stamp or pointer move refused because the record carries a
// claim generation other than the caller's: a later acquisition has stamped
// it, so the caller no longer owns the store and must change nothing.
var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
const fenceAttempts = 5
// Stamp raises instanceID's record to generation, the claim token the caller
// has just acquired or adopted, and returns the record it stamped, whose
// pointer the caller boots from. One above generation is ErrFenced, and one
// deleting is ErrDeleting. A record already at generation is still written by
// CAS, because a follower can answer the read with an older pointer.
func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
if generation == 0 {
return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
}
for range fenceAttempts {
rec, rev, err := s.Get(ctx, instanceID)
if err != nil {
return nil, err
}
switch {
case rec.State == StateDeleting:
return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
case rec.Generation > generation:
return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
}
rec.Generation = generation
_, err = s.CompareAndSet(ctx, rec, rev)
if err == nil {
return rec, nil
}
if !errors.Is(err, kvstore.ErrConflict) {
return nil, err
}
}
return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
}
// MovePointer points instanceID's record at ref only while the record is at
// generation. A record at any other generation is ErrFenced and is left
// unchanged. Every move is a CAS at the revision read, even to the current
// key, so a read a follower answered stale cannot report success.
func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
if err := ref.validate(instanceID); err != nil {
return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
}
for range fenceAttempts {
rec, rev, err := s.Get(ctx, instanceID)
if err != nil {
return err
}
switch {
case rec.State == StateDeleting:
return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
case rec.Generation != generation:
return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
}
rec.Pointer = &ref
_, err = s.CompareAndSet(ctx, rec, rev)
if err == nil {
return nil
}
if !errors.Is(err, kvstore.ErrConflict) {
return err
}
}
return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
}
@@ -0,0 +1,213 @@
package uefistore_test
import (
"testing"
testutil "github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
t.Helper()
_, rev, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
return rev
}
func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
s, _ := openStore(t)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
rec, err := s.Stamp(t.Context(), instanceID, 9)
require.NoError(t, err)
assert.Equal(t, uint64(9), rec.Generation)
assert.Equal(t, pointer("b"), rec.Pointer)
got, _, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, rec, got, "the returned record is the one stored")
}
// A daemon restart adopts its claim at the same generation. The stamp still
// writes by CAS, so a pointer read from a lagging follower fails the write and
// is read again rather than booted from.
func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
s, _ := openStore(t)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
before := revisionOf(t, s)
rec, err := s.Stamp(t.Context(), instanceID, 7)
require.NoError(t, err)
assert.Equal(t, pointer("b"), rec.Pointer)
assert.Equal(t, uint64(7), rec.Generation)
assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
}
func TestStamp_RefusesAndChangesNothing(t *testing.T) {
tests := []struct {
name string
deleting bool
generation uint64
want error
}{
{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, _ := openStore(t)
rev, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
if tt.deleting {
_, err = s.MarkDeleting(t.Context(), instanceID)
require.NoError(t, err)
rev = revisionOf(t, s)
}
_, err = s.Stamp(t.Context(), instanceID, tt.generation)
require.ErrorIs(t, err, tt.want)
assert.Equal(t, rev, revisionOf(t, s))
})
}
}
func TestStamp_MissingRecordIsNotFound(t *testing.T) {
s, _ := openStore(t)
_, err := s.Stamp(t.Context(), instanceID, 1)
require.ErrorIs(t, err, kvstore.ErrNotFound)
}
// The record keeps only the current version, so each move replaces the
// pointer and names nothing it replaced.
func TestMovePointer_ReplacesThePointer(t *testing.T) {
s, _ := openStore(t)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
for _, name := range []string{"c", "d"} {
require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
}
got, _, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, pointer("d"), got.Pointer)
assert.Equal(t, uint64(7), got.Generation)
}
// A move to the key the record already names is still a CAS at the revision
// read, so a fenced writer that read a lagging follower cannot see success.
func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
s, _ := openStore(t)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
before := revisionOf(t, s)
require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
assert.Greater(t, revisionOf(t, s), before)
}
func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
tests := []struct {
name string
deleting bool
generation uint64
ref *uefistore.UEFIVarStoreRef
want error
}{
{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
{name: "a key under another instance", generation: 7,
ref: &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
want: uefistore.ErrInvalidRecord},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, _ := openStore(t)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
if tt.deleting {
_, err = s.MarkDeleting(t.Context(), instanceID)
require.NoError(t, err)
}
rev := revisionOf(t, s)
err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
require.ErrorIs(t, err, tt.want)
got, gotRev, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, rev, gotRev)
assert.Equal(t, pointer("b"), got.Pointer)
})
}
}
// claimedStore is a store record and a claim group per node over one
// JetStream, as two daemons see them.
type claimedStore struct {
records *uefistore.Store
a, b *kvsync.Claim
}
func newClaimedStore(t *testing.T) *claimedStore {
t.Helper()
_, _, js := testutil.StartTestJetStream(t)
records := uefistore.NewStore(js)
require.NoError(t, records.Open(t.Context()))
_, err := records.Create(t.Context(), liveRecord())
require.NoError(t, err)
return &claimedStore{
records: records,
a: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
b: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
}
}
// A node partitioned long enough to be taken over is fenced by the new
// owner's stamp: its pointer move fails and the record keeps what it had.
func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
c := newClaimedStore(t)
first, err := c.a.TryLock(t.Context())
require.NoError(t, err)
_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
require.NoError(t, err)
require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
second, err := c.b.TakeOver(t.Context(), first)
require.NoError(t, err)
_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
require.NoError(t, err)
rev := revisionOf(t, c.records)
err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
require.ErrorIs(t, err, uefistore.ErrFenced)
got, gotRev, err := c.records.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, rev, gotRev)
assert.Equal(t, pointer("b"), got.Pointer)
assert.Equal(t, second.Generation, got.Generation)
}
// The new owner reads the pointer after it stamps, so a move the old holder
// landed between the takeover and the stamp is the one it boots from.
func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
c := newClaimedStore(t)
first, err := c.a.TryLock(t.Context())
require.NoError(t, err)
_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
require.NoError(t, err)
second, err := c.b.TakeOver(t.Context(), first)
require.NoError(t, err)
require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
require.NoError(t, err)
assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
}
@@ -1,6 +1,6 @@
// Package uefistore holds the types that describe an instance's UEFI variable
// store and the bucket of store records that carry its deletion and write-back
// obligations past the life of the instance record.
// store, the bucket of store records that carry its pointer and deletion
// obligation past the life of the instance record, and the claim that fences it.
package uefistore
import (
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
// UEFIStoreRecord is the durable owner of one instance's variable store
// objects. It outlives the instance record, so an obligation recorded here is
// not lost when a terminated record expires.
//
// Pointer names the only version of the store the record keeps, the object the
// instance boots from, and only a writer whose claim token has generation
// Generation may move it.
type UEFIStoreRecord struct {
SchemaVersion uint16 `json:"schema_version"`
InstanceID string `json:"instance_id"`
AccountID string `json:"account_id"`
State string `json:"state"`
WriteBack *UEFIWriteBack `json:"write_back,omitempty"`
SchemaVersion uint16 `json:"schema_version"`
InstanceID string `json:"instance_id"`
AccountID string `json:"account_id"`
State string `json:"state"`
Generation uint64 `json:"generation"`
Pointer *UEFIVarStoreRef `json:"pointer,omitempty"`
}
// UEFIWriteBack is a write-back that failed and is still owed. Key is the
// object every retry puts to, so retries leave no extra objects; SeededFrom is
// the pointer Node seeded from, which a retry must still find on the record.
type UEFIWriteBack struct {
Node string `json:"node"`
Key string `json:"key"`
SeededFrom string `json:"seeded_from"`
Since time.Time `json:"since"`
// UEFIVarStoreRef names the store object an instance launches from, with the
// firmware it was written under, so a launch can check it without reading the
// object's metadata.
type UEFIVarStoreRef struct {
Key string `json:"key"`
Firmware FirmwareIdentity `json:"firmware"`
WriterNode string `json:"writer_node"`
WrittenAt time.Time `json:"written_at"`
}
// KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
return ok && name != "" && !strings.Contains(name, "/")
}
// validate checks rec against key. A write-back key outside the instance's own
// prefix is refused because a superseded obligation deletes that key.
// validate checks rec against key. The pointer must name one object directly
// under the instance's own prefix, because a deletion acts on it.
func (rec *UEFIStoreRecord) validate(key string) error {
switch {
case rec.SchemaVersion > RecordSchemaVersion:
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
case rec.State != StateLive && rec.State != StateDeleting:
return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
case rec.State == StateDeleting && rec.WriteBack != nil:
// A write-back retry puts an object, which would land behind the delete.
return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
}
if wb := rec.WriteBack; wb != nil {
switch {
case wb.Node == "":
return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
case wb.Since.IsZero():
return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
case !ownsKey(rec.InstanceID, wb.Key):
return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
case !ownsKey(rec.InstanceID, wb.SeededFrom):
return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
if p := rec.Pointer; p != nil {
if err := p.validate(rec.InstanceID); err != nil {
return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
}
}
return nil
}
// validate checks ref as a pointer of instanceID's store.
func (ref *UEFIVarStoreRef) validate(instanceID string) error {
switch {
case !ownsKey(instanceID, ref.Key):
return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
case ref.WriterNode == "":
return fmt.Errorf("key %q names no writer node", ref.Key)
}
return nil
}
func bucketConfig() kvstore.Config {
return kvstore.Config{
Name: BucketName,
Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
History: 1,
// Left false: a lost bucket has lost obligations, and an empty
// recreated one would report every store as having none.
@@ -30,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
}
func writeBack() *uefistore.UEFIWriteBack {
return &uefistore.UEFIWriteBack{
Node: "node-2",
Key: uefistore.KeyPrefix(instanceID) + "b",
SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
Since: time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
func pointer(name string) *uefistore.UEFIVarStoreRef {
return &uefistore.UEFIVarStoreRef{
Key: keyOf(name),
Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
WriterNode: "node-2",
WrittenAt: time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
}
}
func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
// pointedRecord is a live record stamped at generation 7 that points at "b".
func pointedRecord() *uefistore.UEFIStoreRecord {
rec := liveRecord()
rec.Generation = 7
rec.Pointer = pointer("b")
return rec
}
func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
t.Helper()
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -66,8 +76,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
s, _ := openStore(t)
rec := liveRecord()
rec.WriteBack = writeBack()
rec := pointedRecord()
rev, err := s.Create(t.Context(), rec)
require.NoError(t, err)
@@ -82,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
// The wire names are what every release reads, so they are pinned.
func TestRecord_WireShape(t *testing.T) {
s, js := openStore(t)
rec := liveRecord()
rec.WriteBack = writeBack()
_, err := s.Create(t.Context(), rec)
_, err := s.Create(t.Context(), pointedRecord())
require.NoError(t, err)
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -96,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
"instance_id": "i-0123456789abcdef0",
"account_id": "111122223333",
"state": "live",
"write_back": {
"node": "node-2",
"generation": 7,
"pointer": {
"key": "uefi/i-0123456789abcdef0/b",
"seeded_from": "uefi/i-0123456789abcdef0/a",
"since": "2026-11-03T01:02:03Z"
"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
"writer_node": "node-2",
"written_at": "2026-11-03T01:02:03Z"
}
}`, string(entry.Value()))
}
@@ -144,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
first, err := s.Create(t.Context(), liveRecord())
require.NoError(t, err)
moved := liveRecord()
moved.WriteBack = writeBack()
moved := pointedRecord()
second, err := s.CompareAndSet(t.Context(), moved, first)
require.NoError(t, err)
assert.Greater(t, second, first)
@@ -159,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, second, rev)
assert.Equal(t, uefistore.StateLive, got.State)
assert.Equal(t, writeBack(), got.WriteBack)
assert.Equal(t, pointer("b"), got.Pointer)
}
// Two actors that read the same revision race to change it; exactly one may
@@ -179,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
)
for i := range writers {
wg.Go(func() {
rec := liveRecord()
rec.WriteBack = writeBack()
rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
rec := pointedRecord()
rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
mu.Lock()
wins = append(wins, rec.WriteBack.Node)
wins = append(wins, rec.Pointer.WriterNode)
mu.Unlock()
} else {
assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -196,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
require.Len(t, wins, 1)
got, _, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, wins[0], got.WriteBack.Node)
assert.Equal(t, wins[0], got.Pointer.WriterNode)
}
func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -217,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
require.ErrorIs(t, err, kvstore.ErrNotFound)
}
// A superseded obligation deletes its recorded key, so a key outside the
// A deletion acts on the key the record points at, so a key outside the
// instance's own prefix must never reach the bucket.
func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
cases := map[string]func(*uefistore.UEFIStoreRecord){
"unknown state": func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
"not an instance ID": func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
"write-back with no node": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
},
"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
},
"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
},
"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
},
"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
},
"unknown state": func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
"not an instance ID": func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
"pointer under another instance": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
"pointer under an image": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
"pointer is the prefix itself": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
"pointer with no firmware arch": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
"pointer with no VARS size": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
"pointer with no writer node": func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
}
for name, mutate := range cases {
t.Run(name, func(t *testing.T) {
@@ -247,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
rev, err := s.Create(t.Context(), liveRecord())
require.NoError(t, err)
rec := liveRecord()
rec.WriteBack = writeBack()
rec := pointedRecord()
mutate(rec)
_, err = s.CompareAndSet(t.Context(), rec, rev)
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -256,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
got, gotRev, err := s.Get(t.Context(), instanceID)
require.NoError(t, err)
assert.Equal(t, rev, gotRev, "an invalid record must not be written")
assert.Nil(t, got.WriteBack)
assert.Nil(t, got.Pointer)
})
}
}
@@ -265,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
// is reported rather than decoded into something a reaper would act on.
func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
cases := map[string]string{
"newer schema": `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
"no schema": `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
"unknown state": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
"newer schema": `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
"no schema": `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
"unknown state": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
"foreign pointer": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
}
for name, body := range cases {
t.Run(name, func(t *testing.T) {
@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
})
}
// uefiClaimHeldMounter fails the launch the way a start does while a live
// node holds the instance's UEFI variable store.
type uefiClaimHeldMounter struct{ raceVolumeMounter }
func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
}
// A held UEFI store is a capacity refusal on both start paths, so the caller
// tries another node rather than reporting a fault.
func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
types, _ := defaultPrepareInstanceTypes()
t.Run("claimed relaunch", func(t *testing.T) {
mgr := vm.NewManager()
mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
})
t.Run("live owner", func(t *testing.T) {
const id = "i-uefi-live"
mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
v, _ := mgr.Get(id)
svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
})
}
@@ -2,7 +2,6 @@ package viperblock
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
@@ -14,9 +13,8 @@ import (
)
// nodeLivenessWindow is how recently a node must have been seen for its dirty
// marker to bar a takeover. It is six heartbeats and above volumeLeaseTTL, and is
// compared across node clocks, where NTP skew is small against 60s.
const nodeLivenessWindow = 60 * time.Second
// marker to bar a takeover: the cluster's heartbeat rule, above volumeLeaseTTL.
const nodeLivenessWindow = clusterv1.LivenessWindow
// nodeLiveness reports when a node was last seen alive. found is false for a
// node that has never been seen or whose record has expired; an error means the
@@ -75,16 +73,6 @@ func (l *heartbeatLiveness) LastSeen(ctx context.Context, node string) (time.Tim
if err != nil {
return time.Time{}, false, fmt.Errorf("read heartbeat for %s: %w", node, err)
}
written, ok := heartbeatTime(entry.Value())
written, ok := clusterv1.HeartbeatWritten(entry.Value())
return written, ok, nil
}
// heartbeatTime dates a stored heartbeat. One that cannot be read vouches for
// nothing, which is an answer rather than a fault.
func heartbeatTime(data []byte) (time.Time, bool) {
var beat clusterv1.HeartbeatTimestamp
if json.Unmarshal(data, &beat) != nil {
return time.Time{}, false
}
return beat.Time()
}
@@ -8,6 +8,7 @@ import (
"log/slog"
"time"
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
@@ -266,10 +267,11 @@ func (cfg *Config) refuseWhileOwnerLive(ctx context.Context, volumeName string,
if !found {
return nil
}
age := time.Since(seen)
if age > nodeLivenessWindow {
now := time.Now()
if !clusterv1.LiveAt(seen, now) {
return nil
}
age := now.Sub(seen)
slog.WarnContext(ctx, "refusing to open a volume whose previous owner is still live",
"volume", volumeName, "previous_owner", record.Owner, "last_seen_ms", telemetry.Millis(age))
return fmt.Errorf("%w: %s is still live and holds unsealed writes", errVolumeLeaseHeld, record.Owner)
+44 -12
View File
@@ -19,6 +19,7 @@ import (
"time"
telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
@@ -82,7 +83,7 @@ func guestOOMScore(managedBy string) int {
// transition to Running, fire OnInstanceUp. Inserts the instance before
// transitioning. Used by RunInstances, start-stopped handler, restore, and crash recovery.
func (m *Manager) Run(ctx context.Context, instance *VM) error {
return m.launch(ctx, instance)
return m.launch(ctx, instance, nil)
}
// Start re-launches a stopped instance by id. Returns ErrInstanceNotFound when
@@ -92,7 +93,8 @@ func (m *Manager) Start(ctx context.Context, id string) error {
if !ok {
return fmt.Errorf("%w: %s", ErrInstanceNotFound, id)
}
if err := CheckUEFIVarStore(instance); err != nil {
rec, err := m.checkUEFIVarStore(ctx, instance)
if err != nil {
return err
}
// StateError is rejected by launchStillValid; move to Pending first so
@@ -102,7 +104,13 @@ func (m *Manager) Start(ctx context.Context, id string) error {
return err
}
}
return m.launch(ctx, instance)
return m.launch(ctx, instance, &uefiPrecheck{rec: rec})
}
// uefiPrecheck is a variable store check a caller of launch already made,
// so launch does not read the store record again.
type uefiPrecheck struct {
rec *uefistore.UEFIStoreRecord
}
// launchStillValid returns true when status is still pending/stopped/provisioning.
@@ -117,8 +125,9 @@ func (m *Manager) launchStillValid(instance *VM) bool {
}
// launch is the orchestrator: pid check, mount volumes, exec QEMU, attach
// QMP, fire OnInstanceUp, transition to Running.
func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
// QMP, fire OnInstanceUp, transition to Running. A nil pre checks the
// variable store here.
func (m *Manager) launch(ctx context.Context, instance *VM, pre *uefiPrecheck) (err error) {
ctx, span := otel.Tracer(vmTracerName).Start(ctx, "vm.launch",
trace.WithAttributes(
attribute.String("instance.id", instance.ID),
@@ -131,10 +140,14 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
}
// Before the reason below is cleared, so a refused start changes nothing.
if err := CheckUEFIVarStore(instance); err != nil {
slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
"instanceId", instance.ID, "err", err)
return err
if pre == nil {
rec, err := m.checkUEFIVarStore(ctx, instance)
if err != nil {
slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
"instanceId", instance.ID, "err", err)
return err
}
pre = &uefiPrecheck{rec: rec}
}
// A reason left by the stop, or by a restore that found this node could not
@@ -159,10 +172,17 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
}
// Before the mount, so a store this node cannot boot fails the launch
// with nothing to roll back.
// with nothing to roll back but the claim, which a launch that fails
// before QEMU starts gives back.
uefiCtx, uefiSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.uefi_var_store")
uefiErr := m.prepareUEFIVarStore(uefiCtx, instance)
uefiHeld, uefiErr := m.prepareUEFIVarStore(uefiCtx, instance, pre.rec)
endSpanWithError(uefiSpan, uefiErr)
guestMayHaveRun := false
defer func() {
if err != nil && !guestMayHaveRun {
m.releaseUEFIClaim(ctx, instance.ID, uefiHeld)
}
}()
if uefiErr != nil {
slog.ErrorContext(ctx, "Failed to prepare the UEFI variable store", "instanceId", instance.ID, "err", uefiErr)
return uefiErr
@@ -197,6 +217,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
_, qemuSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.start_qemu")
qemuErr := m.startQEMU(instance)
guestMayHaveRun = !errors.Is(qemuErr, errQEMUNotStarted)
endSpanWithError(qemuSpan, qemuErr)
if qemuErr != nil {
slog.ErrorContext(ctx, "Failed to launch instance", "err", qemuErr)
@@ -301,10 +322,20 @@ func qemuStderrLevel(line string) slog.Level {
return slog.LevelError
}
// errQEMUNotStarted wraps a startQEMU failure from before the QEMU process
// started, so no guest can have run.
var errQEMUNotStarted = errors.New("qemu process was not started")
// startQEMU launches the QEMU process for instance and waits for startup to
// confirm. The console log is opened in append mode, so a replacement process
// keeps what the guest printed before it.
func (m *Manager) startQEMU(instance *VM) error {
func (m *Manager) startQEMU(instance *VM) (err error) {
started := false
defer func() {
if err != nil && !started {
err = fmt.Errorf("%w: %w", errQEMUNotStarted, err)
}
}()
pidFile, err := hostprocess.GeneratePidFile(instance.ID)
if err != nil {
slog.Error("Failed to generate PID file", "err", err)
@@ -641,6 +672,7 @@ func (m *Manager) startQEMU(instance *VM) error {
if pid == 0 {
return fmt.Errorf("failed to start qemu")
}
started = true
// Race QMP-socket appearance against early process exit; SIGKILL on timeout
// so a wedged QEMU does not orphan its tap / volumes.
-6
View File
@@ -107,10 +107,6 @@ type InstanceStatus struct {
EBSRequests []EBSRequest `json:"ebs_requests,omitempty"`
ENIAvailableSlots []int `json:"eni_available_slots,omitempty"`
ENIAttachedByID map[string]int `json:"eni_attached_by_id,omitempty"`
// UEFIVarStore is optional, so records written without it still decode
// and the bucket schema does not change.
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
}
// Record projects a VM onto the persisted record. QMPClient and the request
@@ -172,7 +168,6 @@ func (v *VM) Record() *InstanceRecord {
EBSRequests: v.EBSRequests.Requests,
ENIAvailableSlots: v.ENIRequests.AvailableSlots,
ENIAttachedByID: v.ENIRequests.AttachedByENIID,
UEFIVarStore: v.UEFIVarStore,
},
}
}
@@ -230,7 +225,6 @@ func VMFromRecord(r *InstanceRecord) *VM {
TerminatedAt: r.Status.TerminatedAt,
ShuttingDownAt: r.Status.ShuttingDownAt,
FirstLaunch: r.Status.FirstLaunch,
UEFIVarStore: r.Status.UEFIVarStore,
}
v.EBSRequests.Requests = r.Status.EBSRequests
v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots
+13 -33
View File
@@ -9,7 +9,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -115,35 +114,22 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
}
// The pointer is optional: a record written before it existed decodes with
// none, and a record without one writes no key, so the bucket schema holds.
func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
// The pointer lives on the store record beside the claim generation that
// fences it, so the running set's unconditional writes cannot move it. A
// record that carried one decodes, and writing it back drops the field.
func TestRecord_CarriesNoUEFIVarStorePointer(t *testing.T) {
var decoded vm.InstanceRecord
require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
assert.Nil(t, decoded.Status.UEFIVarStore)
require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped",`+
`"uefi_var_store":{"key":"uefi/i-old/a","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"node-1"}}}`), &decoded))
encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
require.NoError(t, err)
assert.NotContains(t, string(encoded), "uefi_var_store")
}
// The pointer's wire names are what the record carries across releases, so
// they are pinned rather than left to follow the Go field names.
func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
encoded, err := json.Marshal(populatedVM().Record())
require.NoError(t, err)
var wire struct {
Status struct {
UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
} `json:"status"`
for name, rec := range map[string]*vm.InstanceRecord{
"a record that carried a pointer": vm.VMFromRecord(&decoded).Record(),
"a fully populated record": populatedVM().Record(),
} {
encoded, err := json.Marshal(rec)
require.NoError(t, err)
assert.NotContains(t, string(encoded), "uefi_var_store", name)
}
require.NoError(t, json.Unmarshal(encoded, &wire))
ref := wire.Status.UEFIVarStore
assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
}
// The mutexes are dropped on the way out, so the rebuilt instance must arrive
@@ -238,12 +224,6 @@ func populatedVM() *vm.VM {
CapacityReservationId: "cr-1",
InstanceLifecycle: "spot",
SpotInstanceRequestId: "sir-1",
UEFIVarStore: &vm.UEFIVarStoreRef{
Key: "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
Firmware: uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
WriterNode: "node-3",
WrittenAt: time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
},
Status: vm.StateRunning,
Instance: &ec2.Instance{InstanceId: aws.String("i-round-trip")},
@@ -0,0 +1,197 @@
package vm
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// holdOnPeer has node-b claim the instance's store and stamp its record, as
// the node running the instance has, and returns node-b's token.
func holdOnPeer(t *testing.T, f *uefiFixture) (*VM, uint64) {
t.Helper()
key := keyFor("held")
f.putObject(t, key, gzipped(t, region("node-b booted this")))
instance := f.instance(t, key, f.identity)
tok, err := f.peerClaim("node-b").TryLock(t.Context())
require.NoError(t, err)
_, err = f.records.Stamp(t.Context(), testUEFIInstance, tok.Generation)
require.NoError(t, err)
return instance, tok.Generation
}
// TestS7_UEFIClaimIsNeverTakenOverWhileItsHolderHeartbeats enforces ADR-0003:S7
// for the store claim: a timeout or a missed heartbeat is never a decision, so a
// start elsewhere must not take a claim whose holder heartbeated under
// clusterv1.LivenessWindow ago, or whose heartbeat cannot be read. It is refused
// as retryable, and the claim, the record and the volumes are untouched.
func TestS7_UEFIClaimIsNeverTakenOverWhileItsHolderHeartbeats(t *testing.T) {
tests := []struct {
name string
age time.Duration
err error
reason string
}{
{name: "heartbeat just written", age: 0, reason: "holder_live"},
{name: "heartbeat 30s old", age: 30 * time.Second, reason: "holder_live"},
{name: "heartbeat 59s old", age: 59 * time.Second, reason: "holder_live"},
{name: "heartbeat unreadable", err: errors.New("cluster-state bucket unreachable"), reason: "liveness_unknown"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newUEFIFixture(t)
instance, gen := holdOnPeer(t, f)
f.liveness.heartbeat("node-b", tt.age)
f.liveness.err = tt.err
_, rev := f.record(t)
before := uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", tt.reason)
err := f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrUEFIClaimHeld,
"ADR-0003:S7 (uefi_varstore.go acquireUEFIClaim): a start must be refused while the claim's holder may be live; take over only once clusterv1.LiveAt is false")
require.ErrorIs(t, err, ErrMountRetryable, "the refusal clears once the holder stops or releases, so restore retries it")
holder, held := f.claimHolder(t)
require.True(t, held, "ADR-0003:S7: the holder's claim was removed while it heartbeated")
assert.Equal(t, "node-b", holder.Holder, "ADR-0003:S7: the claim was taken over while its holder heartbeated")
assert.Equal(t, gen, holder.Generation)
_, after := f.record(t)
assert.Equal(t, rev, after, "ADR-0003:S7: a refused start must not stamp the store record")
assert.Empty(t, f.mounter.mounted)
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", tt.reason))
})
}
}
// Once the holder's heartbeat is LivenessWindow old, or gone, its node is lost,
// and the start takes the claim over, stamps the record above the old holder
// and counts it, so the old holder's pointer moves are fenced from then on.
func TestS7_UEFIClaimIsTakenOverOnceItsHolderStopsHeartbeating(t *testing.T) {
tests := []struct {
name string
age time.Duration
noKey bool
}{
{name: "heartbeat 60s old", age: 60 * time.Second},
{name: "heartbeat 10m old", age: 10 * time.Minute},
{name: "no heartbeat", noKey: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newUEFIFixture(t)
instance, gen := holdOnPeer(t, f)
if !tt.noKey {
f.liveness.heartbeat("node-b", tt.age)
}
before := uefiCounter(t, "mulga.vm.uefi_store.claim_takeovers", "", "")
held, err := f.prepare(t, instance)
require.NoError(t, err)
holder, ok := f.claimHolder(t)
require.True(t, ok)
assert.Equal(t, "node-a", holder.Holder)
assert.Greater(t, holder.Generation, gen)
assert.Equal(t, held.token, holder)
rec, _ := f.record(t)
assert.Equal(t, holder.Generation, rec.Generation, "the new owner stamps the record before it reads the pointer")
assert.Equal(t, region("node-b booted this"), f.localVars(t))
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_takeovers", "", ""))
})
}
}
// failingStamp is a store record bucket whose stamp fails transiently.
type failingStamp struct{ UEFIStoreRecords }
func (failingStamp) Stamp(context.Context, string, uint64) (*uefistore.UEFIStoreRecord, error) {
return nil, errors.New("stamp: no responders")
}
// TestS7_UEFIClaimGuardingALocalCopyIsKeptWhenALaunchRollsBack enforces
// ADR-0003:S7: a restart that adopts its claim may hold changes no object
// has, so a launch that fails before reuse is decided must keep the claim.
func TestS7_UEFIClaimGuardingALocalCopyIsKeptWhenALaunchRollsBack(t *testing.T) {
tests := []struct {
name string
local func(f *uefiFixture, key string, gen uint64)
}{
{name: "copy seeded under the claim", local: func(f *uefiFixture, key string, gen uint64) {
f.writeLocal(t, key, gen, region("guest wrote this"))
}},
{name: "unreadable seeded.json", local: func(f *uefiFixture, key string, gen uint64) {
f.writeLocal(t, key, gen, region("guest wrote this"))
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), uefiSeededFileName), []byte("{"), 0o600))
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("adopted")
instance := f.instance(t, key, f.identity)
tok, err := f.peerClaim("node-a").TryLock(t.Context())
require.NoError(t, err)
tt.local(f, key, tok.Generation)
f.m.deps.UEFIVarStore.Records = failingStamp{f.records}
err = f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrMountRetryable)
holder, held := f.claimHolder(t)
require.True(t, held, "ADR-0003:S7 (uefi_varstore.go owedUnder): a rolled-back launch released the claim guarding this node's copy; keep it whenever seeded.json names the claim's generation")
assert.Equal(t, tok, holder)
_, refused := f.peerClaim("node-b").TryLock(t.Context())
assert.ErrorAs(t, refused, new(*kvsync.HeldError), "ADR-0003:S7: another node could take the claim while this node's copy held unwritten changes")
})
}
}
// A copy seeded under an earlier generation was superseded by another node,
// so a launch that rolls back before it seeds releases the claim it took.
func TestUEFIClaim_RollbackReleasesAClaimNoLocalCopyNeeds(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("stale")
instance := f.instance(t, key, f.identity)
earlier, err := f.peerClaim("node-b").TryLock(t.Context())
require.NoError(t, err)
require.NoError(t, f.peerClaim("node-b").Unlock(t.Context(), earlier))
f.writeLocal(t, key, earlier.Generation, region("older run"))
f.m.deps.UEFIVarStore.Records = failingStamp{f.records}
require.ErrorIs(t, f.m.Run(t.Context(), instance), ErrMountRetryable)
_, held := f.claimHolder(t)
assert.False(t, held)
}
// TestS7_UEFICopyThatCannotBeVouchedForIsNeverOverwritten enforces
// ADR-0003:S7: a vars.fd whose seeded.json cannot be read may hold changes no
// object has, so the start is refused as retryable and the file and the claim
// are kept rather than the object fetched over it.
func TestS7_UEFICopyThatCannotBeVouchedForIsNeverOverwritten(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("pointed")
f.putObject(t, key, gzipped(t, region("object")))
instance := f.instance(t, key, f.identity)
tok, err := f.peerClaim("node-a").TryLock(t.Context())
require.NoError(t, err)
f.writeLocal(t, key, tok.Generation, region("guest wrote this"))
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), uefiSeededFileName), []byte("{"), 0o600))
err = f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrMountRetryable)
assert.Equal(t, region("guest wrote this"), f.localVars(t),
"ADR-0003:S7 (uefi_varstore.go prepareUEFIVarStore): a copy that cannot be vouched for was overwritten; refuse the start and keep it")
holder, held := f.claimHolder(t)
require.True(t, held, "ADR-0003:S7: the claim guarding an unvouched copy was released")
assert.Equal(t, tok, holder)
assert.Empty(t, f.mounter.mounted)
}
+311 -82
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path/filepath"
@@ -19,7 +20,10 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/s3"
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"go.opentelemetry.io/otel"
@@ -38,6 +42,9 @@ const (
// object, so a wedged object store fails the start rather than holding it.
uefiStoreGetTimeout = 10 * time.Second
// uefiClaimReleaseTimeout bounds the unlock a rolled-back launch makes.
uefiClaimReleaseTimeout = 10 * time.Second
// uefiStoreBodySlack is how far a compressed body may exceed its region
// before it is refused; gzip grows incompressible data by far less.
uefiStoreBodySlack = 1 << 20
@@ -47,22 +54,40 @@ const (
uefiStateReasonCode = "Server.InternalError"
)
// ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
// store and carries no EFI volume, so it has nothing to boot its variables from.
// ErrUEFIVarStoreMissing is a UEFI instance with no EFI volume whose store
// record is absent, deleting or names no pointer, so it has nothing to boot
// its variables from.
var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
// store and also carries an EFI volume, so either one would be ignored.
var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
// ErrUEFIFirmwareMismatch is a variable store written under firmware whose
// architecture or VARS size differs from this node's, which pflash cannot load.
var ErrUEFIFirmwareMismatch = errors.New("UEFI variable store firmware does not match this node")
// ErrUEFIVarStoreUnusable is a variable store object that is missing or does
// not decode to a region of this node's VARS size. Retrying does not help.
// ErrUEFIVarStoreUnusable is a store record this node cannot trust, or a store
// object that is missing or does not decode to a region of this node's VARS
// size. Retrying does not help.
var ErrUEFIVarStoreUnusable = errors.New("UEFI variable store object is unusable")
// ErrUEFIClaimHeld is a start refused because another node holds the store's
// claim and its heartbeat is fresh or cannot be read: that node may still land
// newer variables. It always wraps ErrMountRetryable.
var ErrUEFIClaimHeld = errors.New("UEFI variable store is claimed by another node that may be live")
// UEFIStoreRecords is the store-record surface a launch reads and stamps.
type UEFIStoreRecords interface {
Get(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
Stamp(ctx context.Context, instanceID string, generation uint64) (*uefistore.UEFIStoreRecord, error)
}
var _ UEFIStoreRecords = (*uefistore.Store)(nil)
// NodeLiveness reports when a node last heartbeated. found is false only for
// a node with no heartbeat. A heartbeat that exists but cannot be dated is an
// error, as is a failed read: the answer is unknown, and never reads as dead.
type NodeLiveness interface {
LastSeen(ctx context.Context, node string) (seen time.Time, found bool, err error)
}
// UEFIVarStoreDeps is where a node reads instance variable stores from and
// keeps the live copy QEMU opens. DataDir is the daemon's persistent data
// directory, so the copy survives a daemon restart and a host reboot.
@@ -70,12 +95,31 @@ type UEFIVarStoreDeps struct {
Objects objectstore.ObjectStore
Bucket string
DataDir string
// Records holds each store's pointer and the claim generation fencing it.
Records UEFIStoreRecords
// Claims is this node's group of store claims; Liveness decides whether
// the holder of a claim this node finds held may be taken over.
Claims *kvsync.ClaimGroup
Liveness NodeLiveness
}
// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from,
// and the claim generation it was seeded or last written back under.
type uefiSeed struct {
Key string `json:"key"`
SeededAt time.Time `json:"seeded_at"`
Key string `json:"key"`
Generation uint64 `json:"generation"`
SeededAt time.Time `json:"seeded_at"`
}
// uefiClaim is the claim a launch acquired. reused is set when the launch
// boots this node's copy. keep is set when this node held a copy under token
// before the launch, which may carry changes no object has, so a rollback
// must not release the claim.
type uefiClaim struct {
claim *kvsync.Claim
token kvsync.Token
reused bool
keep bool
}
// isUEFIBootMode reports whether bootMode boots the guest through UEFI firmware.
@@ -89,24 +133,51 @@ func usesUEFIVarStore(v *VM) bool {
return !v.DirectBoot && isUEFIBootMode(v.BootMode)
}
// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
// two, before anything about it is changed. An EFI volume still counts as a
// store until RunInstances seeds pointers instead. The caller owns v.
func CheckUEFIVarStore(v *VM) error {
if !usesUEFIVarStore(v) {
return nil
}
// hasEFIVolume reports whether v keeps its variables on a <root>-efi volume.
func hasEFIVolume(v *VM) bool {
v.EBSRequests.Mu.Lock()
hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
v.EBSRequests.Mu.Unlock()
defer v.EBSRequests.Mu.Unlock()
return slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
}
switch {
case v.UEFIVarStore == nil && !hasEFIVolume:
return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
case v.UEFIVarStore != nil && hasEFIVolume:
return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
// usesStoreObject reports whether v boots its variables from a store object
// rather than an EFI volume.
func usesStoreObject(v *VM) bool {
return usesUEFIVarStore(v) && !hasEFIVolume(v)
}
// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
// two, before anything about it is changed. It only reads.
func (m *Manager) CheckUEFIVarStore(ctx context.Context, v *VM) error {
_, err := m.checkUEFIVarStore(ctx, v)
return err
}
// checkUEFIVarStore is CheckUEFIVarStore returning the store record it read,
// or nil for an instance that boots no store object. An EFI volume is the
// instance's store until RunInstances seeds store records, so it reads none.
func (m *Manager) checkUEFIVarStore(ctx context.Context, v *VM) (*uefistore.UEFIStoreRecord, error) {
if !usesStoreObject(v) {
return nil, nil
}
return nil
records := m.deps.UEFIVarStore.Records
if records == nil {
return nil, fmt.Errorf("%w: %s, and no store records are wired on this node", ErrUEFIVarStoreMissing, v.ID)
}
rec, _, err := records.Get(ctx, v.ID)
switch {
case errors.Is(err, kvstore.ErrNotFound):
return nil, fmt.Errorf("%w: %s has no store record", ErrUEFIVarStoreMissing, v.ID)
case errors.Is(err, uefistore.ErrInvalidRecord):
return nil, fmt.Errorf("%w: %w", ErrUEFIVarStoreUnusable, err)
case err != nil:
return nil, fmt.Errorf("read UEFI store record for %s: %w: %w", v.ID, err, ErrMountRetryable)
case rec.State != uefistore.StateLive:
return nil, fmt.Errorf("%w: %s store record is %s", ErrUEFIVarStoreMissing, v.ID, rec.State)
case rec.Pointer == nil:
return nil, fmt.Errorf("%w: %s store record names no pointer", ErrUEFIVarStoreMissing, v.ID)
}
return rec, nil
}
// uefiInstanceDir is the directory holding instanceID's vars.fd and
@@ -127,13 +198,11 @@ func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
return filepath.Join(dir, uefiVarsFileName), nil
}
// uefiVarsDrive is pflash unit 1 from this node's copy of the store the
// record points at, or nil with no pointer. Launch seeds the copy and a
// uefiVarsDrive is pflash unit 1 from this node's copy of the store, or nil
// for an instance that boots no store object. Launch seeds the copy and a
// reboot reuses it, so a missing one fails rather than booting without it.
func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
hasPointer := false
m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
if !hasPointer || !usesUEFIVarStore(instance) {
if !usesStoreObject(instance) {
return nil, nil
}
path, err := m.uefiVarsPath(instance.ID)
@@ -146,78 +215,188 @@ func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
return &Drive{File: path, Format: "raw", If: "pflash", Unit: 1}, nil
}
// prepareUEFIVarStore leaves vars.fd holding the store the record points at,
// reusing this node's copy when seeded.json names the pointer and fetching
// the object otherwise. The object and the record's pointer are only read.
func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
var ref *UEFIVarStoreRef
m.Inspect(instance, func(v *VM) {
if v.UEFIVarStore != nil {
cp := *v.UEFIVarStore
ref = &cp
}
})
if ref == nil || !usesUEFIVarStore(instance) {
return nil
// prepareUEFIVarStore checks this node can boot the store rec points at, then
// takes the store's claim, stamps the store record with its generation and
// leaves vars.fd holding the store to boot. rec is the record checkUEFIVarStore
// read. A non-nil claim is returned once one is acquired, failed or not, so
// the caller can release it on a rollback.
func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM, rec *uefistore.UEFIStoreRecord) (*uefiClaim, error) {
if !usesStoreObject(instance) {
return nil, nil
}
deps := m.deps.UEFIVarStore
if deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" {
return errors.New("UEFI variable store access is not wired on this node")
}
if name, ok := strings.CutPrefix(ref.Key, uefistore.KeyPrefix(instance.ID)); !ok || name == "" || strings.Contains(name, "/") {
return fmt.Errorf("%w: pointer %q is not under %s", ErrUEFIVarStoreUnusable, ref.Key, uefistore.KeyPrefix(instance.ID))
if rec == nil || deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" || deps.Records == nil || deps.Claims == nil || deps.Liveness == nil {
return nil, errors.New("UEFI variable store access is not wired on this node")
}
if m.deps.InstanceTypes == nil {
return errors.New("InstanceTypeResolver not wired")
return nil, errors.New("InstanceTypeResolver not wired")
}
spec, ok := m.deps.InstanceTypes.Resolve(instance.InstanceType)
if !ok {
return fmt.Errorf("instance type %s not found", instance.InstanceType)
return nil, fmt.Errorf("instance type %s not found", instance.InstanceType)
}
fw, err := NodeFirmwareFor(spec.Architecture)
if err != nil {
return err
return nil, err
}
if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
return err
}
dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
if err != nil {
return err
return nil, err
}
if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
// Before the claim, so a node that cannot boot the store never takes it
// from a holder that is coming back.
if err := m.checkUEFIFirmware(instance, rec.Pointer, fw); err != nil {
return nil, err
}
held, err := m.acquireUEFIClaim(ctx, instance.ID)
if err != nil {
return nil, err
}
// Before anything that can fail, so a rollback keeps a claim this node's
// copy may owe a write-back under.
local := readLocalUEFICopy(dir)
held.keep = local.owedUnder(held.token.Generation)
checked := rec.Pointer.Firmware
rec, err = deps.Records.Stamp(ctx, instance.ID, held.token.Generation)
switch {
case errors.Is(err, uefistore.ErrFenced):
recordUEFIClaimRefusal("fenced")
slog.WarnContext(ctx, "UEFI store record is stamped by a later claim than the one this node holds; refusing the start",
"instanceId", instance.ID, "generation", held.token.Generation, "err", err)
return held, fmt.Errorf("%w: %w: %w", ErrUEFIClaimHeld, err, ErrMountRetryable)
case errors.Is(err, kvstore.ErrNotFound), errors.Is(err, uefistore.ErrDeleting):
return held, fmt.Errorf("%w: %s: %w", ErrUEFIVarStoreMissing, instance.ID, err)
case errors.Is(err, uefistore.ErrInvalidRecord):
return held, fmt.Errorf("%w: %w", ErrUEFIVarStoreUnusable, err)
case err != nil:
return held, fmt.Errorf("stamp UEFI store record for %s: %w: %w", instance.ID, err, ErrMountRetryable)
case rec.Pointer == nil:
return held, fmt.Errorf("%w: %s store record names no pointer", ErrUEFIVarStoreMissing, instance.ID)
}
// The record validated its pointer as one object under this instance's prefix.
ref := rec.Pointer
if ref.Firmware != checked {
if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
return held, err
}
}
switch {
case local.hasVars && local.seedErr != nil:
slog.ErrorContext(ctx, "This node's UEFI variable store cannot be vouched for; keeping it and refusing the start",
"instanceId", instance.ID, "path", filepath.Join(dir, uefiSeededFileName), "err", local.seedErr)
return held, fmt.Errorf("UEFI variable store for %s has a %s this node cannot read: %w: %w",
instance.ID, uefiSeededFileName, local.seedErr, ErrMountRetryable)
case held.keep && local.varsSize != fw.Identity.VarsSize:
slog.ErrorContext(ctx, "This node's UEFI variable store is not the VARS size; keeping it and refusing the start",
"instanceId", instance.ID, "path", filepath.Join(dir, uefiVarsFileName),
"size", local.varsSize, "want", fw.Identity.VarsSize)
return held, fmt.Errorf("%w: %s for %s is %d bytes, not %d",
ErrUEFIVarStoreUnusable, uefiVarsFileName, instance.ID, local.varsSize, fw.Identity.VarsSize)
case held.keep:
held.reused = true
recordUEFIStoreSeed("local")
slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
return nil
slog.InfoContext(ctx, "Reusing this node's UEFI variable store",
"instanceId", instance.ID, "key", ref.Key, "seededKey", local.seed.Key, "generation", held.token.Generation)
return held, nil
}
start := time.Now()
region, err := fetchUEFIVars(ctx, deps, ref.Key, fw.Identity.VarsSize)
if err != nil {
return err
return held, err
}
if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
return held, fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
}
seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
seed, err := json.Marshal(uefiSeed{Key: ref.Key, Generation: held.token.Generation, SeededAt: time.Now().UTC()})
if err != nil {
return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
return held, fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
}
if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
return held, fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
}
recordUEFIStoreSeed("object")
slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
return nil
"instanceId", instance.ID, "key", ref.Key, "generation", held.token.Generation,
"bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
return held, nil
}
// acquireUEFIClaim takes instanceID's store claim, or adopts it when this node
// already holds it. A claim another node holds is taken over only once that
// node's heartbeat is LivenessWindow old or absent; while it is fresh, or
// cannot be read, the start is refused as retryable (ADR-0003:S7).
func (m *Manager) acquireUEFIClaim(ctx context.Context, instanceID string) (*uefiClaim, error) {
deps := m.deps.UEFIVarStore
claim := deps.Claims.Get(instanceID)
tok, err := claim.TryLock(ctx)
if err == nil {
return &uefiClaim{claim: claim, token: tok}, nil
}
he, ok := errors.AsType[*kvsync.HeldError](err)
if !ok || he.Current.Holder == "" {
return nil, fmt.Errorf("take UEFI store claim for %s: %w: %w", instanceID, err, ErrMountRetryable)
}
holder := he.Current
seen, found, err := deps.Liveness.LastSeen(ctx, holder.Holder)
if err != nil {
recordUEFIClaimRefusal("liveness_unknown")
slog.WarnContext(ctx, "Refusing to start: another node holds the UEFI variable store claim and its heartbeat cannot be read",
"instanceId", instanceID, "holder", holder.Holder, "generation", holder.Generation, "err", err)
return nil, fmt.Errorf("%w: %s holds %s and its liveness could not be read: %w: %w",
ErrUEFIClaimHeld, holder.Holder, instanceID, err, ErrMountRetryable)
}
now := time.Now()
if found && clusterv1.LiveAt(seen, now) {
recordUEFIClaimRefusal("holder_live")
slog.WarnContext(ctx, "Refusing to start: another live node holds the UEFI variable store claim",
"instanceId", instanceID, "holder", holder.Holder, "generation", holder.Generation,
"heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
return nil, fmt.Errorf("%w: %s holds %s and is still heartbeating: %w",
ErrUEFIClaimHeld, holder.Holder, instanceID, ErrMountRetryable)
}
tok, err = claim.TakeOver(ctx, holder)
if err != nil {
if _, moved := errors.AsType[*kvsync.HeldError](err); moved {
recordUEFIClaimRefusal("moved")
return nil, fmt.Errorf("%w: the claim on %s moved on from %s: %w: %w",
ErrUEFIClaimHeld, instanceID, holder.Holder, err, ErrMountRetryable)
}
return nil, fmt.Errorf("take over UEFI store claim for %s: %w: %w", instanceID, err, ErrMountRetryable)
}
recordUEFIClaimTakeover()
attrs := []any{"instanceId", instanceID, "holder", holder.Holder, "fromGeneration", holder.Generation, "generation", tok.Generation}
if found {
attrs = append(attrs, "heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
}
slog.WarnContext(ctx, "Took over the UEFI variable store claim from a node whose heartbeat has stopped; variable changes it never wrote back are lost", attrs...)
return &uefiClaim{claim: claim, token: tok}, nil
}
// releaseUEFIClaim unlocks the claim a launch that rolled back before the
// guest ran acquired. A launch holding this node's copy keeps it: the copy
// may hold changes no object has, and another node would start without them.
func (m *Manager) releaseUEFIClaim(ctx context.Context, instanceID string, held *uefiClaim) {
if held == nil || held.keep {
return
}
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), uefiClaimReleaseTimeout)
defer cancel()
if err := held.claim.Unlock(ctx, held.token); err != nil {
slog.WarnContext(ctx, "Failed to release the UEFI variable store claim after a launch rolled back",
"instanceId", instanceID, "generation", held.token.Generation, "err", err)
}
}
// checkUEFIFirmware fails a store whose arch or VARS size differs from this
// node's, naming both firmwares in the log and the StateReason. A different
// CODE build boots, because a package update changes it on every node.
func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeFirmware) error {
func (m *Manager) checkUEFIFirmware(instance *VM, ref *uefistore.UEFIVarStoreRef, fw NodeFirmware) error {
stored, node := ref.Firmware, fw.Identity
var kind string
switch {
@@ -251,23 +430,45 @@ func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeF
return fmt.Errorf("%w: %s", ErrUEFIFirmwareMismatch, msg)
}
// reuseUEFIVars reports whether dir already holds the store key names: a
// seeded.json naming key beside a vars.fd of the region's size.
func reuseUEFIVars(dir, key string, varsSize int64) bool {
// localUEFICopy is what this node holds for an instance: whether vars.fd
// exists and its size, and seeded.json or the error reading it.
type localUEFICopy struct {
hasVars bool
varsSize int64
hasSeed bool
seed uefiSeed
seedErr error
}
// readLocalUEFICopy reads dir's vars.fd and seeded.json. Only a seeded.json
// that exists and cannot be read or decoded sets seedErr.
func readLocalUEFICopy(dir string) localUEFICopy {
var c localUEFICopy
if info, err := os.Stat(filepath.Join(dir, uefiVarsFileName)); err == nil {
c.hasVars, c.varsSize = true, info.Size()
}
data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
if err != nil {
switch {
case errors.Is(err, fs.ErrNotExist):
return c
case err != nil:
c.seedErr = err
default:
c.seedErr = json.Unmarshal(data, &c.seed)
}
c.hasSeed = c.seedErr == nil
return c
}
// owedUnder reports whether the copy may hold changes no object has under the
// claim generation: vars.fd exists and seeded.json names generation, or cannot
// be read. At one generation only this node moves the pointer, and only from
// vars.fd, so such a file is never older than the pointer whatever key it names.
func (c localUEFICopy) owedUnder(generation uint64) bool {
if !c.hasVars {
return false
}
var seed uefiSeed
if err := json.Unmarshal(data, &seed); err != nil || seed.Key != key {
return false
}
info, err := os.Stat(filepath.Join(dir, uefiVarsFileName))
if err != nil || info.Size() != varsSize {
slog.Warn("UEFI variable store seeded here has no usable vars.fd; fetching it again", "dir", dir, "key", key)
return false
}
return true
return c.seedErr != nil || (c.hasSeed && c.seed.Generation == generation)
}
// fetchUEFIVars gets key and returns its decompressed region, which must be
@@ -354,6 +555,8 @@ var (
uefiStoreMetricsOnce sync.Once
uefiStoreSeedCounter metric.Int64Counter
uefiFirmwareMismatches metric.Int64Counter
uefiClaimRefusals metric.Int64Counter
uefiClaimTakeovers metric.Int64Counter
)
func uefiStoreMetrics() {
@@ -372,6 +575,18 @@ func uefiStoreMetrics() {
if err != nil {
otel.Handle(err)
}
uefiClaimRefusals, err = meter.Int64Counter("mulga.vm.uefi_store.claim_refusals",
metric.WithDescription("UEFI starts refused because another node holds the variable store claim and may be live, by reason."),
metric.WithUnit("{start}"))
if err != nil {
otel.Handle(err)
}
uefiClaimTakeovers, err = meter.Int64Counter("mulga.vm.uefi_store.claim_takeovers",
metric.WithDescription("UEFI variable store claims taken over from a node whose heartbeat had stopped."),
metric.WithUnit("{start}"))
if err != nil {
otel.Handle(err)
}
})
}
@@ -388,3 +603,17 @@ func recordUEFIFirmwareMismatch(kind string) {
uefiFirmwareMismatches.Add(context.Background(), 1, metric.WithAttributes(attribute.String("kind", kind)))
}
}
func recordUEFIClaimRefusal(reason string) {
uefiStoreMetrics()
if uefiClaimRefusals != nil {
uefiClaimRefusals.Add(context.Background(), 1, metric.WithAttributes(attribute.String("reason", reason)))
}
}
func recordUEFIClaimTakeover() {
uefiStoreMetrics()
if uefiClaimTakeovers != nil {
uefiClaimTakeovers.Add(context.Background(), 1)
}
}
+378 -97
View File
@@ -19,8 +19,12 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/s3"
testutil "github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
@@ -41,7 +45,7 @@ var (
)
// uefiCounter returns the sum of the named counter's points whose attribute
// key has value.
// key has value, or of every point when key is empty.
func uefiCounter(t *testing.T, name, key, value string) int64 {
t.Helper()
uefiMetricsReaderOnce.Do(func() {
@@ -58,7 +62,7 @@ func uefiCounter(t *testing.T, name, key, value string) int64 {
continue
}
for _, dp := range sum.DataPoints {
if v, ok := dp.Attributes.Value(attribute.Key(key)); ok && v.AsString() == value {
if v, ok := dp.Attributes.Value(attribute.Key(key)); key == "" || ok && v.AsString() == value {
total += dp.Value
}
}
@@ -67,17 +71,45 @@ func uefiCounter(t *testing.T, name, key, value string) int64 {
return total
}
// uefiFixture is a node with one x86_64 firmware candidate, an in-memory
// object store and a data directory.
// uefiFixture is node-a with one x86_64 firmware candidate, an in-memory
// object store, a data directory, and the store records and claims of a real
// JetStream, with every other node's heartbeat set by the test.
type uefiFixture struct {
m *Manager
mounter *fakeVolumeMounter
objects *objectstore.MemoryObjectStore
records *uefistore.Store
js jetstream.JetStream
liveness *fakeLiveness
dataDir string
codePath string
identity uefistore.FirmwareIdentity
}
// fakeLiveness answers LastSeen for each node from what the test set; a node
// it has no answer for has never heartbeated.
type fakeLiveness struct {
mu sync.Mutex
seen map[string]time.Time
err error
}
func (l *fakeLiveness) LastSeen(_ context.Context, node string) (time.Time, bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.err != nil {
return time.Time{}, false, l.err
}
seen, ok := l.seen[node]
return seen, ok, nil
}
func (l *fakeLiveness) heartbeat(node string, age time.Duration) {
l.mu.Lock()
defer l.mu.Unlock()
l.seen[node] = time.Now().Add(-age)
}
func newUEFIFixture(t *testing.T) *uefiFixture {
t.Helper()
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
@@ -91,10 +123,17 @@ func newUEFIFixture(t *testing.T) *uefiFixture {
FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
t.Cleanup(func() { FirmwarePathCandidates = orig })
_, _, js := testutil.StartTestJetStream(t)
records := uefistore.NewStore(js)
require.NoError(t, records.Open(t.Context()))
sum := sha256.Sum256(codeBytes)
f := &uefiFixture{
mounter: &fakeVolumeMounter{},
objects: objectstore.NewMemoryObjectStore(),
records: records,
js: js,
liveness: &fakeLiveness{seen: map[string]time.Time{}},
dataDir: t.TempDir(),
codePath: code,
identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
@@ -102,11 +141,34 @@ func newUEFIFixture(t *testing.T) *uefiFixture {
f.m = NewManagerWithDeps(Deps{
VolumeMounter: f.mounter,
InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
UEFIVarStore: UEFIVarStoreDeps{Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir},
UEFIVarStore: UEFIVarStoreDeps{
Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir,
Records: records, Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"), Liveness: f.liveness,
},
})
return f
}
// peerClaim is the instance's claim as another node sees it.
func (f *uefiFixture) peerClaim(node string) *kvsync.Claim {
return uefistore.NewClaims(kvstore.OpenSyncBucket(f.js), node).Get(testUEFIInstance)
}
// claimHolder is who holds the instance's claim, read as a third node would.
func (f *uefiFixture) claimHolder(t *testing.T) (kvsync.Token, bool) {
t.Helper()
tok, ok, err := f.peerClaim("observer").Holder(t.Context())
require.NoError(t, err)
return tok, ok
}
func (f *uefiFixture) record(t *testing.T) (*uefistore.UEFIStoreRecord, uint64) {
t.Helper()
rec, rev, err := f.records.Get(t.Context(), testUEFIInstance)
require.NoError(t, err)
return rec, rev
}
// region is a VARS region of the fixture's size whose bytes say which store it is.
func region(tag string) []byte {
b := make([]byte, testUEFIVarsSize)
@@ -146,11 +208,11 @@ func (f *uefiFixture) instanceDir() string {
return filepath.Join(f.dataDir, "uefi", testUEFIInstance)
}
func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, vars []byte) {
func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, generation uint64, vars []byte) {
t.Helper()
require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "vars.fd"), vars, 0o600))
seed, err := json.Marshal(uefiSeed{Key: seededKey, SeededAt: time.Now()})
seed, err := json.Marshal(uefiSeed{Key: seededKey, Generation: generation, SeededAt: time.Now()})
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), seed, 0o600))
}
@@ -162,78 +224,122 @@ func (f *uefiFixture) localVars(t *testing.T) []byte {
return b
}
func (f *uefiFixture) seededKey(t *testing.T) string {
func (f *uefiFixture) seed(t *testing.T) uefiSeed {
t.Helper()
b, err := os.ReadFile(filepath.Join(f.instanceDir(), "seeded.json"))
require.NoError(t, err)
var seed uefiSeed
require.NoError(t, json.Unmarshal(b, &seed))
return seed.Key
return seed
}
func (f *uefiFixture) instance(key string, fw uefistore.FirmwareIdentity) *VM {
// instance is a stopped UEFI instance with no EFI volume. A non-empty key
// gives it a live store record pointing there, at generation 0 as a
// conversion leaves it.
func (f *uefiFixture) instance(t *testing.T, key string, fw uefistore.FirmwareIdentity) *VM {
t.Helper()
v := &VM{
ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Status: StateStopped,
Instance: &ec2.Instance{},
}
if key != "" {
v.UEFIVarStore = &UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-a", WrittenAt: time.Unix(1700000000, 0).UTC()}
_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
InstanceID: testUEFIInstance, AccountID: "111122223333", State: uefistore.StateLive,
Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-c", WrittenAt: time.Unix(1700000000, 0).UTC()},
})
require.NoError(t, err)
}
f.m.Insert(v)
return v
}
// prepare runs the launch's store check and store step and returns the claim
// the step acquired.
func (f *uefiFixture) prepare(t *testing.T, v *VM) (*uefiClaim, error) {
t.Helper()
rec, err := f.m.checkUEFIVarStore(t.Context(), v)
if err != nil {
return nil, err
}
return f.m.prepareUEFIVarStore(t.Context(), v, rec)
}
func keyFor(name string) string { return uefistore.KeyPrefix(testUEFIInstance) + name }
func TestPrepareUEFIVarStore_FetchesTheObjectAndRecordsTheSeed(t *testing.T) {
func TestPrepareUEFIVarStore_ClaimsStampsAndFetchesTheObject(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("11111111-1111-1111-1111-111111111111")
f.putObject(t, key, gzipped(t, region("converted")))
instance := f.instance(key, f.identity)
instance := f.instance(t, key, f.identity)
before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object")
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
held, err := f.prepare(t, instance)
require.NoError(t, err)
holder, ok := f.claimHolder(t)
require.True(t, ok, "the claim is held from before the guest boots")
assert.Equal(t, "node-a", holder.Holder)
assert.Equal(t, held.token, holder)
rec, _ := f.record(t)
assert.Equal(t, holder.Generation, rec.Generation, "the record is stamped with the claim's generation")
assert.Equal(t, key, rec.Pointer.Key, "a launch only reads the pointer")
assert.Equal(t, region("converted"), f.localVars(t), "vars.fd must hold the decompressed region")
assert.Equal(t, key, f.seededKey(t))
assert.Equal(t, uefiSeed{Key: key, Generation: holder.Generation, SeededAt: f.seed(t).SeededAt}, f.seed(t))
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object"))
assert.Equal(t, key, instance.UEFIVarStore.Key, "seeding only reads the pointer")
}
// A start on the node that seeded the pointer boots that node's copy, which
// can be newer than the object, and makes no get at all.
func TestPrepareUEFIVarStore_ReusesTheLocalCopyWhenSeededNamesThePointer(t *testing.T) {
// A daemon restart or host reboot adopts the claim at its generation, and the
// file it left may hold changes no object has yet, so it is booted without a
// get and the record keeps the claim's generation.
func TestPrepareUEFIVarStore_AdoptedClaimReusesTheLocalCopy(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("22222222-2222-2222-2222-222222222222")
f.writeLocal(t, key, region("guest wrote this"))
instance := f.instance(key, f.identity)
instance := f.instance(t, key, f.identity)
held, err := f.peerClaim("node-a").TryLock(t.Context())
require.NoError(t, err)
_, err = f.records.Stamp(t.Context(), testUEFIInstance, held.Generation)
require.NoError(t, err)
f.writeLocal(t, key, held.Generation, region("guest wrote this"))
before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance), "no object exists, so a get would fail")
got, err := f.prepare(t, instance)
require.NoError(t, err, "no object exists, so a get would fail")
assert.True(t, got.reused)
assert.Equal(t, held, got.token, "a restart adopts the claim at the same generation")
rec, _ := f.record(t)
assert.Equal(t, held.Generation, rec.Generation, "adoption keeps the claim's generation")
assert.Equal(t, region("guest wrote this"), f.localVars(t))
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
}
// Once the pointer has moved past what this node seeded, the local copy is
// stale and the pointed object replaces it.
func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing.T) {
// Once the pointer or the generation has moved past what this node seeded,
// the local copy is stale and the pointed object replaces it. A copy from an
// earlier generation predates another node's claim or a restore.
func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotTheCurrentStore(t *testing.T) {
tests := []struct {
name string
local func(f *uefiFixture, t *testing.T)
}{
{name: "seeded from an older pointer", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("older"), region("stale"))
f.writeLocal(t, keyFor("older"), 0, region("stale"))
}},
{name: "seeded under an earlier generation", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("current"), 1, region("stale"))
}},
{name: "seeded before generations were recorded", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("current"), 0, region("stale"))
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte(`{"key":"`+keyFor("current")+`"}`), 0o600))
}},
{name: "vars.fd of the wrong size", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("current"), []byte("short"))
f.writeLocal(t, keyFor("current"), 0, []byte("short"))
}},
{name: "seeded.json unreadable", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("current"), region("stale"))
{name: "seeded.json unreadable with no vars.fd", local: func(f *uefiFixture, t *testing.T) {
require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte("{"), 0o600))
}},
{name: "no vars.fd", local: func(f *uefiFixture, t *testing.T) {
f.writeLocal(t, keyFor("current"), region("stale"))
f.writeLocal(t, keyFor("current"), 0, region("stale"))
require.NoError(t, os.Remove(filepath.Join(f.instanceDir(), "vars.fd")))
}},
}
@@ -243,19 +349,22 @@ func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing
key := keyFor("current")
tt.local(f, t)
f.putObject(t, key, gzipped(t, region("restored")))
instance := f.instance(key, f.identity)
instance := f.instance(t, key, f.identity)
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
held, err := f.prepare(t, instance)
require.NoError(t, err)
assert.False(t, held.reused)
assert.Equal(t, region("restored"), f.localVars(t))
assert.Equal(t, key, f.seededKey(t))
assert.Equal(t, key, f.seed(t).Key)
assert.Equal(t, held.token.Generation, f.seed(t).Generation)
})
}
}
// A store pflash cannot load fails the launch before any volume is mounted,
// names both firmwares, and leaves the object, the pointer and the local
// copy as they were.
// names both firmwares, gives the claim back and leaves the object, the
// pointer and the local copy as they were.
func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
tests := []struct {
name string
@@ -274,11 +383,11 @@ func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
key := keyFor("mismatch")
body := gzipped(t, region("other firmware"))
f.putObject(t, key, body)
f.writeLocal(t, keyFor("older"), region("local"))
f.writeLocal(t, keyFor("older"), 0, region("local"))
stored := f.identity
tt.mutate(&stored)
instance := f.instance(key, stored)
wantRef := *instance.UEFIVarStore
instance := f.instance(t, key, stored)
wantRef, _ := f.record(t)
before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind)
err := f.m.Run(t.Context(), instance)
@@ -291,10 +400,13 @@ func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
for _, s := range append(tt.want, f.identity.CodeSHA256, f.codePath, key) {
assert.Contains(t, aws.StringValue(instance.Instance.StateReason.Message), s)
}
assert.Equal(t, wantRef, *instance.UEFIVarStore, "the pointer is unchanged")
rec, _ := f.record(t)
assert.Equal(t, wantRef.Pointer, rec.Pointer, "the pointer is unchanged")
assert.Equal(t, body, f.objectBody(t, key), "the object is unchanged")
assert.Equal(t, region("local"), f.localVars(t), "the local copy is unchanged")
assert.Equal(t, keyFor("older"), f.seededKey(t))
assert.Equal(t, keyFor("older"), f.seed(t).Key)
_, held := f.claimHolder(t)
assert.False(t, held, "a launch that never started QEMU gives the claim back")
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind))
})
}
@@ -308,10 +420,11 @@ func TestPrepareUEFIVarStore_DifferentCodeBuildBootsAndIsCounted(t *testing.T) {
f.putObject(t, key, gzipped(t, region("older ovmf")))
stored := f.identity
stored.CodeSHA256 = strings.Repeat("0", 64)
instance := f.instance(key, stored)
instance := f.instance(t, key, stored)
before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code")
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
_, err := f.prepare(t, instance)
require.NoError(t, err)
assert.Equal(t, region("older ovmf"), f.localVars(t))
assert.Nil(t, instance.Instance.StateReason)
@@ -333,9 +446,9 @@ func (s *failingGetStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.Ge
func TestPrepareUEFIVarStore_FailedGetIsRetryable(t *testing.T) {
f := newUEFIFixture(t)
f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("connection refused")}
instance := f.instance(keyFor("down"), f.identity)
instance := f.instance(t, keyFor("down"), f.identity)
err := f.m.prepareUEFIVarStore(t.Context(), instance)
_, err := f.prepare(t, instance)
require.ErrorIs(t, err, ErrMountRetryable, "restore backs off on a get as it does on a mount")
assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
@@ -361,9 +474,9 @@ func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
if tt.body != nil {
f.putObject(t, key, tt.body)
}
instance := f.instance(key, f.identity)
instance := f.instance(t, key, f.identity)
err := f.m.prepareUEFIVarStore(t.Context(), instance)
_, err := f.prepare(t, instance)
require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
require.NotErrorIs(t, err, ErrMountRetryable)
@@ -373,56 +486,77 @@ func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
}
}
func TestPrepareUEFIVarStore_PointerOutsideTheInstancePrefixIsRefused(t *testing.T) {
for _, key := range []string{"uefi/i-0aaaaaaaaaaaaaaaa/other", keyFor("a/b"), uefistore.KeyPrefix(testUEFIInstance)} {
t.Run(key, func(t *testing.T) {
f := newUEFIFixture(t)
f.putObject(t, key, gzipped(t, region("x")))
instance := f.instance(key, f.identity)
// A record this node cannot trust, such as one pointing outside the
// instance's prefix, is refused for good before any claim is taken.
func TestRun_UntrustedStoreRecordIsRefused(t *testing.T) {
f := newUEFIFixture(t)
instance := f.instance(t, "", f.identity)
kv, err := f.js.KeyValue(t.Context(), uefistore.BucketName)
require.NoError(t, err)
_, err = kv.Put(t.Context(), testUEFIInstance, []byte(`{"schema_version":1,"instance_id":"`+testUEFIInstance+
`","state":"live","pointer":{"key":"uefi/i-0aaaaaaaaaaaaaaaa/other","firmware":{"arch":"x86_64","vars_size":4096},"writer_node":"n"}}`))
require.NoError(t, err)
require.ErrorIs(t, f.m.prepareUEFIVarStore(t.Context(), instance), ErrUEFIVarStoreUnusable)
})
}
err = f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
require.NotErrorIs(t, err, ErrMountRetryable)
_, held := f.claimHolder(t)
assert.False(t, held)
}
// A BIOS instance, a microvm and a legacy instance with no pointer touch
// neither the object store nor the data directory.
func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
// A BIOS instance, a microvm and an instance whose variables are on an EFI
// volume touch neither the claim, the object store nor the data directory.
func TestPrepareUEFIVarStore_NoStoreObjectIsANoOp(t *testing.T) {
f := newUEFIFixture(t)
f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("must not be called")}
bios := f.instance(keyFor("bios"), f.identity)
bios := f.instance(t, keyFor("bios"), f.identity)
bios.BootMode = ""
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), bios))
held, err := f.prepare(t, bios)
require.NoError(t, err)
assert.Nil(t, held)
legacy := f.instance("", f.identity)
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), legacy))
microvm := &VM{ID: testUEFIInstance, BootMode: "uefi", DirectBoot: true}
held, err = f.prepare(t, microvm)
require.NoError(t, err)
assert.Nil(t, held)
efi := &VM{ID: testUEFIInstance, BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}
held, err = f.prepare(t, efi)
require.NoError(t, err)
assert.Nil(t, held)
_, claimed := f.claimHolder(t)
assert.False(t, claimed)
assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
}
// A UEFI instance with no pointer, or two stores, is refused before the
// launch clears or changes anything on its record.
func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
// A UEFI instance with no usable store record is refused before the launch
// clears or changes anything on its record, and before any claim is taken.
func TestRun_UEFIInstanceWithoutAUsableStoreIsRefused(t *testing.T) {
tests := []struct {
name string
pointer bool
efi bool
want error
name string
record func(f *uefiFixture, t *testing.T)
want error
}{
{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
{name: "no record and no EFI volume", want: ErrUEFIVarStoreMissing},
{name: "a record with no pointer", want: ErrUEFIVarStoreMissing, record: func(f *uefiFixture, t *testing.T) {
_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: testUEFIInstance, AccountID: "a", State: uefistore.StateLive})
require.NoError(t, err)
}},
{name: "a deleting record", want: ErrUEFIVarStoreMissing, record: func(f *uefiFixture, t *testing.T) {
f.instance(t, keyFor("p"), f.identity)
_, err := f.records.MarkDeleting(t.Context(), testUEFIInstance)
require.NoError(t, err)
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newUEFIFixture(t)
key := ""
if tt.pointer {
key = keyFor("p")
}
instance := f.instance(key, f.identity)
if tt.efi {
instance.EBSRequests.Requests = []EBSRequest{efiVolume}
if tt.record != nil {
tt.record(f, t)
}
instance := f.instance(t, "", f.identity)
stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
instance.Instance.StateReason = stale
before := instance.Record()
@@ -432,17 +566,19 @@ func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
require.ErrorIs(t, err, tt.want)
assert.Empty(t, f.mounter.mounted)
assert.Equal(t, before, instance.Record(), "the record is unchanged")
_, held := f.claimHolder(t)
assert.False(t, held, "a refused start takes no claim")
})
}
}
// Start from StateError transitions to pending before launching; a refused
// UEFI instance must not take that transition either.
func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testing.T) {
func TestStart_UEFIInstanceWithoutAStoreIsRefusedBeforeTheTransition(t *testing.T) {
f := newUEFIFixture(t)
transitions := 0
f.m.deps.TransitionState = func(v *VM, s InstanceState) error { transitions++; v.Status = s; return nil }
instance := f.instance("", f.identity)
instance := f.instance(t, "", f.identity)
instance.Status = StateError
err := f.m.Start(t.Context(), instance.ID)
@@ -452,36 +588,56 @@ func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testin
assert.Equal(t, StateError, instance.Status)
}
// Until RunInstances seeds pointers, an instance whose variables are on an
// EFI volume still launches through the volume, and a BIOS or microvm
// Until RunInstances seeds store records, an instance whose variables are on
// an EFI volume still launches through the volume, and a BIOS or microvm
// guest needs no store at all.
func TestCheckUEFIVarStore_Accepts(t *testing.T) {
efi := func() *VM {
return &VM{ID: testUEFIInstance, BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}
}
tests := []struct {
name string
v *VM
name string
v func(f *uefiFixture, t *testing.T) *VM
unwir bool
unreadable bool
}{
{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
{name: "no boot mode", v: &VM{}},
{name: "microvm", v: &VM{BootMode: "uefi", DirectBoot: true}},
{name: "EFI volume", v: func(*uefiFixture, *testing.T) *VM { return efi() }},
{name: "EFI volume on a node with no store records", unwir: true, v: func(*uefiFixture, *testing.T) *VM { return efi() }},
{name: "EFI volume while the store bucket cannot be read", unreadable: true, v: func(*uefiFixture, *testing.T) *VM { return efi() }},
{name: "store record", v: func(f *uefiFixture, t *testing.T) *VM {
v := f.instance(t, keyFor("k"), f.identity)
v.BootMode = "uefi-preferred"
return v
}},
{name: "BIOS", v: func(*uefiFixture, *testing.T) *VM { return &VM{ID: testUEFIInstance, BootMode: "legacy-bios"} }},
{name: "no boot mode", v: func(*uefiFixture, *testing.T) *VM { return &VM{ID: testUEFIInstance} }},
{name: "microvm", v: func(*uefiFixture, *testing.T) *VM {
return &VM{ID: testUEFIInstance, BootMode: "uefi", DirectBoot: true}
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.NoError(t, CheckUEFIVarStore(tt.v))
f := newUEFIFixture(t)
if tt.unwir {
f.m.deps.UEFIVarStore.Records = nil
}
if tt.unreadable {
f.m.deps.UEFIVarStore.Records = failingRecordGet{f.records}
}
assert.NoError(t, f.m.CheckUEFIVarStore(t.Context(), tt.v(f, t)))
})
}
}
// A launch from a pointer opens this node's copy as pflash unit 1 with
// A store-record launch opens this node's copy as pflash unit 1 with
// format=raw, after the read-only CODE blob.
func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
func TestStartQEMU_StoreObjectPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
f := newUEFIFixture(t)
f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
launches := withFakeQEMU(t)
key := keyFor("boot")
f.writeLocal(t, key, region("seeded"))
instance := f.instance(key, f.identity)
f.writeLocal(t, key, 1, region("seeded"))
instance := f.instance(t, key, f.identity)
instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
require.NoError(t, f.m.startQEMU(instance))
@@ -497,11 +653,11 @@ func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
// A reboot reuses the copy the launch seeded; without one there is nothing
// safe to boot, so QEMU is not started.
func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
func TestStartQEMU_StoreObjectWithoutALocalCopyIsNotLaunched(t *testing.T) {
f := newUEFIFixture(t)
f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
launches := withFakeQEMU(t)
instance := f.instance(keyFor("boot"), f.identity)
instance := f.instance(t, keyFor("boot"), f.identity)
instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
err := f.m.startQEMU(instance)
@@ -510,6 +666,68 @@ func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
assert.Empty(t, launches())
}
// A launch that fails before QEMU starts gives back a claim whose store it
// fetched, since nothing on this node is newer than the object.
func TestRun_LaunchFailingBeforeQEMUReleasesAFetchedClaim(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("fetched")
f.putObject(t, key, gzipped(t, region("object")))
instance := f.instance(t, key, f.identity)
f.mounter.mountErr = errors.New("mount refused")
require.Error(t, f.m.Run(t.Context(), instance))
_, held := f.claimHolder(t)
assert.False(t, held, "another node may start the instance at once")
}
// A launch that booted this node's copy keeps the claim when it fails, because
// that copy may hold variable changes no object has.
func TestRun_LaunchFailingAfterReusingTheLocalCopyKeepsTheClaim(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("reused")
instance := f.instance(t, key, f.identity)
tok, err := f.peerClaim("node-a").TryLock(t.Context())
require.NoError(t, err)
_, err = f.records.Stamp(t.Context(), testUEFIInstance, tok.Generation)
require.NoError(t, err)
f.writeLocal(t, key, tok.Generation, region("unlanded"))
f.mounter.mountErr = errors.New("mount refused")
require.Error(t, f.m.Run(t.Context(), instance))
holder, held := f.claimHolder(t)
require.True(t, held)
assert.Equal(t, tok, holder)
}
// A recreated sync bucket restarts generations, so a store record can sit
// above every new token. The start is refused as retryable and the record
// keeps its generation and pointer.
func TestRun_RecordAboveTheClaimGenerationIsRefusedAndUnchanged(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("fenced")
f.putObject(t, key, gzipped(t, region("object")))
instance := f.instance(t, key, f.identity)
rec, rev := f.record(t)
rec.Generation = 1 << 40
_, err := f.records.CompareAndSet(t.Context(), rec, rev)
require.NoError(t, err)
want, wantRev := f.record(t)
before := uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", "fenced")
err = f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrUEFIClaimHeld)
require.ErrorIs(t, err, ErrMountRetryable)
got, gotRev := f.record(t)
assert.Equal(t, wantRev, gotRev, "the record is not written")
assert.Equal(t, want, got)
assert.Empty(t, f.mounter.mounted)
assert.NoFileExists(t, filepath.Join(f.instanceDir(), "vars.fd"))
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", "fenced"))
}
func TestUEFIInstanceDir_RefusesIDsThatAreNotOnePathSegment(t *testing.T) {
for _, id := range []string{"", "..", "i-../x", "i-a/b", `i-a\b`, "vol-1"} {
_, err := uefiInstanceDir("/data", id)
@@ -537,3 +755,66 @@ func TestNodeFirmwareFor_HashesTheCodeBlobAgainAfterItChanges(t *testing.T) {
assert.Equal(t, hex.EncodeToString(sum[:]), second.Identity.CodeSHA256)
assert.Equal(t, f.codePath, second.Code)
}
// failingRecordGet is a store record bucket whose reads fail.
type failingRecordGet struct{ UEFIStoreRecords }
func (failingRecordGet) Get(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
return nil, 0, errors.New("uefi stores: no responders")
}
// At one generation only this node moves the pointer, and only from vars.fd,
// so a crash between a move and the seeded.json rewrite leaves a file newer
// than the key seeded.json names. It is booted, not refetched over.
func TestPrepareUEFIVarStore_ReusesTheCopyOfTheClaimsGenerationWhateverItsKey(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("moved")
f.putObject(t, key, gzipped(t, region("object")))
instance := f.instance(t, key, f.identity)
tok, err := f.peerClaim("node-a").TryLock(t.Context())
require.NoError(t, err)
f.writeLocal(t, keyFor("before-move"), tok.Generation, region("newer than the object"))
held, err := f.prepare(t, instance)
require.NoError(t, err)
assert.True(t, held.reused)
assert.Equal(t, region("newer than the object"), f.localVars(t))
}
// A launch that fails inside startQEMU before the process starts ran no
// guest, so it gives back a claim it fetched the store under.
func TestRun_LaunchFailingBeforeQEMUExecsReleasesAFetchedClaim(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("fetched")
f.putObject(t, key, gzipped(t, region("object")))
instance := f.instance(t, key, f.identity)
err := f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, errQEMUNotStarted, "an instance with no boot volume fails before QEMU execs")
_, held := f.claimHolder(t)
assert.False(t, held, "another node may start the instance at once")
}
// A node that cannot boot the store refuses before it touches the claim, so
// it never takes the store from a holder that is coming back.
func TestRun_FirmwareMismatchRefusesBeforeTakingTheClaim(t *testing.T) {
f := newUEFIFixture(t)
key := keyFor("arm")
stored := f.identity
stored.Arch = "arm64"
instance := f.instance(t, key, stored)
tok, err := f.peerClaim("node-b").TryLock(t.Context())
require.NoError(t, err)
f.liveness.heartbeat("node-b", 10*time.Minute)
err = f.m.Run(t.Context(), instance)
require.ErrorIs(t, err, ErrUEFIFirmwareMismatch)
holder, held := f.claimHolder(t)
require.True(t, held)
assert.Equal(t, tok, holder, "the claim stays with node-b")
rec, _ := f.record(t)
assert.Zero(t, rec.Generation, "the record is not stamped")
}
-15
View File
@@ -20,7 +20,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/contracts/ec2/v1"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
@@ -319,20 +318,6 @@ type VM struct {
// tolerated by migrate.
InstanceLifecycle string `json:"instance_lifecycle,omitempty"`
SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
// UEFIVarStore points at the object holding this instance's UEFI variable
// store. Status, so written only by the node; nil for a BIOS instance.
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
}
// UEFIVarStoreRef names the store object an instance launches from, with the
// firmware it was written under, so a launch can check it without reading the
// object's metadata.
type UEFIVarStoreRef struct {
Key string `json:"key"`
Firmware uefistore.FirmwareIdentity `json:"firmware"`
WriterNode string `json:"writer_node"`
WrittenAt time.Time `json:"written_at"`
}
// UnmarshalJSON folds the fields of legacy records that mixed a request with an
+14 -2
View File
@@ -5,6 +5,7 @@ import (
"log/slog"
"github.com/mulgadc/spinifex/spinifex/domains/dns"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
@@ -16,8 +17,8 @@ import (
"github.com/nats-io/nats.go/jetstream"
)
// daemonLocks is every lease the daemon owns, each constructed once against the
// process's sync bucket and handed to the sites that run under it.
// daemonLocks is every lease and claim the daemon owns, each constructed once
// against the process's sync bucket and handed to the sites that run under it.
type daemonLocks struct {
dns *kvsync.Lease
route53Realize *kvsync.Lease
@@ -26,6 +27,7 @@ type daemonLocks struct {
ecs *kvsync.Lease
ochre *kvsync.Lease
eks *kvsync.LeaseGroup
uefi *kvsync.ClaimGroup
}
func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
@@ -37,6 +39,7 @@ func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
ecs: ecsdomain.NewSchedulerLease(store, node),
ochre: ochre.NewReaperLease(store, node),
eks: ekscluster.NewClusterLeases(store, node),
uefi: uefistore.NewClaims(store, node),
}
}
@@ -64,3 +67,12 @@ func deleteObsoleteBuckets(ctx context.Context, js jetstream.JetStream) {
}
}
}
// uefiClaims is the UEFI store claim group, or nil before startCluster has
// built the daemon's locks.
func (d *Daemon) uefiClaims() *kvsync.ClaimGroup {
if d.locks == nil {
return nil
}
return d.locks.uefi
}
+1
View File
@@ -24,6 +24,7 @@ func TestDaemonLocks_EveryOwnerHasItsOwnKey(t *testing.T) {
reconcilelock.NewLeaderLease(store, admissionquota.KeyReconcile, "node-1")
locks.eks.Get("123456789012/alpha")
locks.eks.Get("123456789012/alpha/teardown")
locks.uefi.Get("i-0123456789abcdef0")
})
}
@@ -0,0 +1,77 @@
package daemon
import (
"context"
"errors"
"fmt"
"time"
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go/jetstream"
)
// clusterHeartbeats satisfies vm.NodeLiveness from the heartbeats every daemon
// writes to the cluster-state bucket.
type clusterHeartbeats struct {
kv jetstream.KeyValue
js jetstream.JetStream
}
var _ vm.NodeLiveness = clusterHeartbeats{}
// LastSeen returns when node last wrote a heartbeat. No entry is found=false;
// a failed read, or an entry that cannot be dated, is an error, because a beat
// that exists may be a live node's.
//
// A replica that lags can only answer with an older beat, so a live one is
// returned as read and anything else is read again from the stream leader.
func (h clusterHeartbeats) LastSeen(ctx context.Context, node string) (time.Time, bool, error) {
key := clusterv1.HeartbeatKey(node)
if entry, err := h.kv.Get(ctx, key); err == nil {
if written, ok := clusterv1.HeartbeatWritten(entry.Value()); ok && clusterv1.LiveAt(written, time.Now()) {
return written, true, nil
}
}
if h.js == nil {
return time.Time{}, false, fmt.Errorf("read heartbeat for %s: no JetStream client to confirm it with the leader", node)
}
bucket := clusterv1.ClusterStateBucket
msg, err := kvstore.LeaderMsgGet(ctx, h.js, "KV_"+bucket, kvstore.MsgGetRequest{LastFor: "$KV." + bucket + "." + key})
if errors.Is(err, jetstream.ErrMsgNotFound) {
return time.Time{}, false, nil
}
if err != nil {
return time.Time{}, false, fmt.Errorf("read heartbeat for %s from the leader: %w", node, err)
}
if op := msg.Header.Get(kvOperationHeader); op == "DEL" || op == "PURGE" {
return time.Time{}, false, nil
}
written, ok := clusterv1.HeartbeatWritten(msg.Data)
if !ok {
return time.Time{}, false, fmt.Errorf("heartbeat for %s cannot be dated", node)
}
return written, true, nil
}
// kvOperationHeader marks a KV stream message that deletes or purges its key.
const kvOperationHeader = "KV-Operation"
// heartbeatLiveness returns nil, not a typed nil, before the cluster-state
// bucket is open, so a launch sees liveness as unwired rather than panicking.
func (d *Daemon) heartbeatLiveness() vm.NodeLiveness {
if d.jsManager == nil || d.jsManager.clusterKV == nil {
return nil
}
return clusterHeartbeats{kv: d.jsManager.clusterKV, js: d.jsManager.js}
}
// uefiStoreRecords returns nil, not a typed nil, when the store bucket is not
// open, so a launch sees store records as unwired.
func (d *Daemon) uefiStoreRecords() vm.UEFIStoreRecords {
if d.jsManager == nil || d.jsManager.UEFIStores() == nil {
return nil
}
return d.jsManager.UEFIStores()
}
@@ -0,0 +1,118 @@
//test:in-package — clusterHeartbeats is an unexported daemon adapter
package daemon
import (
"context"
"encoding/json"
"testing"
"time"
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
testutil "github.com/mulgadc/spinifex/internal/testkit"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// The claim's takeover rule reads the same heartbeat the daemon writes. Only
// an absent beat is not found: one that exists but cannot be dated may be a
// live node's, so it is an error the claim refuses on (ADR-0003:S7).
func TestClusterHeartbeats_LastSeen(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
require.NoError(t, err)
h := clusterHeartbeats{kv: kv, js: js}
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-a"), []byte(`{"node":"node-a","timestamp":"2026-11-05T01:02:03Z"}`))
require.NoError(t, err)
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), []byte(`not json`))
require.NoError(t, err)
seen, found, err := h.LastSeen(t.Context(), "node-a")
require.NoError(t, err)
require.True(t, found)
assert.True(t, seen.Equal(time.Date(2026, 11, 5, 1, 2, 3, 0, time.UTC)))
_, found, err = h.LastSeen(t.Context(), "node-b")
require.Error(t, err, "ADR-0003:S7 (uefi_store_claims.go LastSeen): an undecodable heartbeat must not read as absent, or a live holder is taken over")
assert.False(t, found)
_, found, err = h.LastSeen(t.Context(), "node-c")
require.NoError(t, err)
assert.False(t, found)
}
// A bucket that cannot be read is an error, which the claim reads as a
// holder that may be live.
func TestClusterHeartbeats_UnreadableBucketIsAnError(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
require.NoError(t, err)
require.NoError(t, js.DeleteKeyValue(t.Context(), clusterv1.ClusterStateBucket))
_, _, err = clusterHeartbeats{kv: kv, js: js}.LastSeen(t.Context(), "node-a")
require.Error(t, err)
}
// laggingReplica answers every get as a replica behind the leader would:
// with the beat it held before, or with no key at all.
type laggingReplica struct {
jetstream.KeyValue
old []byte
}
func (r laggingReplica) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
if r.old == nil {
return nil, jetstream.ErrKeyNotFound
}
return staleEntry{value: r.old}, nil
}
type staleEntry struct {
jetstream.KeyValueEntry
value []byte
}
func (e staleEntry) Value() []byte { return e.value }
// TestS7_HeartbeatALaggingReplicaMissesIsReadFromTheLeader enforces
// ADR-0003:S7: a get a lagging replica answers with an old beat, or none, must
// not read as a dead holder while the leader holds a fresh one.
func TestS7_HeartbeatALaggingReplicaMissesIsReadFromTheLeader(t *testing.T) {
old, err := json.Marshal(clusterv1.HeartbeatTimestamp{Timestamp: time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)})
require.NoError(t, err)
for name, replica := range map[string][]byte{"an old beat": old, "no key": nil} {
t.Run(name, func(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
require.NoError(t, err)
fresh := time.Now().UTC().Truncate(time.Second)
body, err := json.Marshal(clusterv1.HeartbeatTimestamp{Timestamp: fresh.Format(time.RFC3339)})
require.NoError(t, err)
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), body)
require.NoError(t, err)
seen, found, err := clusterHeartbeats{kv: laggingReplica{KeyValue: kv, old: replica}, js: js}.LastSeen(t.Context(), "node-b")
require.NoError(t, err)
require.True(t, found, "ADR-0003:S7 (uefi_store_claims.go LastSeen): a lagging replica's answer was trusted; confirm a stale or absent beat with the stream leader")
assert.True(t, seen.Equal(fresh), "ADR-0003:S7: the leader's beat must be the one returned")
assert.True(t, clusterv1.LiveAt(seen, time.Now()))
})
}
}
// A deleted heartbeat is no heartbeat, as an absent key is.
func TestClusterHeartbeats_DeletedBeatIsNotFound(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
require.NoError(t, err)
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), []byte(`{"timestamp":"2026-11-05T01:02:03Z"}`))
require.NoError(t, err)
require.NoError(t, kv.Delete(t.Context(), clusterv1.HeartbeatKey("node-b")))
_, found, err := clusterHeartbeats{kv: kv, js: js}.LastSeen(t.Context(), "node-b")
require.NoError(t, err)
assert.False(t, found)
}
+49 -4
View File
@@ -13,6 +13,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -78,8 +79,9 @@ type uefiInstanceRecords interface {
var _ uefiInstanceRecords = (*JetStreamManager)(nil)
// uefiStoreReaper deletes the variable stores whose record is deleting: each
// object under the instance's prefix, then the record. Terminate, or an
// operator's forget, sets deleting and returns, so this decides nothing.
// object under the instance's prefix, then the store's claim, then the record.
// Terminate, or an operator's forget, sets deleting and returns, so this
// decides nothing.
//
// It also completes a terminate whose mark failed, from the terminated
// record's pending or failed uefi entry. That record is the terminate itself,
@@ -94,14 +96,15 @@ type uefiStoreReaper struct {
objects objectstore.ObjectStore
bucket string
instances uefiInstanceRecords
claims *kvsync.ClaimGroup
}
var _ vm.Reaper = (*uefiStoreReaper)(nil)
// newUEFIStoreReaper builds the reaper, or nil when this node has no store
// bucket or object store to reap with.
// bucket, claim group or object store to reap with.
func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.uefiClaims() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
return nil
}
return &uefiStoreReaper{
@@ -110,6 +113,7 @@ func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
bucket: d.config.Predastore.Bucket,
instances: d.jsManager,
claims: d.uefiClaims(),
}
}
@@ -172,6 +176,10 @@ func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
"instanceId", id, "deleted", deleted, "err", err)
return false
}
if err := r.removeClaim(ctx, id); err != nil {
slog.WarnContext(ctx, "uefi-store: failed to remove the store's claim, retrying next sweep", "instanceId", id, "err", err)
return false
}
if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record, retrying next sweep", "instanceId", id, "err", err)
return false
@@ -187,6 +195,43 @@ func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
return true
}
// removeClaimAttempts bounds removeClaim's retries against a holder that moves
// the key on between the read and the takeover.
const removeClaimAttempts = 3
// removeClaim deletes instanceID's store claim by taking it over from whatever
// holder it names and unlocking it. The store is deleting, so no stamp or
// pointer move can follow, and the holder needs no proof of being dead.
func (r *uefiStoreReaper) removeClaim(ctx context.Context, id string) error {
claim := r.claims.Get(id)
for range removeClaimAttempts {
cur, held, err := claim.Holder(ctx)
if err != nil {
return err
}
if !held {
return nil
}
tok, err := claim.TakeOver(ctx, cur)
if _, moved := errors.AsType[*kvsync.HeldError](err); moved {
continue
}
if err != nil {
return err
}
err = claim.Unlock(ctx, tok)
if errors.Is(err, kvsync.ErrLost) {
continue
}
if err != nil {
return err
}
slog.InfoContext(ctx, "uefi-store: claim removed", "instanceId", id, "holder", cur.Holder, "generation", cur.Generation)
return nil
}
return fmt.Errorf("uefi-store: claim on %s kept moving after %d attempts", id, removeClaimAttempts)
}
// completeTerminatedMarks finishes the mark of each terminate that recorded a
// uefi entry but did not land its store record in deleting.
func (r *uefiStoreReaper) completeTerminatedMarks(ctx context.Context) {
@@ -14,6 +14,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/stretchr/testify/assert"
@@ -36,10 +37,16 @@ func newUEFIReaperFixture(t *testing.T) *uefiReaperFixture {
return &uefiReaperFixture{
m: m,
objects: objects,
r: &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m},
r: &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m,
claims: uefistore.NewClaims(kvstore.OpenSyncBucket(m.js), "node-reaper")},
}
}
// claim is id's store claim as node sees it.
func (f *uefiReaperFixture) claim(id, node string) *kvsync.Claim {
return uefistore.NewClaims(kvstore.OpenSyncBucket(f.m.js), node).Get(id)
}
// seedStore writes a live store record and n objects under its prefix, the
// state a launch leaves, then marks it deleting if asked, as terminate does.
func (f *uefiReaperFixture) seedStore(t *testing.T, id string, n int, deleting bool) {
@@ -344,3 +351,73 @@ func TestS7_UEFIStoreReaperLeavesTheStoreWhenTheTerminatedRecordExpired(t *testi
assert.Equal(t, 3, f.objectCount(t, id), "ADR-0003:S7: store objects deleted with no record proving QEMU exited")
assert.Equal(t, uefistore.StateLive, f.recordState(t, id))
}
// The reaper removes a deleted store's claim whoever holds it, so a later
// instance can never find a stale holder, and leaves every live store's
// claim to the node that may still hold its unlanded variables.
func TestUEFIStoreReaper_RemovesOnlyADeletedStoresClaim(t *testing.T) {
f := newUEFIReaperFixture(t)
const (
deleted = "i-00000000000deleted"
live = "i-0000000000000live"
)
f.writeTerminated(t, deleted, vm.TeardownPending)
f.seedStore(t, deleted, 2, true)
f.writeInstance(t, live, vm.StateRunning)
f.seedStore(t, live, 2, false)
for _, id := range []string{deleted, live} {
_, err := f.claim(id, "node-b").TryLock(t.Context())
require.NoError(t, err)
}
reaped, err := f.r.Sweep(t.Context())
require.NoError(t, err)
assert.Equal(t, 1, reaped)
_, held, err := f.claim(deleted, "observer").Holder(t.Context())
require.NoError(t, err)
assert.False(t, held, "the deleted store's claim must be removed")
assert.Equal(t, "absent", f.recordState(t, deleted))
cur, held, err := f.claim(live, "observer").Holder(t.Context())
require.NoError(t, err)
require.True(t, held, "a live store's claim must survive the sweep")
assert.Equal(t, "node-b", cur.Holder)
}
// failingClaims stands in for a sync bucket that does not answer.
type failingClaims struct{}
func (failingClaims) Load(context.Context, string) ([]byte, uint64, error) {
return nil, 0, kvsync.ErrUnavailable
}
func (failingClaims) CompareAndSwap(context.Context, string, []byte, uint64) (uint64, error) {
return 0, kvsync.ErrUnavailable
}
func (failingClaims) CompareAndDelete(context.Context, string, uint64) error {
return kvsync.ErrUnavailable
}
// A claim the reaper cannot remove keeps the store record, so the next sweep
// still finds the obligation and finishes it.
func TestUEFIStoreReaper_KeepsTheRecordUntilTheClaimIsRemoved(t *testing.T) {
f := newUEFIReaperFixture(t)
const id = "i-0000000000000000a"
f.writeTerminated(t, id, vm.TeardownPending)
f.seedStore(t, id, 2, true)
healthy := f.r.claims
f.r.claims = uefistore.NewClaims(&failingClaims{}, "node-reaper")
reaped, err := f.r.Sweep(t.Context())
require.NoError(t, err)
assert.Zero(t, reaped)
assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
assert.Equal(t, string(vm.TeardownPending), f.terminatedUEFIMark(t, id))
f.r.claims = healthy
reaped, err = f.r.Sweep(t.Context())
require.NoError(t, err)
assert.Equal(t, 1, reaped)
assert.Equal(t, "absent", f.recordState(t, id))
}
+6 -3
View File
@@ -878,9 +878,12 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
ConsumeCleanShutdownMarker: d.consumeCleanShutdownMarker(),
BackingStoreReady: func() bool { return d.checkPredastoreReady() && d.checkViperblockReady() },
UEFIVarStore: vm.UEFIVarStoreDeps{
Objects: d.instanceObjects,
Bucket: d.config.Predastore.Bucket,
DataDir: DataDirOrDefault(d.config.DataDir),
Objects: d.instanceObjects,
Bucket: d.config.Predastore.Bucket,
DataDir: DataDirOrDefault(d.config.DataDir),
Records: d.uefiStoreRecords(),
Claims: d.uefiClaims(),
Liveness: d.heartbeatLiveness(),
},
}
}
@@ -199,6 +199,11 @@ func TestBuildVMManagerDeps_UEFIVarStore(t *testing.T) {
d.config.DataDir = ""
assert.Equal(t, DefaultDataDir, d.buildVMManagerDeps().UEFIVarStore.DataDir)
// Before the buckets and locks exist, each is unwired rather than a typed nil.
assert.Nil(t, got.Records)
assert.Nil(t, got.Claims)
assert.Nil(t, got.Liveness)
}
// --- volumeMounterAdapter ---