mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-05 18:46:28 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c826cbd158 | |||
| 82b3b55d70 | |||
| c864bfbc84 | |||
| c9c6dae176 | |||
| c99ff34d53 |
@@ -1,6 +1,9 @@
|
||||
package clusterv1
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
|
||||
// other cluster-wide state.
|
||||
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
|
||||
return "heartbeat." + node
|
||||
}
|
||||
|
||||
// LivenessWindow is how recently a node must have heartbeated for a reader
|
||||
// deciding on a takeover to count it live. It is six heartbeats, compared
|
||||
// across node clocks, where NTP skew is small against it.
|
||||
const LivenessWindow = 60 * time.Second
|
||||
|
||||
// LiveAt reports whether a heartbeat written at written still vouches for its
|
||||
// node at now: it does while it is under LivenessWindow old.
|
||||
func LiveAt(written, now time.Time) bool {
|
||||
return now.Sub(written) < LivenessWindow
|
||||
}
|
||||
|
||||
// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
|
||||
// or dated vouches for nothing, which is an answer rather than a fault.
|
||||
func HeartbeatWritten(data []byte) (time.Time, bool) {
|
||||
var beat HeartbeatTimestamp
|
||||
if json.Unmarshal(data, &beat) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return beat.Time()
|
||||
}
|
||||
|
||||
// HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
|
||||
// decodes only the timestamp, so a reader does not depend on the capacity
|
||||
// figures the daemon writes beside it.
|
||||
|
||||
@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
|
||||
assert.False(t, ok, "timestamp %q", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// A beat is live strictly under the window, so one exactly the window old is
|
||||
// the first a takeover may treat as stopped.
|
||||
func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
|
||||
now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
|
||||
assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
|
||||
assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
|
||||
assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
|
||||
}
|
||||
|
||||
func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
|
||||
written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
|
||||
require.True(t, ok)
|
||||
assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
|
||||
|
||||
for _, raw := range []string{"not json", `{"node":"node-a"}`} {
|
||||
_, ok := clusterv1.HeartbeatWritten([]byte(raw))
|
||||
assert.False(t, ok, raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1265,6 +1265,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
|
||||
slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
|
||||
return refusal
|
||||
}
|
||||
if errors.Is(err, vm.ErrUEFIClaimHeld) {
|
||||
slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
|
||||
return errInsufficientCapacity
|
||||
}
|
||||
slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
@@ -2760,7 +2764,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
|
||||
func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
|
||||
// Before anything is stamped or claimed, so the record handed back is the
|
||||
// one that was claimed.
|
||||
if err := vm.CheckUEFIVarStore(instance); err != nil {
|
||||
if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
|
||||
slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
|
||||
"instanceId", instance.ID, "err", err)
|
||||
undo()
|
||||
@@ -2893,6 +2897,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
|
||||
if refusal := volumeHeldElsewhereError(err); refusal != nil {
|
||||
return refusal
|
||||
}
|
||||
// A live node holds the variable store, so another node may take it.
|
||||
if errors.Is(err, vm.ErrUEFIClaimHeld) {
|
||||
return s.capacityRefusal(ctx, instance)
|
||||
}
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
|
||||
@@ -25,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
|
||||
return s.kv.Exists(ctx, instanceID)
|
||||
}
|
||||
|
||||
// MarkDeleting records that instanceID's store is to be deleted, and drops any
|
||||
// write-back still owed. It reports false with no error when the instance has
|
||||
// no store record. Only a terminate or an operator's forget may call it.
|
||||
// MarkDeleting records that instanceID's store is to be deleted. It reports
|
||||
// false with no error when the instance has no store record. Only a terminate
|
||||
// or an operator's forget may call it.
|
||||
func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
|
||||
for range markAttempts {
|
||||
rec, rev, err := s.Get(ctx, instanceID)
|
||||
@@ -41,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
|
||||
return true, nil
|
||||
}
|
||||
rec.State = StateDeleting
|
||||
rec.WriteBack = nil
|
||||
_, err = s.CompareAndSet(ctx, rec, rev)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
|
||||
@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
|
||||
}
|
||||
|
||||
// Deletion is final: a writer holding the revision of a deleting record, such
|
||||
// as a write-back retry, must not bring the store back to live.
|
||||
// as a late pointer move, must not bring the store back to live.
|
||||
func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
|
||||
rev, err = s.CompareAndSet(t.Context(), deleting, rev)
|
||||
require.NoError(t, err)
|
||||
|
||||
revived := liveRecord()
|
||||
revived.WriteBack = writeBack()
|
||||
revived := pointedRecord()
|
||||
_, err = s.CompareAndSet(t.Context(), revived, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrDeleting)
|
||||
|
||||
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
|
||||
assert.Equal(t, uefistore.StateDeleting, got.State)
|
||||
}
|
||||
|
||||
// A write-back owed on a deleting store would put an object behind the delete.
|
||||
func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
|
||||
func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
rec := liveRecord()
|
||||
rec.State = uefistore.StateDeleting
|
||||
rec.WriteBack = writeBack()
|
||||
|
||||
_, err = s.CompareAndSet(t.Context(), rec, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
}
|
||||
|
||||
func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
has, err := s.MarkDeleting(t.Context(), instanceID)
|
||||
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
|
||||
got, rev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uefistore.StateDeleting, got.State)
|
||||
assert.Nil(t, got.WriteBack)
|
||||
assert.Equal(t, "111122223333", got.AccountID)
|
||||
assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
|
||||
|
||||
has, err = s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package uefistore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
)
|
||||
|
||||
// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
|
||||
// is ClaimPrefix+instanceID, and names the node that may hold variable changes
|
||||
// no object has yet.
|
||||
const ClaimPrefix = "uefi."
|
||||
|
||||
// NewClaims returns node's group of store claims over store. The daemon builds
|
||||
// it once per process, as kvsync requires of every prefix.
|
||||
func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
|
||||
return kvsync.NewClaimGroup(store, ClaimPrefix, node)
|
||||
}
|
||||
|
||||
// ErrFenced is a stamp or pointer move refused because the record carries a
|
||||
// claim generation other than the caller's: a later acquisition has stamped
|
||||
// it, so the caller no longer owns the store and must change nothing.
|
||||
var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
|
||||
|
||||
// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
|
||||
// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
|
||||
const fenceAttempts = 5
|
||||
|
||||
// Stamp raises instanceID's record to generation, the claim token the caller
|
||||
// has just acquired or adopted, and returns the record it stamped, whose
|
||||
// pointer the caller boots from. One above generation is ErrFenced, and one
|
||||
// deleting is ErrDeleting. A record already at generation is still written by
|
||||
// CAS, because a follower can answer the read with an older pointer.
|
||||
func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
|
||||
if generation == 0 {
|
||||
return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
|
||||
}
|
||||
for range fenceAttempts {
|
||||
rec, rev, err := s.Get(ctx, instanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case rec.State == StateDeleting:
|
||||
return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
|
||||
case rec.Generation > generation:
|
||||
return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
|
||||
}
|
||||
rec.Generation = generation
|
||||
_, err = s.CompareAndSet(ctx, rec, rev)
|
||||
if err == nil {
|
||||
return rec, nil
|
||||
}
|
||||
if !errors.Is(err, kvstore.ErrConflict) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
|
||||
}
|
||||
|
||||
// MovePointer points instanceID's record at ref only while the record is at
|
||||
// generation. A record at any other generation is ErrFenced and is left
|
||||
// unchanged. Every move is a CAS at the revision read, even to the current
|
||||
// key, so a read a follower answered stale cannot report success.
|
||||
func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
|
||||
if err := ref.validate(instanceID); err != nil {
|
||||
return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
|
||||
}
|
||||
for range fenceAttempts {
|
||||
rec, rev, err := s.Get(ctx, instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case rec.State == StateDeleting:
|
||||
return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
|
||||
case rec.Generation != generation:
|
||||
return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
|
||||
}
|
||||
rec.Pointer = &ref
|
||||
_, err = s.CompareAndSet(ctx, rec, rev)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, kvstore.ErrConflict) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package uefistore_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
|
||||
t.Helper()
|
||||
_, rev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
return rev
|
||||
}
|
||||
|
||||
func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
rec, err := s.Stamp(t.Context(), instanceID, 9)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(9), rec.Generation)
|
||||
assert.Equal(t, pointer("b"), rec.Pointer)
|
||||
|
||||
got, _, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rec, got, "the returned record is the one stored")
|
||||
}
|
||||
|
||||
// A daemon restart adopts its claim at the same generation. The stamp still
|
||||
// writes by CAS, so a pointer read from a lagging follower fails the write and
|
||||
// is read again rather than booted from.
|
||||
func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
before := revisionOf(t, s)
|
||||
|
||||
rec, err := s.Stamp(t.Context(), instanceID, 7)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pointer("b"), rec.Pointer)
|
||||
assert.Equal(t, uint64(7), rec.Generation)
|
||||
assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
|
||||
}
|
||||
|
||||
func TestStamp_RefusesAndChangesNothing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deleting bool
|
||||
generation uint64
|
||||
want error
|
||||
}{
|
||||
{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
|
||||
{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
if tt.deleting {
|
||||
_, err = s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
rev = revisionOf(t, s)
|
||||
}
|
||||
|
||||
_, err = s.Stamp(t.Context(), instanceID, tt.generation)
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
assert.Equal(t, rev, revisionOf(t, s))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStamp_MissingRecordIsNotFound(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Stamp(t.Context(), instanceID, 1)
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound)
|
||||
}
|
||||
|
||||
// The record keeps only the current version, so each move replaces the
|
||||
// pointer and names nothing it replaced.
|
||||
func TestMovePointer_ReplacesThePointer(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, name := range []string{"c", "d"} {
|
||||
require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
|
||||
}
|
||||
|
||||
got, _, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pointer("d"), got.Pointer)
|
||||
assert.Equal(t, uint64(7), got.Generation)
|
||||
}
|
||||
|
||||
// A move to the key the record already names is still a CAS at the revision
|
||||
// read, so a fenced writer that read a lagging follower cannot see success.
|
||||
func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
before := revisionOf(t, s)
|
||||
|
||||
require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
|
||||
assert.Greater(t, revisionOf(t, s), before)
|
||||
}
|
||||
|
||||
func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deleting bool
|
||||
generation uint64
|
||||
ref *uefistore.UEFIVarStoreRef
|
||||
want error
|
||||
}{
|
||||
{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
|
||||
{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
|
||||
{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
|
||||
{name: "a key under another instance", generation: 7,
|
||||
ref: &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
|
||||
want: uefistore.ErrInvalidRecord},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
if tt.deleting {
|
||||
_, err = s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
rev := revisionOf(t, s)
|
||||
|
||||
err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
got, gotRev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev)
|
||||
assert.Equal(t, pointer("b"), got.Pointer)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// claimedStore is a store record and a claim group per node over one
|
||||
// JetStream, as two daemons see them.
|
||||
type claimedStore struct {
|
||||
records *uefistore.Store
|
||||
a, b *kvsync.Claim
|
||||
}
|
||||
|
||||
func newClaimedStore(t *testing.T) *claimedStore {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
records := uefistore.NewStore(js)
|
||||
require.NoError(t, records.Open(t.Context()))
|
||||
_, err := records.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
return &claimedStore{
|
||||
records: records,
|
||||
a: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
|
||||
b: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
|
||||
}
|
||||
}
|
||||
|
||||
// A node partitioned long enough to be taken over is fenced by the new
|
||||
// owner's stamp: its pointer move fails and the record keeps what it had.
|
||||
func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
|
||||
c := newClaimedStore(t)
|
||||
first, err := c.a.TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
|
||||
|
||||
second, err := c.b.TakeOver(t.Context(), first)
|
||||
require.NoError(t, err)
|
||||
_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
|
||||
require.NoError(t, err)
|
||||
rev := revisionOf(t, c.records)
|
||||
|
||||
err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
|
||||
require.ErrorIs(t, err, uefistore.ErrFenced)
|
||||
got, gotRev, err := c.records.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev)
|
||||
assert.Equal(t, pointer("b"), got.Pointer)
|
||||
assert.Equal(t, second.Generation, got.Generation)
|
||||
}
|
||||
|
||||
// The new owner reads the pointer after it stamps, so a move the old holder
|
||||
// landed between the takeover and the stamp is the one it boots from.
|
||||
func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
|
||||
c := newClaimedStore(t)
|
||||
first, err := c.a.TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
|
||||
require.NoError(t, err)
|
||||
|
||||
second, err := c.b.TakeOver(t.Context(), first)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
|
||||
|
||||
rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
// Package uefistore holds the types that describe an instance's UEFI variable
|
||||
// store and the bucket of store records that carry its deletion and write-back
|
||||
// obligations past the life of the instance record.
|
||||
// store, the bucket of store records that carry its pointer and deletion
|
||||
// obligation past the life of the instance record, and the claim that fences it.
|
||||
package uefistore
|
||||
|
||||
import (
|
||||
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
|
||||
// UEFIStoreRecord is the durable owner of one instance's variable store
|
||||
// objects. It outlives the instance record, so an obligation recorded here is
|
||||
// not lost when a terminated record expires.
|
||||
//
|
||||
// Pointer names the only version of the store the record keeps, the object the
|
||||
// instance boots from, and only a writer whose claim token has generation
|
||||
// Generation may move it.
|
||||
type UEFIStoreRecord struct {
|
||||
SchemaVersion uint16 `json:"schema_version"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
AccountID string `json:"account_id"`
|
||||
State string `json:"state"`
|
||||
WriteBack *UEFIWriteBack `json:"write_back,omitempty"`
|
||||
SchemaVersion uint16 `json:"schema_version"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
AccountID string `json:"account_id"`
|
||||
State string `json:"state"`
|
||||
Generation uint64 `json:"generation"`
|
||||
Pointer *UEFIVarStoreRef `json:"pointer,omitempty"`
|
||||
}
|
||||
|
||||
// UEFIWriteBack is a write-back that failed and is still owed. Key is the
|
||||
// object every retry puts to, so retries leave no extra objects; SeededFrom is
|
||||
// the pointer Node seeded from, which a retry must still find on the record.
|
||||
type UEFIWriteBack struct {
|
||||
Node string `json:"node"`
|
||||
Key string `json:"key"`
|
||||
SeededFrom string `json:"seeded_from"`
|
||||
Since time.Time `json:"since"`
|
||||
// UEFIVarStoreRef names the store object an instance launches from, with the
|
||||
// firmware it was written under, so a launch can check it without reading the
|
||||
// object's metadata.
|
||||
type UEFIVarStoreRef struct {
|
||||
Key string `json:"key"`
|
||||
Firmware FirmwareIdentity `json:"firmware"`
|
||||
WriterNode string `json:"writer_node"`
|
||||
WrittenAt time.Time `json:"written_at"`
|
||||
}
|
||||
|
||||
// KeyPrefix is the object-store prefix every store object of ownerID (an
|
||||
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
|
||||
return ok && name != "" && !strings.Contains(name, "/")
|
||||
}
|
||||
|
||||
// validate checks rec against key. A write-back key outside the instance's own
|
||||
// prefix is refused because a superseded obligation deletes that key.
|
||||
// validate checks rec against key. The pointer must name one object directly
|
||||
// under the instance's own prefix, because a deletion acts on it.
|
||||
func (rec *UEFIStoreRecord) validate(key string) error {
|
||||
switch {
|
||||
case rec.SchemaVersion > RecordSchemaVersion:
|
||||
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
|
||||
return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
|
||||
case rec.State != StateLive && rec.State != StateDeleting:
|
||||
return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
|
||||
case rec.State == StateDeleting && rec.WriteBack != nil:
|
||||
// A write-back retry puts an object, which would land behind the delete.
|
||||
return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
|
||||
}
|
||||
if wb := rec.WriteBack; wb != nil {
|
||||
switch {
|
||||
case wb.Node == "":
|
||||
return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
|
||||
case wb.Since.IsZero():
|
||||
return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
|
||||
case !ownsKey(rec.InstanceID, wb.Key):
|
||||
return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
|
||||
case !ownsKey(rec.InstanceID, wb.SeededFrom):
|
||||
return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
|
||||
if p := rec.Pointer; p != nil {
|
||||
if err := p.validate(rec.InstanceID); err != nil {
|
||||
return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validate checks ref as a pointer of instanceID's store.
|
||||
func (ref *UEFIVarStoreRef) validate(instanceID string) error {
|
||||
switch {
|
||||
case !ownsKey(instanceID, ref.Key):
|
||||
return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
|
||||
case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
|
||||
return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
|
||||
case ref.WriterNode == "":
|
||||
return fmt.Errorf("key %q names no writer node", ref.Key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func bucketConfig() kvstore.Config {
|
||||
return kvstore.Config{
|
||||
Name: BucketName,
|
||||
Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
|
||||
Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
|
||||
History: 1,
|
||||
// Left false: a lost bucket has lost obligations, and an empty
|
||||
// recreated one would report every store as having none.
|
||||
|
||||
@@ -30,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
|
||||
return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
|
||||
}
|
||||
|
||||
func writeBack() *uefistore.UEFIWriteBack {
|
||||
return &uefistore.UEFIWriteBack{
|
||||
Node: "node-2",
|
||||
Key: uefistore.KeyPrefix(instanceID) + "b",
|
||||
SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
|
||||
Since: time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
|
||||
func pointer(name string) *uefistore.UEFIVarStoreRef {
|
||||
return &uefistore.UEFIVarStoreRef{
|
||||
Key: keyOf(name),
|
||||
Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
|
||||
WriterNode: "node-2",
|
||||
WrittenAt: time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
|
||||
|
||||
// pointedRecord is a live record stamped at generation 7 that points at "b".
|
||||
func pointedRecord() *uefistore.UEFIStoreRecord {
|
||||
rec := liveRecord()
|
||||
rec.Generation = 7
|
||||
rec.Pointer = pointer("b")
|
||||
return rec
|
||||
}
|
||||
|
||||
func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
|
||||
t.Helper()
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
@@ -66,8 +76,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
|
||||
|
||||
func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
rec := pointedRecord()
|
||||
|
||||
rev, err := s.Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
@@ -82,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
|
||||
// The wire names are what every release reads, so they are pinned.
|
||||
func TestRecord_WireShape(t *testing.T) {
|
||||
s, js := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
_, err := s.Create(t.Context(), pointedRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
@@ -96,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
|
||||
"instance_id": "i-0123456789abcdef0",
|
||||
"account_id": "111122223333",
|
||||
"state": "live",
|
||||
"write_back": {
|
||||
"node": "node-2",
|
||||
"generation": 7,
|
||||
"pointer": {
|
||||
"key": "uefi/i-0123456789abcdef0/b",
|
||||
"seeded_from": "uefi/i-0123456789abcdef0/a",
|
||||
"since": "2026-11-03T01:02:03Z"
|
||||
"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
|
||||
"writer_node": "node-2",
|
||||
"written_at": "2026-11-03T01:02:03Z"
|
||||
}
|
||||
}`, string(entry.Value()))
|
||||
}
|
||||
@@ -144,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
|
||||
first, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
moved := liveRecord()
|
||||
moved.WriteBack = writeBack()
|
||||
moved := pointedRecord()
|
||||
second, err := s.CompareAndSet(t.Context(), moved, first)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, second, first)
|
||||
@@ -159,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, second, rev)
|
||||
assert.Equal(t, uefistore.StateLive, got.State)
|
||||
assert.Equal(t, writeBack(), got.WriteBack)
|
||||
assert.Equal(t, pointer("b"), got.Pointer)
|
||||
}
|
||||
|
||||
// Two actors that read the same revision race to change it; exactly one may
|
||||
@@ -179,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
|
||||
)
|
||||
for i := range writers {
|
||||
wg.Go(func() {
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
|
||||
rec := pointedRecord()
|
||||
rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
|
||||
if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
|
||||
mu.Lock()
|
||||
wins = append(wins, rec.WriteBack.Node)
|
||||
wins = append(wins, rec.Pointer.WriterNode)
|
||||
mu.Unlock()
|
||||
} else {
|
||||
assert.ErrorIs(t, err, kvstore.ErrConflict)
|
||||
@@ -196,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
|
||||
require.Len(t, wins, 1)
|
||||
got, _, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wins[0], got.WriteBack.Node)
|
||||
assert.Equal(t, wins[0], got.Pointer.WriterNode)
|
||||
}
|
||||
|
||||
func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
|
||||
@@ -217,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound)
|
||||
}
|
||||
|
||||
// A superseded obligation deletes its recorded key, so a key outside the
|
||||
// A deletion acts on the key the record points at, so a key outside the
|
||||
// instance's own prefix must never reach the bucket.
|
||||
func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
|
||||
other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
|
||||
cases := map[string]func(*uefistore.UEFIStoreRecord){
|
||||
"unknown state": func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
|
||||
"not an instance ID": func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
|
||||
"write-back with no node": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
|
||||
"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
|
||||
"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
|
||||
},
|
||||
"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
|
||||
},
|
||||
"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
|
||||
},
|
||||
"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
|
||||
},
|
||||
"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
|
||||
},
|
||||
"unknown state": func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
|
||||
"not an instance ID": func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
|
||||
"pointer under another instance": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
|
||||
"pointer under an image": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
|
||||
"pointer is the prefix itself": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
|
||||
"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
|
||||
"pointer with no firmware arch": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
|
||||
"pointer with no VARS size": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
|
||||
"pointer with no writer node": func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
@@ -247,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
rec := pointedRecord()
|
||||
mutate(rec)
|
||||
_, err = s.CompareAndSet(t.Context(), rec, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
@@ -256,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
|
||||
got, gotRev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev, "an invalid record must not be written")
|
||||
assert.Nil(t, got.WriteBack)
|
||||
assert.Nil(t, got.Pointer)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -265,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
|
||||
// is reported rather than decoded into something a reaper would act on.
|
||||
func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"newer schema": `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"no schema": `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
|
||||
"unknown state": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
|
||||
"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
|
||||
"newer schema": `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"no schema": `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
|
||||
"unknown state": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
|
||||
"foreign pointer": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
|
||||
@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
|
||||
assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
|
||||
})
|
||||
}
|
||||
|
||||
// uefiClaimHeldMounter fails the launch the way a start does while a live
|
||||
// node holds the instance's UEFI variable store.
|
||||
type uefiClaimHeldMounter struct{ raceVolumeMounter }
|
||||
|
||||
func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
|
||||
return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
|
||||
}
|
||||
|
||||
// A held UEFI store is a capacity refusal on both start paths, so the caller
|
||||
// tries another node rather than reporting a fault.
|
||||
func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
|
||||
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
|
||||
types, _ := defaultPrepareInstanceTypes()
|
||||
|
||||
t.Run("claimed relaunch", func(t *testing.T) {
|
||||
mgr := vm.NewManager()
|
||||
mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
|
||||
svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
|
||||
|
||||
err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
|
||||
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
|
||||
})
|
||||
|
||||
t.Run("live owner", func(t *testing.T) {
|
||||
const id = "i-uefi-live"
|
||||
mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
|
||||
mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
|
||||
v, _ := mgr.Get(id)
|
||||
svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
|
||||
|
||||
err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
|
||||
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package viperblock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
@@ -14,9 +13,8 @@ import (
|
||||
)
|
||||
|
||||
// nodeLivenessWindow is how recently a node must have been seen for its dirty
|
||||
// marker to bar a takeover. It is six heartbeats and above volumeLeaseTTL, and is
|
||||
// compared across node clocks, where NTP skew is small against 60s.
|
||||
const nodeLivenessWindow = 60 * time.Second
|
||||
// marker to bar a takeover: the cluster's heartbeat rule, above volumeLeaseTTL.
|
||||
const nodeLivenessWindow = clusterv1.LivenessWindow
|
||||
|
||||
// nodeLiveness reports when a node was last seen alive. found is false for a
|
||||
// node that has never been seen or whose record has expired; an error means the
|
||||
@@ -75,16 +73,6 @@ func (l *heartbeatLiveness) LastSeen(ctx context.Context, node string) (time.Tim
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("read heartbeat for %s: %w", node, err)
|
||||
}
|
||||
written, ok := heartbeatTime(entry.Value())
|
||||
written, ok := clusterv1.HeartbeatWritten(entry.Value())
|
||||
return written, ok, nil
|
||||
}
|
||||
|
||||
// heartbeatTime dates a stored heartbeat. One that cannot be read vouches for
|
||||
// nothing, which is an answer rather than a fault.
|
||||
func heartbeatTime(data []byte) (time.Time, bool) {
|
||||
var beat clusterv1.HeartbeatTimestamp
|
||||
if json.Unmarshal(data, &beat) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return beat.Time()
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
@@ -266,10 +267,11 @@ func (cfg *Config) refuseWhileOwnerLive(ctx context.Context, volumeName string,
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
age := time.Since(seen)
|
||||
if age > nodeLivenessWindow {
|
||||
now := time.Now()
|
||||
if !clusterv1.LiveAt(seen, now) {
|
||||
return nil
|
||||
}
|
||||
age := now.Sub(seen)
|
||||
slog.WarnContext(ctx, "refusing to open a volume whose previous owner is still live",
|
||||
"volume", volumeName, "previous_owner", record.Owner, "last_seen_ms", telemetry.Millis(age))
|
||||
return fmt.Errorf("%w: %s is still live and holds unsealed writes", errVolumeLeaseHeld, record.Owner)
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"time"
|
||||
|
||||
telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
@@ -82,7 +83,7 @@ func guestOOMScore(managedBy string) int {
|
||||
// transition to Running, fire OnInstanceUp. Inserts the instance before
|
||||
// transitioning. Used by RunInstances, start-stopped handler, restore, and crash recovery.
|
||||
func (m *Manager) Run(ctx context.Context, instance *VM) error {
|
||||
return m.launch(ctx, instance)
|
||||
return m.launch(ctx, instance, nil)
|
||||
}
|
||||
|
||||
// Start re-launches a stopped instance by id. Returns ErrInstanceNotFound when
|
||||
@@ -92,7 +93,8 @@ func (m *Manager) Start(ctx context.Context, id string) error {
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrInstanceNotFound, id)
|
||||
}
|
||||
if err := CheckUEFIVarStore(instance); err != nil {
|
||||
rec, err := m.checkUEFIVarStore(ctx, instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// StateError is rejected by launchStillValid; move to Pending first so
|
||||
@@ -102,7 +104,13 @@ func (m *Manager) Start(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return m.launch(ctx, instance)
|
||||
return m.launch(ctx, instance, &uefiPrecheck{rec: rec})
|
||||
}
|
||||
|
||||
// uefiPrecheck is a variable store check a caller of launch already made,
|
||||
// so launch does not read the store record again.
|
||||
type uefiPrecheck struct {
|
||||
rec *uefistore.UEFIStoreRecord
|
||||
}
|
||||
|
||||
// launchStillValid returns true when status is still pending/stopped/provisioning.
|
||||
@@ -117,8 +125,9 @@ func (m *Manager) launchStillValid(instance *VM) bool {
|
||||
}
|
||||
|
||||
// launch is the orchestrator: pid check, mount volumes, exec QEMU, attach
|
||||
// QMP, fire OnInstanceUp, transition to Running.
|
||||
func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
|
||||
// QMP, fire OnInstanceUp, transition to Running. A nil pre checks the
|
||||
// variable store here.
|
||||
func (m *Manager) launch(ctx context.Context, instance *VM, pre *uefiPrecheck) (err error) {
|
||||
ctx, span := otel.Tracer(vmTracerName).Start(ctx, "vm.launch",
|
||||
trace.WithAttributes(
|
||||
attribute.String("instance.id", instance.ID),
|
||||
@@ -131,10 +140,14 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
|
||||
}
|
||||
|
||||
// Before the reason below is cleared, so a refused start changes nothing.
|
||||
if err := CheckUEFIVarStore(instance); err != nil {
|
||||
slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
|
||||
"instanceId", instance.ID, "err", err)
|
||||
return err
|
||||
if pre == nil {
|
||||
rec, err := m.checkUEFIVarStore(ctx, instance)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
|
||||
"instanceId", instance.ID, "err", err)
|
||||
return err
|
||||
}
|
||||
pre = &uefiPrecheck{rec: rec}
|
||||
}
|
||||
|
||||
// A reason left by the stop, or by a restore that found this node could not
|
||||
@@ -159,10 +172,17 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
|
||||
}
|
||||
|
||||
// Before the mount, so a store this node cannot boot fails the launch
|
||||
// with nothing to roll back.
|
||||
// with nothing to roll back but the claim, which a launch that fails
|
||||
// before QEMU starts gives back.
|
||||
uefiCtx, uefiSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.uefi_var_store")
|
||||
uefiErr := m.prepareUEFIVarStore(uefiCtx, instance)
|
||||
uefiHeld, uefiErr := m.prepareUEFIVarStore(uefiCtx, instance, pre.rec)
|
||||
endSpanWithError(uefiSpan, uefiErr)
|
||||
guestMayHaveRun := false
|
||||
defer func() {
|
||||
if err != nil && !guestMayHaveRun {
|
||||
m.releaseUEFIClaim(ctx, instance.ID, uefiHeld)
|
||||
}
|
||||
}()
|
||||
if uefiErr != nil {
|
||||
slog.ErrorContext(ctx, "Failed to prepare the UEFI variable store", "instanceId", instance.ID, "err", uefiErr)
|
||||
return uefiErr
|
||||
@@ -197,6 +217,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
|
||||
|
||||
_, qemuSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.start_qemu")
|
||||
qemuErr := m.startQEMU(instance)
|
||||
guestMayHaveRun = !errors.Is(qemuErr, errQEMUNotStarted)
|
||||
endSpanWithError(qemuSpan, qemuErr)
|
||||
if qemuErr != nil {
|
||||
slog.ErrorContext(ctx, "Failed to launch instance", "err", qemuErr)
|
||||
@@ -301,10 +322,20 @@ func qemuStderrLevel(line string) slog.Level {
|
||||
return slog.LevelError
|
||||
}
|
||||
|
||||
// errQEMUNotStarted wraps a startQEMU failure from before the QEMU process
|
||||
// started, so no guest can have run.
|
||||
var errQEMUNotStarted = errors.New("qemu process was not started")
|
||||
|
||||
// startQEMU launches the QEMU process for instance and waits for startup to
|
||||
// confirm. The console log is opened in append mode, so a replacement process
|
||||
// keeps what the guest printed before it.
|
||||
func (m *Manager) startQEMU(instance *VM) error {
|
||||
func (m *Manager) startQEMU(instance *VM) (err error) {
|
||||
started := false
|
||||
defer func() {
|
||||
if err != nil && !started {
|
||||
err = fmt.Errorf("%w: %w", errQEMUNotStarted, err)
|
||||
}
|
||||
}()
|
||||
pidFile, err := hostprocess.GeneratePidFile(instance.ID)
|
||||
if err != nil {
|
||||
slog.Error("Failed to generate PID file", "err", err)
|
||||
@@ -641,6 +672,7 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
if pid == 0 {
|
||||
return fmt.Errorf("failed to start qemu")
|
||||
}
|
||||
started = true
|
||||
|
||||
// Race QMP-socket appearance against early process exit; SIGKILL on timeout
|
||||
// so a wedged QEMU does not orphan its tap / volumes.
|
||||
|
||||
@@ -107,10 +107,6 @@ type InstanceStatus struct {
|
||||
EBSRequests []EBSRequest `json:"ebs_requests,omitempty"`
|
||||
ENIAvailableSlots []int `json:"eni_available_slots,omitempty"`
|
||||
ENIAttachedByID map[string]int `json:"eni_attached_by_id,omitempty"`
|
||||
|
||||
// UEFIVarStore is optional, so records written without it still decode
|
||||
// and the bucket schema does not change.
|
||||
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
|
||||
}
|
||||
|
||||
// Record projects a VM onto the persisted record. QMPClient and the request
|
||||
@@ -172,7 +168,6 @@ func (v *VM) Record() *InstanceRecord {
|
||||
EBSRequests: v.EBSRequests.Requests,
|
||||
ENIAvailableSlots: v.ENIRequests.AvailableSlots,
|
||||
ENIAttachedByID: v.ENIRequests.AttachedByENIID,
|
||||
UEFIVarStore: v.UEFIVarStore,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -230,7 +225,6 @@ func VMFromRecord(r *InstanceRecord) *VM {
|
||||
TerminatedAt: r.Status.TerminatedAt,
|
||||
ShuttingDownAt: r.Status.ShuttingDownAt,
|
||||
FirstLaunch: r.Status.FirstLaunch,
|
||||
UEFIVarStore: r.Status.UEFIVarStore,
|
||||
}
|
||||
v.EBSRequests.Requests = r.Status.EBSRequests
|
||||
v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
@@ -115,35 +114,22 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
|
||||
assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
|
||||
}
|
||||
|
||||
// The pointer is optional: a record written before it existed decodes with
|
||||
// none, and a record without one writes no key, so the bucket schema holds.
|
||||
func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
|
||||
// The pointer lives on the store record beside the claim generation that
|
||||
// fences it, so the running set's unconditional writes cannot move it. A
|
||||
// record that carried one decodes, and writing it back drops the field.
|
||||
func TestRecord_CarriesNoUEFIVarStorePointer(t *testing.T) {
|
||||
var decoded vm.InstanceRecord
|
||||
require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
|
||||
assert.Nil(t, decoded.Status.UEFIVarStore)
|
||||
require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped",`+
|
||||
`"uefi_var_store":{"key":"uefi/i-old/a","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"node-1"}}}`), &decoded))
|
||||
|
||||
encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(encoded), "uefi_var_store")
|
||||
}
|
||||
|
||||
// The pointer's wire names are what the record carries across releases, so
|
||||
// they are pinned rather than left to follow the Go field names.
|
||||
func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
|
||||
encoded, err := json.Marshal(populatedVM().Record())
|
||||
require.NoError(t, err)
|
||||
|
||||
var wire struct {
|
||||
Status struct {
|
||||
UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
|
||||
} `json:"status"`
|
||||
for name, rec := range map[string]*vm.InstanceRecord{
|
||||
"a record that carried a pointer": vm.VMFromRecord(&decoded).Record(),
|
||||
"a fully populated record": populatedVM().Record(),
|
||||
} {
|
||||
encoded, err := json.Marshal(rec)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(encoded), "uefi_var_store", name)
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(encoded, &wire))
|
||||
ref := wire.Status.UEFIVarStore
|
||||
assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
|
||||
assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
|
||||
assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
|
||||
assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
|
||||
}
|
||||
|
||||
// The mutexes are dropped on the way out, so the rebuilt instance must arrive
|
||||
@@ -238,12 +224,6 @@ func populatedVM() *vm.VM {
|
||||
CapacityReservationId: "cr-1",
|
||||
InstanceLifecycle: "spot",
|
||||
SpotInstanceRequestId: "sir-1",
|
||||
UEFIVarStore: &vm.UEFIVarStoreRef{
|
||||
Key: "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
|
||||
Firmware: uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
|
||||
WriterNode: "node-3",
|
||||
WrittenAt: time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
|
||||
},
|
||||
|
||||
Status: vm.StateRunning,
|
||||
Instance: &ec2.Instance{InstanceId: aws.String("i-round-trip")},
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// holdOnPeer has node-b claim the instance's store and stamp its record, as
|
||||
// the node running the instance has, and returns node-b's token.
|
||||
func holdOnPeer(t *testing.T, f *uefiFixture) (*VM, uint64) {
|
||||
t.Helper()
|
||||
key := keyFor("held")
|
||||
f.putObject(t, key, gzipped(t, region("node-b booted this")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
tok, err := f.peerClaim("node-b").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
_, err = f.records.Stamp(t.Context(), testUEFIInstance, tok.Generation)
|
||||
require.NoError(t, err)
|
||||
return instance, tok.Generation
|
||||
}
|
||||
|
||||
// TestS7_UEFIClaimIsNeverTakenOverWhileItsHolderHeartbeats enforces ADR-0003:S7
|
||||
// for the store claim: a timeout or a missed heartbeat is never a decision, so a
|
||||
// start elsewhere must not take a claim whose holder heartbeated under
|
||||
// clusterv1.LivenessWindow ago, or whose heartbeat cannot be read. It is refused
|
||||
// as retryable, and the claim, the record and the volumes are untouched.
|
||||
func TestS7_UEFIClaimIsNeverTakenOverWhileItsHolderHeartbeats(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
age time.Duration
|
||||
err error
|
||||
reason string
|
||||
}{
|
||||
{name: "heartbeat just written", age: 0, reason: "holder_live"},
|
||||
{name: "heartbeat 30s old", age: 30 * time.Second, reason: "holder_live"},
|
||||
{name: "heartbeat 59s old", age: 59 * time.Second, reason: "holder_live"},
|
||||
{name: "heartbeat unreadable", err: errors.New("cluster-state bucket unreachable"), reason: "liveness_unknown"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
instance, gen := holdOnPeer(t, f)
|
||||
f.liveness.heartbeat("node-b", tt.age)
|
||||
f.liveness.err = tt.err
|
||||
_, rev := f.record(t)
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", tt.reason)
|
||||
|
||||
err := f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrUEFIClaimHeld,
|
||||
"ADR-0003:S7 (uefi_varstore.go acquireUEFIClaim): a start must be refused while the claim's holder may be live; take over only once clusterv1.LiveAt is false")
|
||||
require.ErrorIs(t, err, ErrMountRetryable, "the refusal clears once the holder stops or releases, so restore retries it")
|
||||
holder, held := f.claimHolder(t)
|
||||
require.True(t, held, "ADR-0003:S7: the holder's claim was removed while it heartbeated")
|
||||
assert.Equal(t, "node-b", holder.Holder, "ADR-0003:S7: the claim was taken over while its holder heartbeated")
|
||||
assert.Equal(t, gen, holder.Generation)
|
||||
_, after := f.record(t)
|
||||
assert.Equal(t, rev, after, "ADR-0003:S7: a refused start must not stamp the store record")
|
||||
assert.Empty(t, f.mounter.mounted)
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", tt.reason))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Once the holder's heartbeat is LivenessWindow old, or gone, its node is lost,
|
||||
// and the start takes the claim over, stamps the record above the old holder
|
||||
// and counts it, so the old holder's pointer moves are fenced from then on.
|
||||
func TestS7_UEFIClaimIsTakenOverOnceItsHolderStopsHeartbeating(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
age time.Duration
|
||||
noKey bool
|
||||
}{
|
||||
{name: "heartbeat 60s old", age: 60 * time.Second},
|
||||
{name: "heartbeat 10m old", age: 10 * time.Minute},
|
||||
{name: "no heartbeat", noKey: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
instance, gen := holdOnPeer(t, f)
|
||||
if !tt.noKey {
|
||||
f.liveness.heartbeat("node-b", tt.age)
|
||||
}
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.claim_takeovers", "", "")
|
||||
|
||||
held, err := f.prepare(t, instance)
|
||||
|
||||
require.NoError(t, err)
|
||||
holder, ok := f.claimHolder(t)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "node-a", holder.Holder)
|
||||
assert.Greater(t, holder.Generation, gen)
|
||||
assert.Equal(t, held.token, holder)
|
||||
rec, _ := f.record(t)
|
||||
assert.Equal(t, holder.Generation, rec.Generation, "the new owner stamps the record before it reads the pointer")
|
||||
assert.Equal(t, region("node-b booted this"), f.localVars(t))
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_takeovers", "", ""))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// failingStamp is a store record bucket whose stamp fails transiently.
|
||||
type failingStamp struct{ UEFIStoreRecords }
|
||||
|
||||
func (failingStamp) Stamp(context.Context, string, uint64) (*uefistore.UEFIStoreRecord, error) {
|
||||
return nil, errors.New("stamp: no responders")
|
||||
}
|
||||
|
||||
// TestS7_UEFIClaimGuardingALocalCopyIsKeptWhenALaunchRollsBack enforces
|
||||
// ADR-0003:S7: a restart that adopts its claim may hold changes no object
|
||||
// has, so a launch that fails before reuse is decided must keep the claim.
|
||||
func TestS7_UEFIClaimGuardingALocalCopyIsKeptWhenALaunchRollsBack(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
local func(f *uefiFixture, key string, gen uint64)
|
||||
}{
|
||||
{name: "copy seeded under the claim", local: func(f *uefiFixture, key string, gen uint64) {
|
||||
f.writeLocal(t, key, gen, region("guest wrote this"))
|
||||
}},
|
||||
{name: "unreadable seeded.json", local: func(f *uefiFixture, key string, gen uint64) {
|
||||
f.writeLocal(t, key, gen, region("guest wrote this"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), uefiSeededFileName), []byte("{"), 0o600))
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("adopted")
|
||||
instance := f.instance(t, key, f.identity)
|
||||
tok, err := f.peerClaim("node-a").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
tt.local(f, key, tok.Generation)
|
||||
f.m.deps.UEFIVarStore.Records = failingStamp{f.records}
|
||||
|
||||
err = f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrMountRetryable)
|
||||
holder, held := f.claimHolder(t)
|
||||
require.True(t, held, "ADR-0003:S7 (uefi_varstore.go owedUnder): a rolled-back launch released the claim guarding this node's copy; keep it whenever seeded.json names the claim's generation")
|
||||
assert.Equal(t, tok, holder)
|
||||
_, refused := f.peerClaim("node-b").TryLock(t.Context())
|
||||
assert.ErrorAs(t, refused, new(*kvsync.HeldError), "ADR-0003:S7: another node could take the claim while this node's copy held unwritten changes")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A copy seeded under an earlier generation was superseded by another node,
|
||||
// so a launch that rolls back before it seeds releases the claim it took.
|
||||
func TestUEFIClaim_RollbackReleasesAClaimNoLocalCopyNeeds(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("stale")
|
||||
instance := f.instance(t, key, f.identity)
|
||||
earlier, err := f.peerClaim("node-b").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.peerClaim("node-b").Unlock(t.Context(), earlier))
|
||||
f.writeLocal(t, key, earlier.Generation, region("older run"))
|
||||
f.m.deps.UEFIVarStore.Records = failingStamp{f.records}
|
||||
|
||||
require.ErrorIs(t, f.m.Run(t.Context(), instance), ErrMountRetryable)
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held)
|
||||
}
|
||||
|
||||
// TestS7_UEFICopyThatCannotBeVouchedForIsNeverOverwritten enforces
|
||||
// ADR-0003:S7: a vars.fd whose seeded.json cannot be read may hold changes no
|
||||
// object has, so the start is refused as retryable and the file and the claim
|
||||
// are kept rather than the object fetched over it.
|
||||
func TestS7_UEFICopyThatCannotBeVouchedForIsNeverOverwritten(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("pointed")
|
||||
f.putObject(t, key, gzipped(t, region("object")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
tok, err := f.peerClaim("node-a").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
f.writeLocal(t, key, tok.Generation, region("guest wrote this"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), uefiSeededFileName), []byte("{"), 0o600))
|
||||
|
||||
err = f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrMountRetryable)
|
||||
assert.Equal(t, region("guest wrote this"), f.localVars(t),
|
||||
"ADR-0003:S7 (uefi_varstore.go prepareUEFIVarStore): a copy that cannot be vouched for was overwritten; refuse the start and keep it")
|
||||
holder, held := f.claimHolder(t)
|
||||
require.True(t, held, "ADR-0003:S7: the claim guarding an unvouched copy was released")
|
||||
assert.Equal(t, tok, holder)
|
||||
assert.Empty(t, f.mounter.mounted)
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -19,7 +20,10 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"go.opentelemetry.io/otel"
|
||||
@@ -38,6 +42,9 @@ const (
|
||||
// object, so a wedged object store fails the start rather than holding it.
|
||||
uefiStoreGetTimeout = 10 * time.Second
|
||||
|
||||
// uefiClaimReleaseTimeout bounds the unlock a rolled-back launch makes.
|
||||
uefiClaimReleaseTimeout = 10 * time.Second
|
||||
|
||||
// uefiStoreBodySlack is how far a compressed body may exceed its region
|
||||
// before it is refused; gzip grows incompressible data by far less.
|
||||
uefiStoreBodySlack = 1 << 20
|
||||
@@ -47,22 +54,40 @@ const (
|
||||
uefiStateReasonCode = "Server.InternalError"
|
||||
)
|
||||
|
||||
// ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
|
||||
// store and carries no EFI volume, so it has nothing to boot its variables from.
|
||||
// ErrUEFIVarStoreMissing is a UEFI instance with no EFI volume whose store
|
||||
// record is absent, deleting or names no pointer, so it has nothing to boot
|
||||
// its variables from.
|
||||
var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
|
||||
|
||||
// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
|
||||
// store and also carries an EFI volume, so either one would be ignored.
|
||||
var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
|
||||
|
||||
// ErrUEFIFirmwareMismatch is a variable store written under firmware whose
|
||||
// architecture or VARS size differs from this node's, which pflash cannot load.
|
||||
var ErrUEFIFirmwareMismatch = errors.New("UEFI variable store firmware does not match this node")
|
||||
|
||||
// ErrUEFIVarStoreUnusable is a variable store object that is missing or does
|
||||
// not decode to a region of this node's VARS size. Retrying does not help.
|
||||
// ErrUEFIVarStoreUnusable is a store record this node cannot trust, or a store
|
||||
// object that is missing or does not decode to a region of this node's VARS
|
||||
// size. Retrying does not help.
|
||||
var ErrUEFIVarStoreUnusable = errors.New("UEFI variable store object is unusable")
|
||||
|
||||
// ErrUEFIClaimHeld is a start refused because another node holds the store's
|
||||
// claim and its heartbeat is fresh or cannot be read: that node may still land
|
||||
// newer variables. It always wraps ErrMountRetryable.
|
||||
var ErrUEFIClaimHeld = errors.New("UEFI variable store is claimed by another node that may be live")
|
||||
|
||||
// UEFIStoreRecords is the store-record surface a launch reads and stamps.
|
||||
type UEFIStoreRecords interface {
|
||||
Get(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
|
||||
Stamp(ctx context.Context, instanceID string, generation uint64) (*uefistore.UEFIStoreRecord, error)
|
||||
}
|
||||
|
||||
var _ UEFIStoreRecords = (*uefistore.Store)(nil)
|
||||
|
||||
// NodeLiveness reports when a node last heartbeated. found is false only for
|
||||
// a node with no heartbeat. A heartbeat that exists but cannot be dated is an
|
||||
// error, as is a failed read: the answer is unknown, and never reads as dead.
|
||||
type NodeLiveness interface {
|
||||
LastSeen(ctx context.Context, node string) (seen time.Time, found bool, err error)
|
||||
}
|
||||
|
||||
// UEFIVarStoreDeps is where a node reads instance variable stores from and
|
||||
// keeps the live copy QEMU opens. DataDir is the daemon's persistent data
|
||||
// directory, so the copy survives a daemon restart and a host reboot.
|
||||
@@ -70,12 +95,31 @@ type UEFIVarStoreDeps struct {
|
||||
Objects objectstore.ObjectStore
|
||||
Bucket string
|
||||
DataDir string
|
||||
// Records holds each store's pointer and the claim generation fencing it.
|
||||
Records UEFIStoreRecords
|
||||
// Claims is this node's group of store claims; Liveness decides whether
|
||||
// the holder of a claim this node finds held may be taken over.
|
||||
Claims *kvsync.ClaimGroup
|
||||
Liveness NodeLiveness
|
||||
}
|
||||
|
||||
// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
|
||||
// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from,
|
||||
// and the claim generation it was seeded or last written back under.
|
||||
type uefiSeed struct {
|
||||
Key string `json:"key"`
|
||||
SeededAt time.Time `json:"seeded_at"`
|
||||
Key string `json:"key"`
|
||||
Generation uint64 `json:"generation"`
|
||||
SeededAt time.Time `json:"seeded_at"`
|
||||
}
|
||||
|
||||
// uefiClaim is the claim a launch acquired. reused is set when the launch
|
||||
// boots this node's copy. keep is set when this node held a copy under token
|
||||
// before the launch, which may carry changes no object has, so a rollback
|
||||
// must not release the claim.
|
||||
type uefiClaim struct {
|
||||
claim *kvsync.Claim
|
||||
token kvsync.Token
|
||||
reused bool
|
||||
keep bool
|
||||
}
|
||||
|
||||
// isUEFIBootMode reports whether bootMode boots the guest through UEFI firmware.
|
||||
@@ -89,24 +133,51 @@ func usesUEFIVarStore(v *VM) bool {
|
||||
return !v.DirectBoot && isUEFIBootMode(v.BootMode)
|
||||
}
|
||||
|
||||
// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
|
||||
// two, before anything about it is changed. An EFI volume still counts as a
|
||||
// store until RunInstances seeds pointers instead. The caller owns v.
|
||||
func CheckUEFIVarStore(v *VM) error {
|
||||
if !usesUEFIVarStore(v) {
|
||||
return nil
|
||||
}
|
||||
// hasEFIVolume reports whether v keeps its variables on a <root>-efi volume.
|
||||
func hasEFIVolume(v *VM) bool {
|
||||
v.EBSRequests.Mu.Lock()
|
||||
hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
|
||||
v.EBSRequests.Mu.Unlock()
|
||||
defer v.EBSRequests.Mu.Unlock()
|
||||
return slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
|
||||
}
|
||||
|
||||
switch {
|
||||
case v.UEFIVarStore == nil && !hasEFIVolume:
|
||||
return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
|
||||
case v.UEFIVarStore != nil && hasEFIVolume:
|
||||
return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
|
||||
// usesStoreObject reports whether v boots its variables from a store object
|
||||
// rather than an EFI volume.
|
||||
func usesStoreObject(v *VM) bool {
|
||||
return usesUEFIVarStore(v) && !hasEFIVolume(v)
|
||||
}
|
||||
|
||||
// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
|
||||
// two, before anything about it is changed. It only reads.
|
||||
func (m *Manager) CheckUEFIVarStore(ctx context.Context, v *VM) error {
|
||||
_, err := m.checkUEFIVarStore(ctx, v)
|
||||
return err
|
||||
}
|
||||
|
||||
// checkUEFIVarStore is CheckUEFIVarStore returning the store record it read,
|
||||
// or nil for an instance that boots no store object. An EFI volume is the
|
||||
// instance's store until RunInstances seeds store records, so it reads none.
|
||||
func (m *Manager) checkUEFIVarStore(ctx context.Context, v *VM) (*uefistore.UEFIStoreRecord, error) {
|
||||
if !usesStoreObject(v) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil
|
||||
records := m.deps.UEFIVarStore.Records
|
||||
if records == nil {
|
||||
return nil, fmt.Errorf("%w: %s, and no store records are wired on this node", ErrUEFIVarStoreMissing, v.ID)
|
||||
}
|
||||
rec, _, err := records.Get(ctx, v.ID)
|
||||
switch {
|
||||
case errors.Is(err, kvstore.ErrNotFound):
|
||||
return nil, fmt.Errorf("%w: %s has no store record", ErrUEFIVarStoreMissing, v.ID)
|
||||
case errors.Is(err, uefistore.ErrInvalidRecord):
|
||||
return nil, fmt.Errorf("%w: %w", ErrUEFIVarStoreUnusable, err)
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("read UEFI store record for %s: %w: %w", v.ID, err, ErrMountRetryable)
|
||||
case rec.State != uefistore.StateLive:
|
||||
return nil, fmt.Errorf("%w: %s store record is %s", ErrUEFIVarStoreMissing, v.ID, rec.State)
|
||||
case rec.Pointer == nil:
|
||||
return nil, fmt.Errorf("%w: %s store record names no pointer", ErrUEFIVarStoreMissing, v.ID)
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// uefiInstanceDir is the directory holding instanceID's vars.fd and
|
||||
@@ -127,13 +198,11 @@ func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
|
||||
return filepath.Join(dir, uefiVarsFileName), nil
|
||||
}
|
||||
|
||||
// uefiVarsDrive is pflash unit 1 from this node's copy of the store the
|
||||
// record points at, or nil with no pointer. Launch seeds the copy and a
|
||||
// uefiVarsDrive is pflash unit 1 from this node's copy of the store, or nil
|
||||
// for an instance that boots no store object. Launch seeds the copy and a
|
||||
// reboot reuses it, so a missing one fails rather than booting without it.
|
||||
func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
|
||||
hasPointer := false
|
||||
m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
|
||||
if !hasPointer || !usesUEFIVarStore(instance) {
|
||||
if !usesStoreObject(instance) {
|
||||
return nil, nil
|
||||
}
|
||||
path, err := m.uefiVarsPath(instance.ID)
|
||||
@@ -146,78 +215,188 @@ func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
|
||||
return &Drive{File: path, Format: "raw", If: "pflash", Unit: 1}, nil
|
||||
}
|
||||
|
||||
// prepareUEFIVarStore leaves vars.fd holding the store the record points at,
|
||||
// reusing this node's copy when seeded.json names the pointer and fetching
|
||||
// the object otherwise. The object and the record's pointer are only read.
|
||||
func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
|
||||
var ref *UEFIVarStoreRef
|
||||
m.Inspect(instance, func(v *VM) {
|
||||
if v.UEFIVarStore != nil {
|
||||
cp := *v.UEFIVarStore
|
||||
ref = &cp
|
||||
}
|
||||
})
|
||||
if ref == nil || !usesUEFIVarStore(instance) {
|
||||
return nil
|
||||
// prepareUEFIVarStore checks this node can boot the store rec points at, then
|
||||
// takes the store's claim, stamps the store record with its generation and
|
||||
// leaves vars.fd holding the store to boot. rec is the record checkUEFIVarStore
|
||||
// read. A non-nil claim is returned once one is acquired, failed or not, so
|
||||
// the caller can release it on a rollback.
|
||||
func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM, rec *uefistore.UEFIStoreRecord) (*uefiClaim, error) {
|
||||
if !usesStoreObject(instance) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
deps := m.deps.UEFIVarStore
|
||||
if deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" {
|
||||
return errors.New("UEFI variable store access is not wired on this node")
|
||||
}
|
||||
if name, ok := strings.CutPrefix(ref.Key, uefistore.KeyPrefix(instance.ID)); !ok || name == "" || strings.Contains(name, "/") {
|
||||
return fmt.Errorf("%w: pointer %q is not under %s", ErrUEFIVarStoreUnusable, ref.Key, uefistore.KeyPrefix(instance.ID))
|
||||
if rec == nil || deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" || deps.Records == nil || deps.Claims == nil || deps.Liveness == nil {
|
||||
return nil, errors.New("UEFI variable store access is not wired on this node")
|
||||
}
|
||||
if m.deps.InstanceTypes == nil {
|
||||
return errors.New("InstanceTypeResolver not wired")
|
||||
return nil, errors.New("InstanceTypeResolver not wired")
|
||||
}
|
||||
spec, ok := m.deps.InstanceTypes.Resolve(instance.InstanceType)
|
||||
if !ok {
|
||||
return fmt.Errorf("instance type %s not found", instance.InstanceType)
|
||||
return nil, fmt.Errorf("instance type %s not found", instance.InstanceType)
|
||||
}
|
||||
fw, err := NodeFirmwareFor(spec.Architecture)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
|
||||
// Before the claim, so a node that cannot boot the store never takes it
|
||||
// from a holder that is coming back.
|
||||
if err := m.checkUEFIFirmware(instance, rec.Pointer, fw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
held, err := m.acquireUEFIClaim(ctx, instance.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Before anything that can fail, so a rollback keeps a claim this node's
|
||||
// copy may owe a write-back under.
|
||||
local := readLocalUEFICopy(dir)
|
||||
held.keep = local.owedUnder(held.token.Generation)
|
||||
checked := rec.Pointer.Firmware
|
||||
rec, err = deps.Records.Stamp(ctx, instance.ID, held.token.Generation)
|
||||
switch {
|
||||
case errors.Is(err, uefistore.ErrFenced):
|
||||
recordUEFIClaimRefusal("fenced")
|
||||
slog.WarnContext(ctx, "UEFI store record is stamped by a later claim than the one this node holds; refusing the start",
|
||||
"instanceId", instance.ID, "generation", held.token.Generation, "err", err)
|
||||
return held, fmt.Errorf("%w: %w: %w", ErrUEFIClaimHeld, err, ErrMountRetryable)
|
||||
case errors.Is(err, kvstore.ErrNotFound), errors.Is(err, uefistore.ErrDeleting):
|
||||
return held, fmt.Errorf("%w: %s: %w", ErrUEFIVarStoreMissing, instance.ID, err)
|
||||
case errors.Is(err, uefistore.ErrInvalidRecord):
|
||||
return held, fmt.Errorf("%w: %w", ErrUEFIVarStoreUnusable, err)
|
||||
case err != nil:
|
||||
return held, fmt.Errorf("stamp UEFI store record for %s: %w: %w", instance.ID, err, ErrMountRetryable)
|
||||
case rec.Pointer == nil:
|
||||
return held, fmt.Errorf("%w: %s store record names no pointer", ErrUEFIVarStoreMissing, instance.ID)
|
||||
}
|
||||
// The record validated its pointer as one object under this instance's prefix.
|
||||
ref := rec.Pointer
|
||||
|
||||
if ref.Firmware != checked {
|
||||
if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
|
||||
return held, err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case local.hasVars && local.seedErr != nil:
|
||||
slog.ErrorContext(ctx, "This node's UEFI variable store cannot be vouched for; keeping it and refusing the start",
|
||||
"instanceId", instance.ID, "path", filepath.Join(dir, uefiSeededFileName), "err", local.seedErr)
|
||||
return held, fmt.Errorf("UEFI variable store for %s has a %s this node cannot read: %w: %w",
|
||||
instance.ID, uefiSeededFileName, local.seedErr, ErrMountRetryable)
|
||||
case held.keep && local.varsSize != fw.Identity.VarsSize:
|
||||
slog.ErrorContext(ctx, "This node's UEFI variable store is not the VARS size; keeping it and refusing the start",
|
||||
"instanceId", instance.ID, "path", filepath.Join(dir, uefiVarsFileName),
|
||||
"size", local.varsSize, "want", fw.Identity.VarsSize)
|
||||
return held, fmt.Errorf("%w: %s for %s is %d bytes, not %d",
|
||||
ErrUEFIVarStoreUnusable, uefiVarsFileName, instance.ID, local.varsSize, fw.Identity.VarsSize)
|
||||
case held.keep:
|
||||
held.reused = true
|
||||
recordUEFIStoreSeed("local")
|
||||
slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
|
||||
return nil
|
||||
slog.InfoContext(ctx, "Reusing this node's UEFI variable store",
|
||||
"instanceId", instance.ID, "key", ref.Key, "seededKey", local.seed.Key, "generation", held.token.Generation)
|
||||
return held, nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
region, err := fetchUEFIVars(ctx, deps, ref.Key, fw.Identity.VarsSize)
|
||||
if err != nil {
|
||||
return err
|
||||
return held, err
|
||||
}
|
||||
if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
|
||||
return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
|
||||
return held, fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
|
||||
}
|
||||
seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
|
||||
seed, err := json.Marshal(uefiSeed{Key: ref.Key, Generation: held.token.Generation, SeededAt: time.Now().UTC()})
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
|
||||
return held, fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
|
||||
}
|
||||
if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
|
||||
return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
|
||||
return held, fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
|
||||
}
|
||||
recordUEFIStoreSeed("object")
|
||||
slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
|
||||
"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
|
||||
return nil
|
||||
"instanceId", instance.ID, "key", ref.Key, "generation", held.token.Generation,
|
||||
"bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// acquireUEFIClaim takes instanceID's store claim, or adopts it when this node
|
||||
// already holds it. A claim another node holds is taken over only once that
|
||||
// node's heartbeat is LivenessWindow old or absent; while it is fresh, or
|
||||
// cannot be read, the start is refused as retryable (ADR-0003:S7).
|
||||
func (m *Manager) acquireUEFIClaim(ctx context.Context, instanceID string) (*uefiClaim, error) {
|
||||
deps := m.deps.UEFIVarStore
|
||||
claim := deps.Claims.Get(instanceID)
|
||||
tok, err := claim.TryLock(ctx)
|
||||
if err == nil {
|
||||
return &uefiClaim{claim: claim, token: tok}, nil
|
||||
}
|
||||
he, ok := errors.AsType[*kvsync.HeldError](err)
|
||||
if !ok || he.Current.Holder == "" {
|
||||
return nil, fmt.Errorf("take UEFI store claim for %s: %w: %w", instanceID, err, ErrMountRetryable)
|
||||
}
|
||||
holder := he.Current
|
||||
|
||||
seen, found, err := deps.Liveness.LastSeen(ctx, holder.Holder)
|
||||
if err != nil {
|
||||
recordUEFIClaimRefusal("liveness_unknown")
|
||||
slog.WarnContext(ctx, "Refusing to start: another node holds the UEFI variable store claim and its heartbeat cannot be read",
|
||||
"instanceId", instanceID, "holder", holder.Holder, "generation", holder.Generation, "err", err)
|
||||
return nil, fmt.Errorf("%w: %s holds %s and its liveness could not be read: %w: %w",
|
||||
ErrUEFIClaimHeld, holder.Holder, instanceID, err, ErrMountRetryable)
|
||||
}
|
||||
now := time.Now()
|
||||
if found && clusterv1.LiveAt(seen, now) {
|
||||
recordUEFIClaimRefusal("holder_live")
|
||||
slog.WarnContext(ctx, "Refusing to start: another live node holds the UEFI variable store claim",
|
||||
"instanceId", instanceID, "holder", holder.Holder, "generation", holder.Generation,
|
||||
"heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
|
||||
return nil, fmt.Errorf("%w: %s holds %s and is still heartbeating: %w",
|
||||
ErrUEFIClaimHeld, holder.Holder, instanceID, ErrMountRetryable)
|
||||
}
|
||||
|
||||
tok, err = claim.TakeOver(ctx, holder)
|
||||
if err != nil {
|
||||
if _, moved := errors.AsType[*kvsync.HeldError](err); moved {
|
||||
recordUEFIClaimRefusal("moved")
|
||||
return nil, fmt.Errorf("%w: the claim on %s moved on from %s: %w: %w",
|
||||
ErrUEFIClaimHeld, instanceID, holder.Holder, err, ErrMountRetryable)
|
||||
}
|
||||
return nil, fmt.Errorf("take over UEFI store claim for %s: %w: %w", instanceID, err, ErrMountRetryable)
|
||||
}
|
||||
recordUEFIClaimTakeover()
|
||||
attrs := []any{"instanceId", instanceID, "holder", holder.Holder, "fromGeneration", holder.Generation, "generation", tok.Generation}
|
||||
if found {
|
||||
attrs = append(attrs, "heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
|
||||
}
|
||||
slog.WarnContext(ctx, "Took over the UEFI variable store claim from a node whose heartbeat has stopped; variable changes it never wrote back are lost", attrs...)
|
||||
return &uefiClaim{claim: claim, token: tok}, nil
|
||||
}
|
||||
|
||||
// releaseUEFIClaim unlocks the claim a launch that rolled back before the
|
||||
// guest ran acquired. A launch holding this node's copy keeps it: the copy
|
||||
// may hold changes no object has, and another node would start without them.
|
||||
func (m *Manager) releaseUEFIClaim(ctx context.Context, instanceID string, held *uefiClaim) {
|
||||
if held == nil || held.keep {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), uefiClaimReleaseTimeout)
|
||||
defer cancel()
|
||||
if err := held.claim.Unlock(ctx, held.token); err != nil {
|
||||
slog.WarnContext(ctx, "Failed to release the UEFI variable store claim after a launch rolled back",
|
||||
"instanceId", instanceID, "generation", held.token.Generation, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// checkUEFIFirmware fails a store whose arch or VARS size differs from this
|
||||
// node's, naming both firmwares in the log and the StateReason. A different
|
||||
// CODE build boots, because a package update changes it on every node.
|
||||
func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeFirmware) error {
|
||||
func (m *Manager) checkUEFIFirmware(instance *VM, ref *uefistore.UEFIVarStoreRef, fw NodeFirmware) error {
|
||||
stored, node := ref.Firmware, fw.Identity
|
||||
var kind string
|
||||
switch {
|
||||
@@ -251,23 +430,45 @@ func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeF
|
||||
return fmt.Errorf("%w: %s", ErrUEFIFirmwareMismatch, msg)
|
||||
}
|
||||
|
||||
// reuseUEFIVars reports whether dir already holds the store key names: a
|
||||
// seeded.json naming key beside a vars.fd of the region's size.
|
||||
func reuseUEFIVars(dir, key string, varsSize int64) bool {
|
||||
// localUEFICopy is what this node holds for an instance: whether vars.fd
|
||||
// exists and its size, and seeded.json or the error reading it.
|
||||
type localUEFICopy struct {
|
||||
hasVars bool
|
||||
varsSize int64
|
||||
hasSeed bool
|
||||
seed uefiSeed
|
||||
seedErr error
|
||||
}
|
||||
|
||||
// readLocalUEFICopy reads dir's vars.fd and seeded.json. Only a seeded.json
|
||||
// that exists and cannot be read or decoded sets seedErr.
|
||||
func readLocalUEFICopy(dir string) localUEFICopy {
|
||||
var c localUEFICopy
|
||||
if info, err := os.Stat(filepath.Join(dir, uefiVarsFileName)); err == nil {
|
||||
c.hasVars, c.varsSize = true, info.Size()
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return c
|
||||
case err != nil:
|
||||
c.seedErr = err
|
||||
default:
|
||||
c.seedErr = json.Unmarshal(data, &c.seed)
|
||||
}
|
||||
c.hasSeed = c.seedErr == nil
|
||||
return c
|
||||
}
|
||||
|
||||
// owedUnder reports whether the copy may hold changes no object has under the
|
||||
// claim generation: vars.fd exists and seeded.json names generation, or cannot
|
||||
// be read. At one generation only this node moves the pointer, and only from
|
||||
// vars.fd, so such a file is never older than the pointer whatever key it names.
|
||||
func (c localUEFICopy) owedUnder(generation uint64) bool {
|
||||
if !c.hasVars {
|
||||
return false
|
||||
}
|
||||
var seed uefiSeed
|
||||
if err := json.Unmarshal(data, &seed); err != nil || seed.Key != key {
|
||||
return false
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(dir, uefiVarsFileName))
|
||||
if err != nil || info.Size() != varsSize {
|
||||
slog.Warn("UEFI variable store seeded here has no usable vars.fd; fetching it again", "dir", dir, "key", key)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return c.seedErr != nil || (c.hasSeed && c.seed.Generation == generation)
|
||||
}
|
||||
|
||||
// fetchUEFIVars gets key and returns its decompressed region, which must be
|
||||
@@ -354,6 +555,8 @@ var (
|
||||
uefiStoreMetricsOnce sync.Once
|
||||
uefiStoreSeedCounter metric.Int64Counter
|
||||
uefiFirmwareMismatches metric.Int64Counter
|
||||
uefiClaimRefusals metric.Int64Counter
|
||||
uefiClaimTakeovers metric.Int64Counter
|
||||
)
|
||||
|
||||
func uefiStoreMetrics() {
|
||||
@@ -372,6 +575,18 @@ func uefiStoreMetrics() {
|
||||
if err != nil {
|
||||
otel.Handle(err)
|
||||
}
|
||||
uefiClaimRefusals, err = meter.Int64Counter("mulga.vm.uefi_store.claim_refusals",
|
||||
metric.WithDescription("UEFI starts refused because another node holds the variable store claim and may be live, by reason."),
|
||||
metric.WithUnit("{start}"))
|
||||
if err != nil {
|
||||
otel.Handle(err)
|
||||
}
|
||||
uefiClaimTakeovers, err = meter.Int64Counter("mulga.vm.uefi_store.claim_takeovers",
|
||||
metric.WithDescription("UEFI variable store claims taken over from a node whose heartbeat had stopped."),
|
||||
metric.WithUnit("{start}"))
|
||||
if err != nil {
|
||||
otel.Handle(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -388,3 +603,17 @@ func recordUEFIFirmwareMismatch(kind string) {
|
||||
uefiFirmwareMismatches.Add(context.Background(), 1, metric.WithAttributes(attribute.String("kind", kind)))
|
||||
}
|
||||
}
|
||||
|
||||
func recordUEFIClaimRefusal(reason string) {
|
||||
uefiStoreMetrics()
|
||||
if uefiClaimRefusals != nil {
|
||||
uefiClaimRefusals.Add(context.Background(), 1, metric.WithAttributes(attribute.String("reason", reason)))
|
||||
}
|
||||
}
|
||||
|
||||
func recordUEFIClaimTakeover() {
|
||||
uefiStoreMetrics()
|
||||
if uefiClaimTakeovers != nil {
|
||||
uefiClaimTakeovers.Add(context.Background(), 1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,12 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel"
|
||||
@@ -41,7 +45,7 @@ var (
|
||||
)
|
||||
|
||||
// uefiCounter returns the sum of the named counter's points whose attribute
|
||||
// key has value.
|
||||
// key has value, or of every point when key is empty.
|
||||
func uefiCounter(t *testing.T, name, key, value string) int64 {
|
||||
t.Helper()
|
||||
uefiMetricsReaderOnce.Do(func() {
|
||||
@@ -58,7 +62,7 @@ func uefiCounter(t *testing.T, name, key, value string) int64 {
|
||||
continue
|
||||
}
|
||||
for _, dp := range sum.DataPoints {
|
||||
if v, ok := dp.Attributes.Value(attribute.Key(key)); ok && v.AsString() == value {
|
||||
if v, ok := dp.Attributes.Value(attribute.Key(key)); key == "" || ok && v.AsString() == value {
|
||||
total += dp.Value
|
||||
}
|
||||
}
|
||||
@@ -67,17 +71,45 @@ func uefiCounter(t *testing.T, name, key, value string) int64 {
|
||||
return total
|
||||
}
|
||||
|
||||
// uefiFixture is a node with one x86_64 firmware candidate, an in-memory
|
||||
// object store and a data directory.
|
||||
// uefiFixture is node-a with one x86_64 firmware candidate, an in-memory
|
||||
// object store, a data directory, and the store records and claims of a real
|
||||
// JetStream, with every other node's heartbeat set by the test.
|
||||
type uefiFixture struct {
|
||||
m *Manager
|
||||
mounter *fakeVolumeMounter
|
||||
objects *objectstore.MemoryObjectStore
|
||||
records *uefistore.Store
|
||||
js jetstream.JetStream
|
||||
liveness *fakeLiveness
|
||||
dataDir string
|
||||
codePath string
|
||||
identity uefistore.FirmwareIdentity
|
||||
}
|
||||
|
||||
// fakeLiveness answers LastSeen for each node from what the test set; a node
|
||||
// it has no answer for has never heartbeated.
|
||||
type fakeLiveness struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]time.Time
|
||||
err error
|
||||
}
|
||||
|
||||
func (l *fakeLiveness) LastSeen(_ context.Context, node string) (time.Time, bool, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.err != nil {
|
||||
return time.Time{}, false, l.err
|
||||
}
|
||||
seen, ok := l.seen[node]
|
||||
return seen, ok, nil
|
||||
}
|
||||
|
||||
func (l *fakeLiveness) heartbeat(node string, age time.Duration) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.seen[node] = time.Now().Add(-age)
|
||||
}
|
||||
|
||||
func newUEFIFixture(t *testing.T) *uefiFixture {
|
||||
t.Helper()
|
||||
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
|
||||
@@ -91,10 +123,17 @@ func newUEFIFixture(t *testing.T) *uefiFixture {
|
||||
FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
|
||||
t.Cleanup(func() { FirmwarePathCandidates = orig })
|
||||
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
records := uefistore.NewStore(js)
|
||||
require.NoError(t, records.Open(t.Context()))
|
||||
|
||||
sum := sha256.Sum256(codeBytes)
|
||||
f := &uefiFixture{
|
||||
mounter: &fakeVolumeMounter{},
|
||||
objects: objectstore.NewMemoryObjectStore(),
|
||||
records: records,
|
||||
js: js,
|
||||
liveness: &fakeLiveness{seen: map[string]time.Time{}},
|
||||
dataDir: t.TempDir(),
|
||||
codePath: code,
|
||||
identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
|
||||
@@ -102,11 +141,34 @@ func newUEFIFixture(t *testing.T) *uefiFixture {
|
||||
f.m = NewManagerWithDeps(Deps{
|
||||
VolumeMounter: f.mounter,
|
||||
InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
|
||||
UEFIVarStore: UEFIVarStoreDeps{Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir},
|
||||
UEFIVarStore: UEFIVarStoreDeps{
|
||||
Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir,
|
||||
Records: records, Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"), Liveness: f.liveness,
|
||||
},
|
||||
})
|
||||
return f
|
||||
}
|
||||
|
||||
// peerClaim is the instance's claim as another node sees it.
|
||||
func (f *uefiFixture) peerClaim(node string) *kvsync.Claim {
|
||||
return uefistore.NewClaims(kvstore.OpenSyncBucket(f.js), node).Get(testUEFIInstance)
|
||||
}
|
||||
|
||||
// claimHolder is who holds the instance's claim, read as a third node would.
|
||||
func (f *uefiFixture) claimHolder(t *testing.T) (kvsync.Token, bool) {
|
||||
t.Helper()
|
||||
tok, ok, err := f.peerClaim("observer").Holder(t.Context())
|
||||
require.NoError(t, err)
|
||||
return tok, ok
|
||||
}
|
||||
|
||||
func (f *uefiFixture) record(t *testing.T) (*uefistore.UEFIStoreRecord, uint64) {
|
||||
t.Helper()
|
||||
rec, rev, err := f.records.Get(t.Context(), testUEFIInstance)
|
||||
require.NoError(t, err)
|
||||
return rec, rev
|
||||
}
|
||||
|
||||
// region is a VARS region of the fixture's size whose bytes say which store it is.
|
||||
func region(tag string) []byte {
|
||||
b := make([]byte, testUEFIVarsSize)
|
||||
@@ -146,11 +208,11 @@ func (f *uefiFixture) instanceDir() string {
|
||||
return filepath.Join(f.dataDir, "uefi", testUEFIInstance)
|
||||
}
|
||||
|
||||
func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, vars []byte) {
|
||||
func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, generation uint64, vars []byte) {
|
||||
t.Helper()
|
||||
require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "vars.fd"), vars, 0o600))
|
||||
seed, err := json.Marshal(uefiSeed{Key: seededKey, SeededAt: time.Now()})
|
||||
seed, err := json.Marshal(uefiSeed{Key: seededKey, Generation: generation, SeededAt: time.Now()})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), seed, 0o600))
|
||||
}
|
||||
@@ -162,78 +224,122 @@ func (f *uefiFixture) localVars(t *testing.T) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
func (f *uefiFixture) seededKey(t *testing.T) string {
|
||||
func (f *uefiFixture) seed(t *testing.T) uefiSeed {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join(f.instanceDir(), "seeded.json"))
|
||||
require.NoError(t, err)
|
||||
var seed uefiSeed
|
||||
require.NoError(t, json.Unmarshal(b, &seed))
|
||||
return seed.Key
|
||||
return seed
|
||||
}
|
||||
|
||||
func (f *uefiFixture) instance(key string, fw uefistore.FirmwareIdentity) *VM {
|
||||
// instance is a stopped UEFI instance with no EFI volume. A non-empty key
|
||||
// gives it a live store record pointing there, at generation 0 as a
|
||||
// conversion leaves it.
|
||||
func (f *uefiFixture) instance(t *testing.T, key string, fw uefistore.FirmwareIdentity) *VM {
|
||||
t.Helper()
|
||||
v := &VM{
|
||||
ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Status: StateStopped,
|
||||
Instance: &ec2.Instance{},
|
||||
}
|
||||
if key != "" {
|
||||
v.UEFIVarStore = &UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-a", WrittenAt: time.Unix(1700000000, 0).UTC()}
|
||||
_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
|
||||
InstanceID: testUEFIInstance, AccountID: "111122223333", State: uefistore.StateLive,
|
||||
Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-c", WrittenAt: time.Unix(1700000000, 0).UTC()},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
f.m.Insert(v)
|
||||
return v
|
||||
}
|
||||
|
||||
// prepare runs the launch's store check and store step and returns the claim
|
||||
// the step acquired.
|
||||
func (f *uefiFixture) prepare(t *testing.T, v *VM) (*uefiClaim, error) {
|
||||
t.Helper()
|
||||
rec, err := f.m.checkUEFIVarStore(t.Context(), v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.m.prepareUEFIVarStore(t.Context(), v, rec)
|
||||
}
|
||||
|
||||
func keyFor(name string) string { return uefistore.KeyPrefix(testUEFIInstance) + name }
|
||||
|
||||
func TestPrepareUEFIVarStore_FetchesTheObjectAndRecordsTheSeed(t *testing.T) {
|
||||
func TestPrepareUEFIVarStore_ClaimsStampsAndFetchesTheObject(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("11111111-1111-1111-1111-111111111111")
|
||||
f.putObject(t, key, gzipped(t, region("converted")))
|
||||
instance := f.instance(key, f.identity)
|
||||
instance := f.instance(t, key, f.identity)
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object")
|
||||
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
|
||||
held, err := f.prepare(t, instance)
|
||||
require.NoError(t, err)
|
||||
|
||||
holder, ok := f.claimHolder(t)
|
||||
require.True(t, ok, "the claim is held from before the guest boots")
|
||||
assert.Equal(t, "node-a", holder.Holder)
|
||||
assert.Equal(t, held.token, holder)
|
||||
rec, _ := f.record(t)
|
||||
assert.Equal(t, holder.Generation, rec.Generation, "the record is stamped with the claim's generation")
|
||||
assert.Equal(t, key, rec.Pointer.Key, "a launch only reads the pointer")
|
||||
assert.Equal(t, region("converted"), f.localVars(t), "vars.fd must hold the decompressed region")
|
||||
assert.Equal(t, key, f.seededKey(t))
|
||||
assert.Equal(t, uefiSeed{Key: key, Generation: holder.Generation, SeededAt: f.seed(t).SeededAt}, f.seed(t))
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object"))
|
||||
assert.Equal(t, key, instance.UEFIVarStore.Key, "seeding only reads the pointer")
|
||||
}
|
||||
|
||||
// A start on the node that seeded the pointer boots that node's copy, which
|
||||
// can be newer than the object, and makes no get at all.
|
||||
func TestPrepareUEFIVarStore_ReusesTheLocalCopyWhenSeededNamesThePointer(t *testing.T) {
|
||||
// A daemon restart or host reboot adopts the claim at its generation, and the
|
||||
// file it left may hold changes no object has yet, so it is booted without a
|
||||
// get and the record keeps the claim's generation.
|
||||
func TestPrepareUEFIVarStore_AdoptedClaimReusesTheLocalCopy(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("22222222-2222-2222-2222-222222222222")
|
||||
f.writeLocal(t, key, region("guest wrote this"))
|
||||
instance := f.instance(key, f.identity)
|
||||
instance := f.instance(t, key, f.identity)
|
||||
held, err := f.peerClaim("node-a").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
_, err = f.records.Stamp(t.Context(), testUEFIInstance, held.Generation)
|
||||
require.NoError(t, err)
|
||||
f.writeLocal(t, key, held.Generation, region("guest wrote this"))
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
|
||||
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance), "no object exists, so a get would fail")
|
||||
got, err := f.prepare(t, instance)
|
||||
|
||||
require.NoError(t, err, "no object exists, so a get would fail")
|
||||
assert.True(t, got.reused)
|
||||
assert.Equal(t, held, got.token, "a restart adopts the claim at the same generation")
|
||||
rec, _ := f.record(t)
|
||||
assert.Equal(t, held.Generation, rec.Generation, "adoption keeps the claim's generation")
|
||||
assert.Equal(t, region("guest wrote this"), f.localVars(t))
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
|
||||
}
|
||||
|
||||
// Once the pointer has moved past what this node seeded, the local copy is
|
||||
// stale and the pointed object replaces it.
|
||||
func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing.T) {
|
||||
// Once the pointer or the generation has moved past what this node seeded,
|
||||
// the local copy is stale and the pointed object replaces it. A copy from an
|
||||
// earlier generation predates another node's claim or a restore.
|
||||
func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotTheCurrentStore(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
local func(f *uefiFixture, t *testing.T)
|
||||
}{
|
||||
{name: "seeded from an older pointer", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("older"), region("stale"))
|
||||
f.writeLocal(t, keyFor("older"), 0, region("stale"))
|
||||
}},
|
||||
{name: "seeded under an earlier generation", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("current"), 1, region("stale"))
|
||||
}},
|
||||
{name: "seeded before generations were recorded", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("current"), 0, region("stale"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte(`{"key":"`+keyFor("current")+`"}`), 0o600))
|
||||
}},
|
||||
{name: "vars.fd of the wrong size", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("current"), []byte("short"))
|
||||
f.writeLocal(t, keyFor("current"), 0, []byte("short"))
|
||||
}},
|
||||
{name: "seeded.json unreadable", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("current"), region("stale"))
|
||||
{name: "seeded.json unreadable with no vars.fd", local: func(f *uefiFixture, t *testing.T) {
|
||||
require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte("{"), 0o600))
|
||||
}},
|
||||
{name: "no vars.fd", local: func(f *uefiFixture, t *testing.T) {
|
||||
f.writeLocal(t, keyFor("current"), region("stale"))
|
||||
f.writeLocal(t, keyFor("current"), 0, region("stale"))
|
||||
require.NoError(t, os.Remove(filepath.Join(f.instanceDir(), "vars.fd")))
|
||||
}},
|
||||
}
|
||||
@@ -243,19 +349,22 @@ func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing
|
||||
key := keyFor("current")
|
||||
tt.local(f, t)
|
||||
f.putObject(t, key, gzipped(t, region("restored")))
|
||||
instance := f.instance(key, f.identity)
|
||||
instance := f.instance(t, key, f.identity)
|
||||
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
|
||||
held, err := f.prepare(t, instance)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.False(t, held.reused)
|
||||
assert.Equal(t, region("restored"), f.localVars(t))
|
||||
assert.Equal(t, key, f.seededKey(t))
|
||||
assert.Equal(t, key, f.seed(t).Key)
|
||||
assert.Equal(t, held.token.Generation, f.seed(t).Generation)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A store pflash cannot load fails the launch before any volume is mounted,
|
||||
// names both firmwares, and leaves the object, the pointer and the local
|
||||
// copy as they were.
|
||||
// names both firmwares, gives the claim back and leaves the object, the
|
||||
// pointer and the local copy as they were.
|
||||
func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -274,11 +383,11 @@ func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
|
||||
key := keyFor("mismatch")
|
||||
body := gzipped(t, region("other firmware"))
|
||||
f.putObject(t, key, body)
|
||||
f.writeLocal(t, keyFor("older"), region("local"))
|
||||
f.writeLocal(t, keyFor("older"), 0, region("local"))
|
||||
stored := f.identity
|
||||
tt.mutate(&stored)
|
||||
instance := f.instance(key, stored)
|
||||
wantRef := *instance.UEFIVarStore
|
||||
instance := f.instance(t, key, stored)
|
||||
wantRef, _ := f.record(t)
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind)
|
||||
|
||||
err := f.m.Run(t.Context(), instance)
|
||||
@@ -291,10 +400,13 @@ func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
|
||||
for _, s := range append(tt.want, f.identity.CodeSHA256, f.codePath, key) {
|
||||
assert.Contains(t, aws.StringValue(instance.Instance.StateReason.Message), s)
|
||||
}
|
||||
assert.Equal(t, wantRef, *instance.UEFIVarStore, "the pointer is unchanged")
|
||||
rec, _ := f.record(t)
|
||||
assert.Equal(t, wantRef.Pointer, rec.Pointer, "the pointer is unchanged")
|
||||
assert.Equal(t, body, f.objectBody(t, key), "the object is unchanged")
|
||||
assert.Equal(t, region("local"), f.localVars(t), "the local copy is unchanged")
|
||||
assert.Equal(t, keyFor("older"), f.seededKey(t))
|
||||
assert.Equal(t, keyFor("older"), f.seed(t).Key)
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held, "a launch that never started QEMU gives the claim back")
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind))
|
||||
})
|
||||
}
|
||||
@@ -308,10 +420,11 @@ func TestPrepareUEFIVarStore_DifferentCodeBuildBootsAndIsCounted(t *testing.T) {
|
||||
f.putObject(t, key, gzipped(t, region("older ovmf")))
|
||||
stored := f.identity
|
||||
stored.CodeSHA256 = strings.Repeat("0", 64)
|
||||
instance := f.instance(key, stored)
|
||||
instance := f.instance(t, key, stored)
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code")
|
||||
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
|
||||
_, err := f.prepare(t, instance)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, region("older ovmf"), f.localVars(t))
|
||||
assert.Nil(t, instance.Instance.StateReason)
|
||||
@@ -333,9 +446,9 @@ func (s *failingGetStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.Ge
|
||||
func TestPrepareUEFIVarStore_FailedGetIsRetryable(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("connection refused")}
|
||||
instance := f.instance(keyFor("down"), f.identity)
|
||||
instance := f.instance(t, keyFor("down"), f.identity)
|
||||
|
||||
err := f.m.prepareUEFIVarStore(t.Context(), instance)
|
||||
_, err := f.prepare(t, instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrMountRetryable, "restore backs off on a get as it does on a mount")
|
||||
assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
|
||||
@@ -361,9 +474,9 @@ func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
|
||||
if tt.body != nil {
|
||||
f.putObject(t, key, tt.body)
|
||||
}
|
||||
instance := f.instance(key, f.identity)
|
||||
instance := f.instance(t, key, f.identity)
|
||||
|
||||
err := f.m.prepareUEFIVarStore(t.Context(), instance)
|
||||
_, err := f.prepare(t, instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
|
||||
require.NotErrorIs(t, err, ErrMountRetryable)
|
||||
@@ -373,56 +486,77 @@ func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareUEFIVarStore_PointerOutsideTheInstancePrefixIsRefused(t *testing.T) {
|
||||
for _, key := range []string{"uefi/i-0aaaaaaaaaaaaaaaa/other", keyFor("a/b"), uefistore.KeyPrefix(testUEFIInstance)} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
f.putObject(t, key, gzipped(t, region("x")))
|
||||
instance := f.instance(key, f.identity)
|
||||
// A record this node cannot trust, such as one pointing outside the
|
||||
// instance's prefix, is refused for good before any claim is taken.
|
||||
func TestRun_UntrustedStoreRecordIsRefused(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
instance := f.instance(t, "", f.identity)
|
||||
kv, err := f.js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), testUEFIInstance, []byte(`{"schema_version":1,"instance_id":"`+testUEFIInstance+
|
||||
`","state":"live","pointer":{"key":"uefi/i-0aaaaaaaaaaaaaaaa/other","firmware":{"arch":"x86_64","vars_size":4096},"writer_node":"n"}}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.ErrorIs(t, f.m.prepareUEFIVarStore(t.Context(), instance), ErrUEFIVarStoreUnusable)
|
||||
})
|
||||
}
|
||||
err = f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
|
||||
require.NotErrorIs(t, err, ErrMountRetryable)
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held)
|
||||
}
|
||||
|
||||
// A BIOS instance, a microvm and a legacy instance with no pointer touch
|
||||
// neither the object store nor the data directory.
|
||||
func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
|
||||
// A BIOS instance, a microvm and an instance whose variables are on an EFI
|
||||
// volume touch neither the claim, the object store nor the data directory.
|
||||
func TestPrepareUEFIVarStore_NoStoreObjectIsANoOp(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("must not be called")}
|
||||
bios := f.instance(keyFor("bios"), f.identity)
|
||||
bios := f.instance(t, keyFor("bios"), f.identity)
|
||||
bios.BootMode = ""
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), bios))
|
||||
held, err := f.prepare(t, bios)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, held)
|
||||
|
||||
legacy := f.instance("", f.identity)
|
||||
require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), legacy))
|
||||
microvm := &VM{ID: testUEFIInstance, BootMode: "uefi", DirectBoot: true}
|
||||
held, err = f.prepare(t, microvm)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, held)
|
||||
|
||||
efi := &VM{ID: testUEFIInstance, BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}
|
||||
held, err = f.prepare(t, efi)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, held)
|
||||
|
||||
_, claimed := f.claimHolder(t)
|
||||
assert.False(t, claimed)
|
||||
assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
|
||||
}
|
||||
|
||||
// A UEFI instance with no pointer, or two stores, is refused before the
|
||||
// launch clears or changes anything on its record.
|
||||
func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
|
||||
efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
|
||||
// A UEFI instance with no usable store record is refused before the launch
|
||||
// clears or changes anything on its record, and before any claim is taken.
|
||||
func TestRun_UEFIInstanceWithoutAUsableStoreIsRefused(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pointer bool
|
||||
efi bool
|
||||
want error
|
||||
name string
|
||||
record func(f *uefiFixture, t *testing.T)
|
||||
want error
|
||||
}{
|
||||
{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
|
||||
{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
|
||||
{name: "no record and no EFI volume", want: ErrUEFIVarStoreMissing},
|
||||
{name: "a record with no pointer", want: ErrUEFIVarStoreMissing, record: func(f *uefiFixture, t *testing.T) {
|
||||
_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: testUEFIInstance, AccountID: "a", State: uefistore.StateLive})
|
||||
require.NoError(t, err)
|
||||
}},
|
||||
{name: "a deleting record", want: ErrUEFIVarStoreMissing, record: func(f *uefiFixture, t *testing.T) {
|
||||
f.instance(t, keyFor("p"), f.identity)
|
||||
_, err := f.records.MarkDeleting(t.Context(), testUEFIInstance)
|
||||
require.NoError(t, err)
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := ""
|
||||
if tt.pointer {
|
||||
key = keyFor("p")
|
||||
}
|
||||
instance := f.instance(key, f.identity)
|
||||
if tt.efi {
|
||||
instance.EBSRequests.Requests = []EBSRequest{efiVolume}
|
||||
if tt.record != nil {
|
||||
tt.record(f, t)
|
||||
}
|
||||
instance := f.instance(t, "", f.identity)
|
||||
stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
|
||||
instance.Instance.StateReason = stale
|
||||
before := instance.Record()
|
||||
@@ -432,17 +566,19 @@ func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
assert.Empty(t, f.mounter.mounted)
|
||||
assert.Equal(t, before, instance.Record(), "the record is unchanged")
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held, "a refused start takes no claim")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Start from StateError transitions to pending before launching; a refused
|
||||
// UEFI instance must not take that transition either.
|
||||
func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testing.T) {
|
||||
func TestStart_UEFIInstanceWithoutAStoreIsRefusedBeforeTheTransition(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
transitions := 0
|
||||
f.m.deps.TransitionState = func(v *VM, s InstanceState) error { transitions++; v.Status = s; return nil }
|
||||
instance := f.instance("", f.identity)
|
||||
instance := f.instance(t, "", f.identity)
|
||||
instance.Status = StateError
|
||||
|
||||
err := f.m.Start(t.Context(), instance.ID)
|
||||
@@ -452,36 +588,56 @@ func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testin
|
||||
assert.Equal(t, StateError, instance.Status)
|
||||
}
|
||||
|
||||
// Until RunInstances seeds pointers, an instance whose variables are on an
|
||||
// EFI volume still launches through the volume, and a BIOS or microvm
|
||||
// Until RunInstances seeds store records, an instance whose variables are on
|
||||
// an EFI volume still launches through the volume, and a BIOS or microvm
|
||||
// guest needs no store at all.
|
||||
func TestCheckUEFIVarStore_Accepts(t *testing.T) {
|
||||
efi := func() *VM {
|
||||
return &VM{ID: testUEFIInstance, BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
v *VM
|
||||
name string
|
||||
v func(f *uefiFixture, t *testing.T) *VM
|
||||
unwir bool
|
||||
unreadable bool
|
||||
}{
|
||||
{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
|
||||
{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
|
||||
{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
|
||||
{name: "no boot mode", v: &VM{}},
|
||||
{name: "microvm", v: &VM{BootMode: "uefi", DirectBoot: true}},
|
||||
{name: "EFI volume", v: func(*uefiFixture, *testing.T) *VM { return efi() }},
|
||||
{name: "EFI volume on a node with no store records", unwir: true, v: func(*uefiFixture, *testing.T) *VM { return efi() }},
|
||||
{name: "EFI volume while the store bucket cannot be read", unreadable: true, v: func(*uefiFixture, *testing.T) *VM { return efi() }},
|
||||
{name: "store record", v: func(f *uefiFixture, t *testing.T) *VM {
|
||||
v := f.instance(t, keyFor("k"), f.identity)
|
||||
v.BootMode = "uefi-preferred"
|
||||
return v
|
||||
}},
|
||||
{name: "BIOS", v: func(*uefiFixture, *testing.T) *VM { return &VM{ID: testUEFIInstance, BootMode: "legacy-bios"} }},
|
||||
{name: "no boot mode", v: func(*uefiFixture, *testing.T) *VM { return &VM{ID: testUEFIInstance} }},
|
||||
{name: "microvm", v: func(*uefiFixture, *testing.T) *VM {
|
||||
return &VM{ID: testUEFIInstance, BootMode: "uefi", DirectBoot: true}
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.NoError(t, CheckUEFIVarStore(tt.v))
|
||||
f := newUEFIFixture(t)
|
||||
if tt.unwir {
|
||||
f.m.deps.UEFIVarStore.Records = nil
|
||||
}
|
||||
if tt.unreadable {
|
||||
f.m.deps.UEFIVarStore.Records = failingRecordGet{f.records}
|
||||
}
|
||||
assert.NoError(t, f.m.CheckUEFIVarStore(t.Context(), tt.v(f, t)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A launch from a pointer opens this node's copy as pflash unit 1 with
|
||||
// A store-record launch opens this node's copy as pflash unit 1 with
|
||||
// format=raw, after the read-only CODE blob.
|
||||
func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
|
||||
func TestStartQEMU_StoreObjectPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
|
||||
launches := withFakeQEMU(t)
|
||||
key := keyFor("boot")
|
||||
f.writeLocal(t, key, region("seeded"))
|
||||
instance := f.instance(key, f.identity)
|
||||
f.writeLocal(t, key, 1, region("seeded"))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
|
||||
|
||||
require.NoError(t, f.m.startQEMU(instance))
|
||||
@@ -497,11 +653,11 @@ func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
|
||||
|
||||
// A reboot reuses the copy the launch seeded; without one there is nothing
|
||||
// safe to boot, so QEMU is not started.
|
||||
func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
|
||||
func TestStartQEMU_StoreObjectWithoutALocalCopyIsNotLaunched(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
|
||||
launches := withFakeQEMU(t)
|
||||
instance := f.instance(keyFor("boot"), f.identity)
|
||||
instance := f.instance(t, keyFor("boot"), f.identity)
|
||||
instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
|
||||
|
||||
err := f.m.startQEMU(instance)
|
||||
@@ -510,6 +666,68 @@ func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
|
||||
assert.Empty(t, launches())
|
||||
}
|
||||
|
||||
// A launch that fails before QEMU starts gives back a claim whose store it
|
||||
// fetched, since nothing on this node is newer than the object.
|
||||
func TestRun_LaunchFailingBeforeQEMUReleasesAFetchedClaim(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("fetched")
|
||||
f.putObject(t, key, gzipped(t, region("object")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
f.mounter.mountErr = errors.New("mount refused")
|
||||
|
||||
require.Error(t, f.m.Run(t.Context(), instance))
|
||||
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held, "another node may start the instance at once")
|
||||
}
|
||||
|
||||
// A launch that booted this node's copy keeps the claim when it fails, because
|
||||
// that copy may hold variable changes no object has.
|
||||
func TestRun_LaunchFailingAfterReusingTheLocalCopyKeepsTheClaim(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("reused")
|
||||
instance := f.instance(t, key, f.identity)
|
||||
tok, err := f.peerClaim("node-a").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
_, err = f.records.Stamp(t.Context(), testUEFIInstance, tok.Generation)
|
||||
require.NoError(t, err)
|
||||
f.writeLocal(t, key, tok.Generation, region("unlanded"))
|
||||
f.mounter.mountErr = errors.New("mount refused")
|
||||
|
||||
require.Error(t, f.m.Run(t.Context(), instance))
|
||||
|
||||
holder, held := f.claimHolder(t)
|
||||
require.True(t, held)
|
||||
assert.Equal(t, tok, holder)
|
||||
}
|
||||
|
||||
// A recreated sync bucket restarts generations, so a store record can sit
|
||||
// above every new token. The start is refused as retryable and the record
|
||||
// keeps its generation and pointer.
|
||||
func TestRun_RecordAboveTheClaimGenerationIsRefusedAndUnchanged(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("fenced")
|
||||
f.putObject(t, key, gzipped(t, region("object")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
rec, rev := f.record(t)
|
||||
rec.Generation = 1 << 40
|
||||
_, err := f.records.CompareAndSet(t.Context(), rec, rev)
|
||||
require.NoError(t, err)
|
||||
want, wantRev := f.record(t)
|
||||
before := uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", "fenced")
|
||||
|
||||
err = f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrUEFIClaimHeld)
|
||||
require.ErrorIs(t, err, ErrMountRetryable)
|
||||
got, gotRev := f.record(t)
|
||||
assert.Equal(t, wantRev, gotRev, "the record is not written")
|
||||
assert.Equal(t, want, got)
|
||||
assert.Empty(t, f.mounter.mounted)
|
||||
assert.NoFileExists(t, filepath.Join(f.instanceDir(), "vars.fd"))
|
||||
assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.claim_refusals", "reason", "fenced"))
|
||||
}
|
||||
|
||||
func TestUEFIInstanceDir_RefusesIDsThatAreNotOnePathSegment(t *testing.T) {
|
||||
for _, id := range []string{"", "..", "i-../x", "i-a/b", `i-a\b`, "vol-1"} {
|
||||
_, err := uefiInstanceDir("/data", id)
|
||||
@@ -537,3 +755,66 @@ func TestNodeFirmwareFor_HashesTheCodeBlobAgainAfterItChanges(t *testing.T) {
|
||||
assert.Equal(t, hex.EncodeToString(sum[:]), second.Identity.CodeSHA256)
|
||||
assert.Equal(t, f.codePath, second.Code)
|
||||
}
|
||||
|
||||
// failingRecordGet is a store record bucket whose reads fail.
|
||||
type failingRecordGet struct{ UEFIStoreRecords }
|
||||
|
||||
func (failingRecordGet) Get(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
|
||||
return nil, 0, errors.New("uefi stores: no responders")
|
||||
}
|
||||
|
||||
// At one generation only this node moves the pointer, and only from vars.fd,
|
||||
// so a crash between a move and the seeded.json rewrite leaves a file newer
|
||||
// than the key seeded.json names. It is booted, not refetched over.
|
||||
func TestPrepareUEFIVarStore_ReusesTheCopyOfTheClaimsGenerationWhateverItsKey(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("moved")
|
||||
f.putObject(t, key, gzipped(t, region("object")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
tok, err := f.peerClaim("node-a").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
f.writeLocal(t, keyFor("before-move"), tok.Generation, region("newer than the object"))
|
||||
|
||||
held, err := f.prepare(t, instance)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.True(t, held.reused)
|
||||
assert.Equal(t, region("newer than the object"), f.localVars(t))
|
||||
}
|
||||
|
||||
// A launch that fails inside startQEMU before the process starts ran no
|
||||
// guest, so it gives back a claim it fetched the store under.
|
||||
func TestRun_LaunchFailingBeforeQEMUExecsReleasesAFetchedClaim(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("fetched")
|
||||
f.putObject(t, key, gzipped(t, region("object")))
|
||||
instance := f.instance(t, key, f.identity)
|
||||
|
||||
err := f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, errQEMUNotStarted, "an instance with no boot volume fails before QEMU execs")
|
||||
_, held := f.claimHolder(t)
|
||||
assert.False(t, held, "another node may start the instance at once")
|
||||
}
|
||||
|
||||
// A node that cannot boot the store refuses before it touches the claim, so
|
||||
// it never takes the store from a holder that is coming back.
|
||||
func TestRun_FirmwareMismatchRefusesBeforeTakingTheClaim(t *testing.T) {
|
||||
f := newUEFIFixture(t)
|
||||
key := keyFor("arm")
|
||||
stored := f.identity
|
||||
stored.Arch = "arm64"
|
||||
instance := f.instance(t, key, stored)
|
||||
tok, err := f.peerClaim("node-b").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
f.liveness.heartbeat("node-b", 10*time.Minute)
|
||||
|
||||
err = f.m.Run(t.Context(), instance)
|
||||
|
||||
require.ErrorIs(t, err, ErrUEFIFirmwareMismatch)
|
||||
holder, held := f.claimHolder(t)
|
||||
require.True(t, held)
|
||||
assert.Equal(t, tok, holder, "the claim stays with node-b")
|
||||
rec, _ := f.record(t)
|
||||
assert.Zero(t, rec.Generation, "the record is not stamped")
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
@@ -319,20 +318,6 @@ type VM struct {
|
||||
// tolerated by migrate.
|
||||
InstanceLifecycle string `json:"instance_lifecycle,omitempty"`
|
||||
SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
|
||||
|
||||
// UEFIVarStore points at the object holding this instance's UEFI variable
|
||||
// store. Status, so written only by the node; nil for a BIOS instance.
|
||||
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
|
||||
}
|
||||
|
||||
// UEFIVarStoreRef names the store object an instance launches from, with the
|
||||
// firmware it was written under, so a launch can check it without reading the
|
||||
// object's metadata.
|
||||
type UEFIVarStoreRef struct {
|
||||
Key string `json:"key"`
|
||||
Firmware uefistore.FirmwareIdentity `json:"firmware"`
|
||||
WriterNode string `json:"writer_node"`
|
||||
WrittenAt time.Time `json:"written_at"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON folds the fields of legacy records that mixed a request with an
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
|
||||
reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
|
||||
@@ -16,8 +17,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// daemonLocks is every lease the daemon owns, each constructed once against the
|
||||
// process's sync bucket and handed to the sites that run under it.
|
||||
// daemonLocks is every lease and claim the daemon owns, each constructed once
|
||||
// against the process's sync bucket and handed to the sites that run under it.
|
||||
type daemonLocks struct {
|
||||
dns *kvsync.Lease
|
||||
route53Realize *kvsync.Lease
|
||||
@@ -26,6 +27,7 @@ type daemonLocks struct {
|
||||
ecs *kvsync.Lease
|
||||
ochre *kvsync.Lease
|
||||
eks *kvsync.LeaseGroup
|
||||
uefi *kvsync.ClaimGroup
|
||||
}
|
||||
|
||||
func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
|
||||
@@ -37,6 +39,7 @@ func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
|
||||
ecs: ecsdomain.NewSchedulerLease(store, node),
|
||||
ochre: ochre.NewReaperLease(store, node),
|
||||
eks: ekscluster.NewClusterLeases(store, node),
|
||||
uefi: uefistore.NewClaims(store, node),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,3 +67,12 @@ func deleteObsoleteBuckets(ctx context.Context, js jetstream.JetStream) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// uefiClaims is the UEFI store claim group, or nil before startCluster has
|
||||
// built the daemon's locks.
|
||||
func (d *Daemon) uefiClaims() *kvsync.ClaimGroup {
|
||||
if d.locks == nil {
|
||||
return nil
|
||||
}
|
||||
return d.locks.uefi
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ func TestDaemonLocks_EveryOwnerHasItsOwnKey(t *testing.T) {
|
||||
reconcilelock.NewLeaderLease(store, admissionquota.KeyReconcile, "node-1")
|
||||
locks.eks.Get("123456789012/alpha")
|
||||
locks.eks.Get("123456789012/alpha/teardown")
|
||||
locks.uefi.Get("i-0123456789abcdef0")
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// clusterHeartbeats satisfies vm.NodeLiveness from the heartbeats every daemon
|
||||
// writes to the cluster-state bucket.
|
||||
type clusterHeartbeats struct {
|
||||
kv jetstream.KeyValue
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
var _ vm.NodeLiveness = clusterHeartbeats{}
|
||||
|
||||
// LastSeen returns when node last wrote a heartbeat. No entry is found=false;
|
||||
// a failed read, or an entry that cannot be dated, is an error, because a beat
|
||||
// that exists may be a live node's.
|
||||
//
|
||||
// A replica that lags can only answer with an older beat, so a live one is
|
||||
// returned as read and anything else is read again from the stream leader.
|
||||
func (h clusterHeartbeats) LastSeen(ctx context.Context, node string) (time.Time, bool, error) {
|
||||
key := clusterv1.HeartbeatKey(node)
|
||||
if entry, err := h.kv.Get(ctx, key); err == nil {
|
||||
if written, ok := clusterv1.HeartbeatWritten(entry.Value()); ok && clusterv1.LiveAt(written, time.Now()) {
|
||||
return written, true, nil
|
||||
}
|
||||
}
|
||||
if h.js == nil {
|
||||
return time.Time{}, false, fmt.Errorf("read heartbeat for %s: no JetStream client to confirm it with the leader", node)
|
||||
}
|
||||
bucket := clusterv1.ClusterStateBucket
|
||||
msg, err := kvstore.LeaderMsgGet(ctx, h.js, "KV_"+bucket, kvstore.MsgGetRequest{LastFor: "$KV." + bucket + "." + key})
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("read heartbeat for %s from the leader: %w", node, err)
|
||||
}
|
||||
if op := msg.Header.Get(kvOperationHeader); op == "DEL" || op == "PURGE" {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
written, ok := clusterv1.HeartbeatWritten(msg.Data)
|
||||
if !ok {
|
||||
return time.Time{}, false, fmt.Errorf("heartbeat for %s cannot be dated", node)
|
||||
}
|
||||
return written, true, nil
|
||||
}
|
||||
|
||||
// kvOperationHeader marks a KV stream message that deletes or purges its key.
|
||||
const kvOperationHeader = "KV-Operation"
|
||||
|
||||
// heartbeatLiveness returns nil, not a typed nil, before the cluster-state
|
||||
// bucket is open, so a launch sees liveness as unwired rather than panicking.
|
||||
func (d *Daemon) heartbeatLiveness() vm.NodeLiveness {
|
||||
if d.jsManager == nil || d.jsManager.clusterKV == nil {
|
||||
return nil
|
||||
}
|
||||
return clusterHeartbeats{kv: d.jsManager.clusterKV, js: d.jsManager.js}
|
||||
}
|
||||
|
||||
// uefiStoreRecords returns nil, not a typed nil, when the store bucket is not
|
||||
// open, so a launch sees store records as unwired.
|
||||
func (d *Daemon) uefiStoreRecords() vm.UEFIStoreRecords {
|
||||
if d.jsManager == nil || d.jsManager.UEFIStores() == nil {
|
||||
return nil
|
||||
}
|
||||
return d.jsManager.UEFIStores()
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//test:in-package — clusterHeartbeats is an unexported daemon adapter
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The claim's takeover rule reads the same heartbeat the daemon writes. Only
|
||||
// an absent beat is not found: one that exists but cannot be dated may be a
|
||||
// live node's, so it is an error the claim refuses on (ADR-0003:S7).
|
||||
func TestClusterHeartbeats_LastSeen(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
|
||||
require.NoError(t, err)
|
||||
h := clusterHeartbeats{kv: kv, js: js}
|
||||
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-a"), []byte(`{"node":"node-a","timestamp":"2026-11-05T01:02:03Z"}`))
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), []byte(`not json`))
|
||||
require.NoError(t, err)
|
||||
|
||||
seen, found, err := h.LastSeen(t.Context(), "node-a")
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
assert.True(t, seen.Equal(time.Date(2026, 11, 5, 1, 2, 3, 0, time.UTC)))
|
||||
|
||||
_, found, err = h.LastSeen(t.Context(), "node-b")
|
||||
require.Error(t, err, "ADR-0003:S7 (uefi_store_claims.go LastSeen): an undecodable heartbeat must not read as absent, or a live holder is taken over")
|
||||
assert.False(t, found)
|
||||
|
||||
_, found, err = h.LastSeen(t.Context(), "node-c")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
}
|
||||
|
||||
// A bucket that cannot be read is an error, which the claim reads as a
|
||||
// holder that may be live.
|
||||
func TestClusterHeartbeats_UnreadableBucketIsAnError(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, js.DeleteKeyValue(t.Context(), clusterv1.ClusterStateBucket))
|
||||
|
||||
_, _, err = clusterHeartbeats{kv: kv, js: js}.LastSeen(t.Context(), "node-a")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// laggingReplica answers every get as a replica behind the leader would:
|
||||
// with the beat it held before, or with no key at all.
|
||||
type laggingReplica struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
old []byte
|
||||
}
|
||||
|
||||
func (r laggingReplica) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
if r.old == nil {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return staleEntry{value: r.old}, nil
|
||||
}
|
||||
|
||||
type staleEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e staleEntry) Value() []byte { return e.value }
|
||||
|
||||
// TestS7_HeartbeatALaggingReplicaMissesIsReadFromTheLeader enforces
|
||||
// ADR-0003:S7: a get a lagging replica answers with an old beat, or none, must
|
||||
// not read as a dead holder while the leader holds a fresh one.
|
||||
func TestS7_HeartbeatALaggingReplicaMissesIsReadFromTheLeader(t *testing.T) {
|
||||
old, err := json.Marshal(clusterv1.HeartbeatTimestamp{Timestamp: time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)})
|
||||
require.NoError(t, err)
|
||||
for name, replica := range map[string][]byte{"an old beat": old, "no key": nil} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
|
||||
require.NoError(t, err)
|
||||
fresh := time.Now().UTC().Truncate(time.Second)
|
||||
body, err := json.Marshal(clusterv1.HeartbeatTimestamp{Timestamp: fresh.Format(time.RFC3339)})
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), body)
|
||||
require.NoError(t, err)
|
||||
|
||||
seen, found, err := clusterHeartbeats{kv: laggingReplica{KeyValue: kv, old: replica}, js: js}.LastSeen(t.Context(), "node-b")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, found, "ADR-0003:S7 (uefi_store_claims.go LastSeen): a lagging replica's answer was trusted; confirm a stale or absent beat with the stream leader")
|
||||
assert.True(t, seen.Equal(fresh), "ADR-0003:S7: the leader's beat must be the one returned")
|
||||
assert.True(t, clusterv1.LiveAt(seen, time.Now()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A deleted heartbeat is no heartbeat, as an absent key is.
|
||||
func TestClusterHeartbeats_DeletedBeatIsNotFound(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), clusterv1.HeartbeatKey("node-b"), []byte(`{"timestamp":"2026-11-05T01:02:03Z"}`))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, kv.Delete(t.Context(), clusterv1.HeartbeatKey("node-b")))
|
||||
|
||||
_, found, err := clusterHeartbeats{kv: kv, js: js}.LastSeen(t.Context(), "node-b")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
@@ -78,8 +79,9 @@ type uefiInstanceRecords interface {
|
||||
var _ uefiInstanceRecords = (*JetStreamManager)(nil)
|
||||
|
||||
// uefiStoreReaper deletes the variable stores whose record is deleting: each
|
||||
// object under the instance's prefix, then the record. Terminate, or an
|
||||
// operator's forget, sets deleting and returns, so this decides nothing.
|
||||
// object under the instance's prefix, then the store's claim, then the record.
|
||||
// Terminate, or an operator's forget, sets deleting and returns, so this
|
||||
// decides nothing.
|
||||
//
|
||||
// It also completes a terminate whose mark failed, from the terminated
|
||||
// record's pending or failed uefi entry. That record is the terminate itself,
|
||||
@@ -94,14 +96,15 @@ type uefiStoreReaper struct {
|
||||
objects objectstore.ObjectStore
|
||||
bucket string
|
||||
instances uefiInstanceRecords
|
||||
claims *kvsync.ClaimGroup
|
||||
}
|
||||
|
||||
var _ vm.Reaper = (*uefiStoreReaper)(nil)
|
||||
|
||||
// newUEFIStoreReaper builds the reaper, or nil when this node has no store
|
||||
// bucket or object store to reap with.
|
||||
// bucket, claim group or object store to reap with.
|
||||
func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
|
||||
if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
|
||||
if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.uefiClaims() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
|
||||
return nil
|
||||
}
|
||||
return &uefiStoreReaper{
|
||||
@@ -110,6 +113,7 @@ func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
|
||||
d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
|
||||
bucket: d.config.Predastore.Bucket,
|
||||
instances: d.jsManager,
|
||||
claims: d.uefiClaims(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +176,10 @@ func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
|
||||
"instanceId", id, "deleted", deleted, "err", err)
|
||||
return false
|
||||
}
|
||||
if err := r.removeClaim(ctx, id); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to remove the store's claim, retrying next sweep", "instanceId", id, "err", err)
|
||||
return false
|
||||
}
|
||||
if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record, retrying next sweep", "instanceId", id, "err", err)
|
||||
return false
|
||||
@@ -187,6 +195,43 @@ func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// removeClaimAttempts bounds removeClaim's retries against a holder that moves
|
||||
// the key on between the read and the takeover.
|
||||
const removeClaimAttempts = 3
|
||||
|
||||
// removeClaim deletes instanceID's store claim by taking it over from whatever
|
||||
// holder it names and unlocking it. The store is deleting, so no stamp or
|
||||
// pointer move can follow, and the holder needs no proof of being dead.
|
||||
func (r *uefiStoreReaper) removeClaim(ctx context.Context, id string) error {
|
||||
claim := r.claims.Get(id)
|
||||
for range removeClaimAttempts {
|
||||
cur, held, err := claim.Holder(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
return nil
|
||||
}
|
||||
tok, err := claim.TakeOver(ctx, cur)
|
||||
if _, moved := errors.AsType[*kvsync.HeldError](err); moved {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = claim.Unlock(ctx, tok)
|
||||
if errors.Is(err, kvsync.ErrLost) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.InfoContext(ctx, "uefi-store: claim removed", "instanceId", id, "holder", cur.Holder, "generation", cur.Generation)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("uefi-store: claim on %s kept moving after %d attempts", id, removeClaimAttempts)
|
||||
}
|
||||
|
||||
// completeTerminatedMarks finishes the mark of each terminate that recorded a
|
||||
// uefi entry but did not land its store record in deleting.
|
||||
func (r *uefiStoreReaper) completeTerminatedMarks(ctx context.Context) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -36,10 +37,16 @@ func newUEFIReaperFixture(t *testing.T) *uefiReaperFixture {
|
||||
return &uefiReaperFixture{
|
||||
m: m,
|
||||
objects: objects,
|
||||
r: &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m},
|
||||
r: &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m,
|
||||
claims: uefistore.NewClaims(kvstore.OpenSyncBucket(m.js), "node-reaper")},
|
||||
}
|
||||
}
|
||||
|
||||
// claim is id's store claim as node sees it.
|
||||
func (f *uefiReaperFixture) claim(id, node string) *kvsync.Claim {
|
||||
return uefistore.NewClaims(kvstore.OpenSyncBucket(f.m.js), node).Get(id)
|
||||
}
|
||||
|
||||
// seedStore writes a live store record and n objects under its prefix, the
|
||||
// state a launch leaves, then marks it deleting if asked, as terminate does.
|
||||
func (f *uefiReaperFixture) seedStore(t *testing.T, id string, n int, deleting bool) {
|
||||
@@ -344,3 +351,73 @@ func TestS7_UEFIStoreReaperLeavesTheStoreWhenTheTerminatedRecordExpired(t *testi
|
||||
assert.Equal(t, 3, f.objectCount(t, id), "ADR-0003:S7: store objects deleted with no record proving QEMU exited")
|
||||
assert.Equal(t, uefistore.StateLive, f.recordState(t, id))
|
||||
}
|
||||
|
||||
// The reaper removes a deleted store's claim whoever holds it, so a later
|
||||
// instance can never find a stale holder, and leaves every live store's
|
||||
// claim to the node that may still hold its unlanded variables.
|
||||
func TestUEFIStoreReaper_RemovesOnlyADeletedStoresClaim(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const (
|
||||
deleted = "i-00000000000deleted"
|
||||
live = "i-0000000000000live"
|
||||
)
|
||||
f.writeTerminated(t, deleted, vm.TeardownPending)
|
||||
f.seedStore(t, deleted, 2, true)
|
||||
f.writeInstance(t, live, vm.StateRunning)
|
||||
f.seedStore(t, live, 2, false)
|
||||
for _, id := range []string{deleted, live} {
|
||||
_, err := f.claim(id, "node-b").TryLock(t.Context())
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
|
||||
_, held, err := f.claim(deleted, "observer").Holder(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.False(t, held, "the deleted store's claim must be removed")
|
||||
assert.Equal(t, "absent", f.recordState(t, deleted))
|
||||
cur, held, err := f.claim(live, "observer").Holder(t.Context())
|
||||
require.NoError(t, err)
|
||||
require.True(t, held, "a live store's claim must survive the sweep")
|
||||
assert.Equal(t, "node-b", cur.Holder)
|
||||
}
|
||||
|
||||
// failingClaims stands in for a sync bucket that does not answer.
|
||||
type failingClaims struct{}
|
||||
|
||||
func (failingClaims) Load(context.Context, string) ([]byte, uint64, error) {
|
||||
return nil, 0, kvsync.ErrUnavailable
|
||||
}
|
||||
|
||||
func (failingClaims) CompareAndSwap(context.Context, string, []byte, uint64) (uint64, error) {
|
||||
return 0, kvsync.ErrUnavailable
|
||||
}
|
||||
|
||||
func (failingClaims) CompareAndDelete(context.Context, string, uint64) error {
|
||||
return kvsync.ErrUnavailable
|
||||
}
|
||||
|
||||
// A claim the reaper cannot remove keeps the store record, so the next sweep
|
||||
// still finds the obligation and finishes it.
|
||||
func TestUEFIStoreReaper_KeepsTheRecordUntilTheClaimIsRemoved(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.writeTerminated(t, id, vm.TeardownPending)
|
||||
f.seedStore(t, id, 2, true)
|
||||
healthy := f.r.claims
|
||||
f.r.claims = uefistore.NewClaims(&failingClaims{}, "node-reaper")
|
||||
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reaped)
|
||||
assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
|
||||
assert.Equal(t, string(vm.TeardownPending), f.terminatedUEFIMark(t, id))
|
||||
|
||||
f.r.claims = healthy
|
||||
reaped, err = f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
assert.Equal(t, "absent", f.recordState(t, id))
|
||||
}
|
||||
|
||||
@@ -878,9 +878,12 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
|
||||
ConsumeCleanShutdownMarker: d.consumeCleanShutdownMarker(),
|
||||
BackingStoreReady: func() bool { return d.checkPredastoreReady() && d.checkViperblockReady() },
|
||||
UEFIVarStore: vm.UEFIVarStoreDeps{
|
||||
Objects: d.instanceObjects,
|
||||
Bucket: d.config.Predastore.Bucket,
|
||||
DataDir: DataDirOrDefault(d.config.DataDir),
|
||||
Objects: d.instanceObjects,
|
||||
Bucket: d.config.Predastore.Bucket,
|
||||
DataDir: DataDirOrDefault(d.config.DataDir),
|
||||
Records: d.uefiStoreRecords(),
|
||||
Claims: d.uefiClaims(),
|
||||
Liveness: d.heartbeatLiveness(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,6 +199,11 @@ func TestBuildVMManagerDeps_UEFIVarStore(t *testing.T) {
|
||||
|
||||
d.config.DataDir = ""
|
||||
assert.Equal(t, DefaultDataDir, d.buildVMManagerDeps().UEFIVarStore.DataDir)
|
||||
|
||||
// Before the buckets and locks exist, each is unwired rather than a typed nil.
|
||||
assert.Nil(t, got.Records)
|
||||
assert.Nil(t, got.Claims)
|
||||
assert.Nil(t, got.Liveness)
|
||||
}
|
||||
|
||||
// --- volumeMounterAdapter ---
|
||||
|
||||
Reference in New Issue
Block a user