mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-03 17:54:42 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3a79df1198 | |||
| 085e9edf8e | |||
| e36a77ed80 |
+1
-1
@@ -135,7 +135,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
|
||||
| `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
|
||||
| `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
|
||||
| `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
@@ -121,6 +122,15 @@ type StoppedInstanceStore interface {
|
||||
ClaimStoppedInstance(instanceID string) (*vm.VM, error)
|
||||
}
|
||||
|
||||
// UEFIStoreMarker records a terminated instance's UEFI variable store for
|
||||
// deletion. Implemented by uefistore.Store.
|
||||
type UEFIStoreMarker interface {
|
||||
Exists(ctx context.Context, instanceID string) (bool, error)
|
||||
MarkDeleting(ctx context.Context, instanceID string) (bool, error)
|
||||
}
|
||||
|
||||
var _ UEFIStoreMarker = (*uefistore.Store)(nil)
|
||||
|
||||
// InstanceTagWriter projects an instance record's full tag set into the
|
||||
// central tag store, and removes it on terminate. Implemented by
|
||||
// domains/ec2/tags.TagsServiceImpl.
|
||||
|
||||
@@ -234,6 +234,7 @@ type InstanceServiceImpl struct {
|
||||
eniDeleter ENIDeleter
|
||||
ipReleaser PublicIPReleaser
|
||||
tagWriter InstanceTagWriter
|
||||
uefiStores UEFIStoreMarker
|
||||
gpuClaimer GPUClaimer
|
||||
amiLoader AMIMetaLoader
|
||||
ebsProvider ebsprovider.EBSProvider
|
||||
@@ -314,6 +315,12 @@ func (s *InstanceServiceImpl) SetTerminationDeps(vd VolumeDeleter, ed ENIDeleter
|
||||
s.tagWriter = tw
|
||||
}
|
||||
|
||||
// SetUEFIStores wires the store records a stopped instance's terminate marks
|
||||
// for deletion. nil leaves terminate recording no UEFI store obligation.
|
||||
func (s *InstanceServiceImpl) SetUEFIStores(m UEFIStoreMarker) {
|
||||
s.uefiStores = m
|
||||
}
|
||||
|
||||
// SetGPUClaimer wires the GPU passthrough claim/release dependency used by
|
||||
// StartStoppedInstance. nil disables GPU passthrough for the service.
|
||||
func (s *InstanceServiceImpl) SetGPUClaimer(g GPUClaimer) {
|
||||
@@ -2927,6 +2934,7 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
|
||||
s.deleteInstanceVolumes(ctx, instance, input.InstanceID)
|
||||
s.releaseInstancePublicIP(ctx, instance, input.InstanceID)
|
||||
s.deleteInstanceENI(ctx, instance, input.InstanceID)
|
||||
hasUEFIStore := s.stampUEFIStoreTeardown(ctx, instance)
|
||||
|
||||
// Write to terminated KV FIRST so the instance is visible in DescribeInstances.
|
||||
// If this fails the instance stays in the stopped bucket — safe to retry.
|
||||
@@ -2936,6 +2944,16 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
// Marked only once the terminate is durable: a stopped instance whose
|
||||
// terminate failed keeps its store. A failed mark leaves the teardown entry
|
||||
// pending, from which the store reaper completes it.
|
||||
if hasUEFIStore {
|
||||
if _, err := s.uefiStores.MarkDeleting(ctx, input.InstanceID); err != nil {
|
||||
slog.WarnContext(ctx, "TerminateStoppedInstance: failed to record UEFI variable store for deletion",
|
||||
"instanceId", input.InstanceID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Now safe to remove from stopped KV. Retry once on failure so the instance
|
||||
// doesn't appear in both buckets.
|
||||
if err := s.stoppedStore.DeleteStoppedInstance(input.InstanceID); err != nil {
|
||||
@@ -2953,6 +2971,29 @@ func (s *InstanceServiceImpl) TerminateStoppedInstance(ctx context.Context, inpu
|
||||
return &TerminateStoppedInstanceOutput{Status: "terminated", InstanceID: input.InstanceID}, nil
|
||||
}
|
||||
|
||||
// stampUEFIStoreTeardown records the UEFI store teardown entry the terminated
|
||||
// record carries, and reports whether the instance has a store to mark. An
|
||||
// unreadable bucket is stamped failed, which the store reaper resolves.
|
||||
func (s *InstanceServiceImpl) stampUEFIStoreTeardown(ctx context.Context, instance *vm.VM) bool {
|
||||
if s.uefiStores == nil {
|
||||
return false
|
||||
}
|
||||
has, err := s.uefiStores.Exists(ctx, instance.ID)
|
||||
state := vm.TeardownPending
|
||||
switch {
|
||||
case err != nil:
|
||||
slog.WarnContext(ctx, "TerminateStoppedInstance: failed to read UEFI store record", "instanceId", instance.ID, "err", err)
|
||||
state = vm.TeardownFailed
|
||||
case !has:
|
||||
return false
|
||||
}
|
||||
if instance.Teardown == nil {
|
||||
instance.Teardown = make(map[string]string)
|
||||
}
|
||||
instance.Teardown[vm.TeardownUEFI] = string(state)
|
||||
return has
|
||||
}
|
||||
|
||||
func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instance *vm.VM, instanceID string) {
|
||||
instance.EBSRequests.Mu.Lock()
|
||||
defer instance.EBSRequests.Mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package instance
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const uefiTerminateID = "i-0123456789abcdef0"
|
||||
|
||||
// stoppedUEFIFixture is a stopped instance with a live store record and three
|
||||
// retained objects, the state a launch and a few stops leave.
|
||||
func stoppedUEFIFixture(t *testing.T) (*InstanceServiceImpl, *vmmock.StateStore, *uefistore.Store, *objectstore.MemoryObjectStore) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
stores := uefistore.NewStore(js)
|
||||
require.NoError(t, stores.Open(t.Context()))
|
||||
_, err := stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: uefiTerminateID, AccountID: "acc", State: uefistore.StateLive})
|
||||
require.NoError(t, err)
|
||||
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
for _, k := range []string{"a", "b", "c"} {
|
||||
_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
|
||||
Bucket: aws.String("spinifex"), Key: aws.String(uefistore.KeyPrefix(uefiTerminateID) + k), Body: bytes.NewReader([]byte("vars")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
|
||||
uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
|
||||
}}
|
||||
svc := &InstanceServiceImpl{stoppedStore: store, objectStore: objects}
|
||||
svc.SetUEFIStores(stores)
|
||||
return svc, store, stores, objects
|
||||
}
|
||||
|
||||
func storeState(t *testing.T, stores *uefistore.Store) string {
|
||||
t.Helper()
|
||||
rec, _, err := stores.Get(t.Context(), uefiTerminateID)
|
||||
require.NoError(t, err)
|
||||
return rec.State
|
||||
}
|
||||
|
||||
// TestS8_TerminateStoppedInstanceRecordsTheUEFIStoreObligationAndReturns
|
||||
// enforces ADR-0003:S8: the request deletes no store object, it records the
|
||||
// obligation on the store record and the terminated record, and returns.
|
||||
func TestS8_TerminateStoppedInstanceRecordsTheUEFIStoreObligationAndReturns(t *testing.T) {
|
||||
svc, store, stores, objects := stoppedUEFIFixture(t)
|
||||
|
||||
_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
|
||||
require.NoError(t, err)
|
||||
|
||||
keys, err := uefistore.ListObjects(t.Context(), objects, "spinifex", uefiTerminateID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, keys, 3, "ADR-0003:S8: the request must leave the object deletion to the reaper")
|
||||
assert.Equal(t, uefistore.StateDeleting, storeState(t, stores), "the store record carries the obligation")
|
||||
require.NotNil(t, store.WroteTerminated[uefiTerminateID])
|
||||
assert.Equal(t, string(vm.TeardownPending), store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI])
|
||||
}
|
||||
|
||||
// TestS7_FailedStoppedTerminateKeepsTheUEFIStoreLive enforces ADR-0003:S7: a
|
||||
// terminate that fails before it is durable leaves the instance stopped, so
|
||||
// its store must stay live rather than be handed to the reaper.
|
||||
func TestS7_FailedStoppedTerminateKeepsTheUEFIStoreLive(t *testing.T) {
|
||||
svc, store, stores, _ := stoppedUEFIFixture(t)
|
||||
store.WriteTerminatedErr = errors.New("kv write boom")
|
||||
|
||||
_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Equal(t, uefistore.StateLive, storeState(t, stores),
|
||||
"ADR-0003:S7: a stopped instance whose terminate failed must keep its store")
|
||||
}
|
||||
|
||||
func TestTerminateStoppedInstance_NoUEFIStoreStampsNoEntry(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
stores := uefistore.NewStore(js)
|
||||
require.NoError(t, stores.Open(t.Context()))
|
||||
store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
|
||||
uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
|
||||
}}
|
||||
svc := &InstanceServiceImpl{stoppedStore: store}
|
||||
svc.SetUEFIStores(stores)
|
||||
|
||||
_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, stamped := store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI]
|
||||
assert.False(t, stamped)
|
||||
}
|
||||
|
||||
type unreadableUEFIStores struct{ marked bool }
|
||||
|
||||
func (u *unreadableUEFIStores) Exists(context.Context, string) (bool, error) {
|
||||
return false, errors.New("kv unavailable")
|
||||
}
|
||||
|
||||
func (u *unreadableUEFIStores) MarkDeleting(context.Context, string) (bool, error) {
|
||||
u.marked = true
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// An unreadable bucket does not fail the terminate; the entry is stamped
|
||||
// failed so the store reaper settles it from the terminated record.
|
||||
func TestTerminateStoppedInstance_UnreadableUEFIStoreIsStampedFailed(t *testing.T) {
|
||||
store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
|
||||
uefiTerminateID: {ID: uefiTerminateID, Status: vm.StateStopped, AccountID: "acc"},
|
||||
}}
|
||||
stores := &unreadableUEFIStores{}
|
||||
svc := &InstanceServiceImpl{stoppedStore: store}
|
||||
svc.SetUEFIStores(stores)
|
||||
|
||||
_, err := svc.TerminateStoppedInstance(t.Context(), &TerminateStoppedInstanceInput{InstanceID: uefiTerminateID}, "acc")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, string(vm.TeardownFailed), store.WroteTerminated[uefiTerminateID].Teardown[vm.TeardownUEFI])
|
||||
assert.False(t, stores.marked)
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package uefistore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// markAttempts bounds MarkDeleting's retries against concurrent writers. Each
|
||||
// lost race means another writer changed the record, and none of them can
|
||||
// return it to live, so a few rounds always settle.
|
||||
const markAttempts = 5
|
||||
|
||||
// Exists reports whether instanceID has a store record, without decoding it,
|
||||
// so a record this node cannot read still counts as one.
|
||||
func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
|
||||
return s.kv.Exists(ctx, instanceID)
|
||||
}
|
||||
|
||||
// MarkDeleting records that instanceID's store is to be deleted, and drops any
|
||||
// write-back still owed. It reports false with no error when the instance has
|
||||
// no store record. Only a terminate or an operator's forget may call it.
|
||||
func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
|
||||
for range markAttempts {
|
||||
rec, rev, err := s.Get(ctx, instanceID)
|
||||
if errors.Is(err, kvstore.ErrNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if rec.State == StateDeleting {
|
||||
return true, nil
|
||||
}
|
||||
rec.State = StateDeleting
|
||||
rec.WriteBack = nil
|
||||
_, err = s.CompareAndSet(ctx, rec, rev)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if !errors.Is(err, kvstore.ErrConflict) {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("uefistore: mark %s deleting: %w after %d attempts", instanceID, kvstore.ErrConflict, markAttempts)
|
||||
}
|
||||
|
||||
// InstanceIDs lists the key of every store record, sorted. It decodes
|
||||
// nothing, so a caller reads each record through Get and its validation.
|
||||
func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
|
||||
kv, err := s.kv.KV(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("uefistore: list %s: %w", BucketName, err)
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
|
||||
var ids []string
|
||||
for key := range lister.Keys() {
|
||||
if key != kvutil.VersionKey {
|
||||
ids = append(ids, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(ids)
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// validOwner reports whether ownerID is an instance or image ID that names
|
||||
// exactly one prefix, so a malformed ID can never widen a delete.
|
||||
func validOwner(ownerID string) bool {
|
||||
if strings.ContainsAny(ownerID, "/*") {
|
||||
return false
|
||||
}
|
||||
for _, p := range []string{"i-", "ami-"} {
|
||||
if rest, ok := strings.CutPrefix(ownerID, p); ok && rest != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ListObjects returns the key of every object under ownerID's prefix, across
|
||||
// every page of the listing.
|
||||
func ListObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID string) ([]string, error) {
|
||||
if !validOwner(ownerID) {
|
||||
return nil, fmt.Errorf("uefistore: %q is not an instance or image ID", ownerID)
|
||||
}
|
||||
contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(KeyPrefix(ownerID)),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
|
||||
}
|
||||
keys := make([]string, 0, len(contents))
|
||||
for _, obj := range contents {
|
||||
keys = append(keys, aws.StringValue(obj.Key))
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// DeleteObjects deletes every object under ownerID's prefix and reports how
|
||||
// many it removed. It succeeds only once a fresh listing finds the prefix
|
||||
// empty, so an object written during the delete is not left unaccounted for.
|
||||
func DeleteObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID string) (int, error) {
|
||||
keys, err := ListObjects(ctx, objects, bucket, ownerID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
deleted := 0
|
||||
for _, key := range keys {
|
||||
_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
if err != nil && !objectstore.IsNoSuchKeyError(err) {
|
||||
return deleted, fmt.Errorf("uefistore: delete %s: %w", key, err)
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
left, err := ListObjects(ctx, objects, bucket, ownerID)
|
||||
if err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
if len(left) > 0 {
|
||||
return deleted, fmt.Errorf("uefistore: %d object(s) remain under %s", len(left), KeyPrefix(ownerID))
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package uefistore_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const bucket = "spinifex"
|
||||
|
||||
func putObject(t *testing.T, objects objectstore.ObjectStore, key string) {
|
||||
t.Helper()
|
||||
_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader([]byte("vars")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []string {
|
||||
t.Helper()
|
||||
contents, _, err := objectstore.ListAll(t.Context(), objects, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(prefix)})
|
||||
require.NoError(t, err)
|
||||
var keys []string
|
||||
for _, o := range contents {
|
||||
keys = append(keys, aws.StringValue(o.Key))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Deletion is final: a writer holding the revision of a deleting record, such
|
||||
// as a write-back retry, must not bring the store back to live.
|
||||
func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
deleting := liveRecord()
|
||||
deleting.State = uefistore.StateDeleting
|
||||
rev, err = s.CompareAndSet(t.Context(), deleting, rev)
|
||||
require.NoError(t, err)
|
||||
|
||||
revived := liveRecord()
|
||||
revived.WriteBack = writeBack()
|
||||
_, err = s.CompareAndSet(t.Context(), revived, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrDeleting)
|
||||
|
||||
got, gotRev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev)
|
||||
assert.Equal(t, uefistore.StateDeleting, got.State)
|
||||
}
|
||||
|
||||
// A write-back owed on a deleting store would put an object behind the delete.
|
||||
func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
rec := liveRecord()
|
||||
rec.State = uefistore.StateDeleting
|
||||
rec.WriteBack = writeBack()
|
||||
|
||||
_, err = s.CompareAndSet(t.Context(), rec, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
}
|
||||
|
||||
func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
|
||||
has, err := s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has)
|
||||
|
||||
got, rev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uefistore.StateDeleting, got.State)
|
||||
assert.Nil(t, got.WriteBack)
|
||||
assert.Equal(t, "111122223333", got.AccountID)
|
||||
|
||||
has, err = s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has, "a repeated terminate finds the obligation already recorded")
|
||||
_, again, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, again, "a repeat writes nothing")
|
||||
}
|
||||
|
||||
func TestMarkDeleting_NoRecordIsNoStore(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
has, err := s.MarkDeleting(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, has)
|
||||
_, _, err = s.Get(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound, "marking must not create a record")
|
||||
}
|
||||
|
||||
// A record this node cannot trust is reported, never rewritten as deleting.
|
||||
func TestMarkDeleting_RefusesAnUntrustedRecord(t *testing.T) {
|
||||
s, js := openStore(t)
|
||||
rawPut(t, js, instanceID, `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`)
|
||||
|
||||
_, err := s.MarkDeleting(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
}
|
||||
|
||||
func TestExists_CountsAnUndecodableRecord(t *testing.T) {
|
||||
s, js := openStore(t)
|
||||
has, err := s.Exists(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, has)
|
||||
|
||||
rawPut(t, js, instanceID, `not json`)
|
||||
has, err = s.Exists(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has)
|
||||
}
|
||||
|
||||
func TestInstanceIDs_ListsRecordsButNotTheVersionKey(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
ids, err := s.InstanceIDs(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ids)
|
||||
|
||||
for _, id := range []string{"i-0000000000000000b", "i-0000000000000000a"} {
|
||||
rec := liveRecord()
|
||||
rec.InstanceID = id
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
ids, err = s.InstanceIDs(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"i-0000000000000000a", "i-0000000000000000b"}, ids)
|
||||
}
|
||||
|
||||
// The prefix carries its trailing slash, so an instance whose ID extends
|
||||
// another's keeps its objects.
|
||||
func TestDeleteObjects_RemovesOnlyTheOwnersPrefix(t *testing.T) {
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
const other = instanceID + "1"
|
||||
for _, key := range []string{"a", "b", "c"} {
|
||||
putObject(t, objects, uefistore.KeyPrefix(instanceID)+key)
|
||||
}
|
||||
putObject(t, objects, uefistore.KeyPrefix(other)+"a")
|
||||
putObject(t, objects, uefistore.KeyPrefix("ami-0123456789abcdef0")+"a")
|
||||
|
||||
deleted, err := uefistore.DeleteObjects(t.Context(), objects, bucket, instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, deleted)
|
||||
assert.Empty(t, keysUnder(t, objects, uefistore.KeyPrefix(instanceID)))
|
||||
assert.Len(t, keysUnder(t, objects, uefistore.KeyPrefix(other)), 1)
|
||||
assert.Len(t, keysUnder(t, objects, uefistore.KeyPrefix("ami-0123456789abcdef0")), 1)
|
||||
|
||||
deleted, err = uefistore.DeleteObjects(t.Context(), objects, bucket, instanceID)
|
||||
require.NoError(t, err, "an empty prefix is already deleted")
|
||||
assert.Zero(t, deleted)
|
||||
}
|
||||
|
||||
func TestDeleteObjects_RefusesAnIDThatWidensTheDelete(t *testing.T) {
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
putObject(t, objects, uefistore.KeyPrefix(instanceID)+"a")
|
||||
for _, id := range []string{"", "i-", "ami-", "vol-0123", "i-0/../", "i-*", "x"} {
|
||||
_, err := uefistore.DeleteObjects(t.Context(), objects, bucket, id)
|
||||
require.Errorf(t, err, "owner %q", id)
|
||||
}
|
||||
assert.Len(t, keysUnder(t, objects, uefistore.ObjectRoot), 1)
|
||||
}
|
||||
|
||||
// failingDeletes refuses every delete, as predastore does while it is down.
|
||||
type failingDeletes struct {
|
||||
objectstore.ObjectStore
|
||||
}
|
||||
|
||||
func (failingDeletes) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return nil, errors.New("connection refused")
|
||||
}
|
||||
|
||||
func TestDeleteObjects_ReportsAFailedDelete(t *testing.T) {
|
||||
mem := objectstore.NewMemoryObjectStore()
|
||||
putObject(t, mem, uefistore.KeyPrefix(instanceID)+"a")
|
||||
|
||||
_, err := uefistore.DeleteObjects(t.Context(), failingDeletes{mem}, bucket, instanceID)
|
||||
require.Error(t, err)
|
||||
assert.Len(t, keysUnder(t, mem, uefistore.KeyPrefix(instanceID)), 1)
|
||||
}
|
||||
|
||||
// noopDeletes reports success without deleting, so the closing listing is
|
||||
// what notices the prefix is not empty.
|
||||
type noopDeletes struct {
|
||||
objectstore.ObjectStore
|
||||
}
|
||||
|
||||
func (noopDeletes) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func TestDeleteObjects_FailsWhileAnObjectRemains(t *testing.T) {
|
||||
mem := objectstore.NewMemoryObjectStore()
|
||||
putObject(t, mem, uefistore.KeyPrefix(instanceID)+"a")
|
||||
|
||||
_, err := uefistore.DeleteObjects(t.Context(), noopDeletes{mem}, bucket, instanceID)
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
// Package uefistore holds the types that describe an instance's UEFI variable
|
||||
// store and the bucket of store records that carry its deletion and write-back
|
||||
// obligations past the life of the instance record.
|
||||
package uefistore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
const (
|
||||
// BucketName holds one store record per instance, keyed by instance ID.
|
||||
BucketName = "spinifex-uefi-stores"
|
||||
// BucketVersion is the bucket's schema stamp.
|
||||
BucketVersion = 1
|
||||
// RecordSchemaVersion is the version every record is written at.
|
||||
RecordSchemaVersion uint16 = 1
|
||||
|
||||
// StateLive is a store whose objects belong to a running or stopped instance.
|
||||
StateLive = "live"
|
||||
// StateDeleting is a store whose objects are to be deleted, and then the record.
|
||||
StateDeleting = "deleting"
|
||||
|
||||
// ObjectRoot is the object-store prefix every store object is under.
|
||||
ObjectRoot = "uefi/"
|
||||
)
|
||||
|
||||
// ErrInvalidRecord is returned for a store record that breaks its invariants,
|
||||
// whether it is about to be written or was just read.
|
||||
var ErrInvalidRecord = errors.New("uefistore: invalid store record")
|
||||
|
||||
// ErrDeleting is returned for a change that would return a deleting store to
|
||||
// live. Deletion follows only a terminate or an operator's forget, and both are
|
||||
// final, so a writer that meets it drops whatever it was about to record.
|
||||
var ErrDeleting = errors.New("uefistore: store is being deleted")
|
||||
|
||||
// FirmwareIdentity names the firmware a variable store was written under. A
|
||||
// store only boots where Arch and VarsSize match the node's firmware, because
|
||||
// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
|
||||
type FirmwareIdentity struct {
|
||||
Arch string `json:"arch"`
|
||||
VarsSize int64 `json:"vars_size"`
|
||||
CodeSHA256 string `json:"code_sha256"`
|
||||
}
|
||||
|
||||
// UEFIStoreRecord is the durable owner of one instance's variable store
|
||||
// objects. It outlives the instance record, so an obligation recorded here is
|
||||
// not lost when a terminated record expires.
|
||||
type UEFIStoreRecord struct {
|
||||
SchemaVersion uint16 `json:"schema_version"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
AccountID string `json:"account_id"`
|
||||
State string `json:"state"`
|
||||
WriteBack *UEFIWriteBack `json:"write_back,omitempty"`
|
||||
}
|
||||
|
||||
// UEFIWriteBack is a write-back that failed and is still owed. Key is the
|
||||
// object every retry puts to, so retries leave no extra objects; SeededFrom is
|
||||
// the pointer Node seeded from, which a retry must still find on the record.
|
||||
type UEFIWriteBack struct {
|
||||
Node string `json:"node"`
|
||||
Key string `json:"key"`
|
||||
SeededFrom string `json:"seeded_from"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeyPrefix is the object-store prefix every store object of ownerID (an
|
||||
// instance or image ID) is written under.
|
||||
func KeyPrefix(ownerID string) string {
|
||||
return ObjectRoot + ownerID + "/"
|
||||
}
|
||||
|
||||
// ownsKey reports whether key names one object directly under ownerID's prefix.
|
||||
func ownsKey(ownerID, key string) bool {
|
||||
name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
|
||||
return ok && name != "" && !strings.Contains(name, "/")
|
||||
}
|
||||
|
||||
// validate checks rec against key. A write-back key outside the instance's own
|
||||
// prefix is refused because a superseded obligation deletes that key.
|
||||
func (rec *UEFIStoreRecord) validate(key string) error {
|
||||
switch {
|
||||
case rec.SchemaVersion > RecordSchemaVersion:
|
||||
return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
|
||||
ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
|
||||
case rec.SchemaVersion != RecordSchemaVersion:
|
||||
return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
|
||||
case !strings.HasPrefix(rec.InstanceID, "i-"):
|
||||
return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
|
||||
case rec.InstanceID != key:
|
||||
return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
|
||||
case rec.State != StateLive && rec.State != StateDeleting:
|
||||
return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
|
||||
case rec.State == StateDeleting && rec.WriteBack != nil:
|
||||
// A write-back retry puts an object, which would land behind the delete.
|
||||
return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
|
||||
}
|
||||
if wb := rec.WriteBack; wb != nil {
|
||||
switch {
|
||||
case wb.Node == "":
|
||||
return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
|
||||
case wb.Since.IsZero():
|
||||
return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
|
||||
case !ownsKey(rec.InstanceID, wb.Key):
|
||||
return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
|
||||
case !ownsKey(rec.InstanceID, wb.SeededFrom):
|
||||
return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func bucketConfig() kvstore.Config {
|
||||
return kvstore.Config{
|
||||
Name: BucketName,
|
||||
Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
|
||||
History: 1,
|
||||
// Left false: a lost bucket has lost obligations, and an empty
|
||||
// recreated one would report every store as having none.
|
||||
RecreateIfMissing: false,
|
||||
OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
|
||||
return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
|
||||
},
|
||||
Missing: "UEFI store KV bucket not initialized",
|
||||
}
|
||||
}
|
||||
|
||||
// Store reads and writes store records. Every change is guarded by the
|
||||
// revision the caller read, so a lost race is reported rather than retried.
|
||||
type Store struct {
|
||||
kv *kvstore.Store[UEFIStoreRecord]
|
||||
}
|
||||
|
||||
// NewStore returns a Store over js. The bucket is created on Open or first use.
|
||||
func NewStore(js jetstream.JetStream) *Store {
|
||||
return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
|
||||
}
|
||||
|
||||
// NewReader returns a Store over the existing bucket that neither creates it
|
||||
// nor migrates it, for an operator report that must not change the cluster.
|
||||
func NewReader(js jetstream.JetStream) *Store {
|
||||
cfg := bucketConfig()
|
||||
cfg.AttachOnly = true
|
||||
cfg.OnOpen = nil
|
||||
return &Store{kv: kvstore.New[UEFIStoreRecord](js, cfg)}
|
||||
}
|
||||
|
||||
// Open creates the bucket if it is missing and stamps its schema version.
|
||||
func (s *Store) Open(ctx context.Context) error {
|
||||
_, err := s.kv.KV(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns instanceID's record and the revision it was read at. A missing
|
||||
// record is kvstore.ErrNotFound; one that breaks its invariants is
|
||||
// ErrInvalidRecord, so a newer or corrupt record is never acted on.
|
||||
func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
|
||||
rec, rev, err := s.kv.Get(ctx, instanceID)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if err := rec.validate(instanceID); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return rec, rev, nil
|
||||
}
|
||||
|
||||
// Create writes a new record, which must be live: a store is recorded before
|
||||
// its first object is written. An existing record is kvstore.ErrExists. Every
|
||||
// write stamps rec with RecordSchemaVersion.
|
||||
func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
|
||||
rec.SchemaVersion = RecordSchemaVersion
|
||||
if rec.State != StateLive {
|
||||
return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
|
||||
}
|
||||
if err := rec.validate(rec.InstanceID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.kv.Create(ctx, rec.InstanceID, rec)
|
||||
}
|
||||
|
||||
// CompareAndSet replaces the record only if it is still at rev, returning
|
||||
// kvstore.ErrConflict when it is not, and the revision written otherwise. A
|
||||
// record at rev that is deleting cannot be set live again: that is ErrDeleting.
|
||||
func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
|
||||
rec.SchemaVersion = RecordSchemaVersion
|
||||
if err := rec.validate(rec.InstanceID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if rec.State == StateLive {
|
||||
// Read unvalidated: only the state at rev matters, and any other
|
||||
// revision fails the guarded write below anyway.
|
||||
cur, curRev, err := s.kv.Get(ctx, rec.InstanceID)
|
||||
if err != nil && !errors.Is(err, kvstore.ErrNotFound) {
|
||||
return 0, err
|
||||
}
|
||||
if err == nil && curRev == rev && cur.State == StateDeleting {
|
||||
return 0, fmt.Errorf("%w: %s", ErrDeleting, rec.InstanceID)
|
||||
}
|
||||
}
|
||||
return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
|
||||
}
|
||||
|
||||
// CompareAndDelete removes instanceID's record only if it is still at rev,
|
||||
// returning kvstore.ErrConflict when it is not.
|
||||
func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
|
||||
return s.kv.CompareAndDelete(ctx, instanceID, rev)
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package uefistore_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const instanceID = "i-0123456789abcdef0"
|
||||
|
||||
func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
s := uefistore.NewStore(js)
|
||||
require.NoError(t, s.Open(t.Context()))
|
||||
return s, js
|
||||
}
|
||||
|
||||
func liveRecord() *uefistore.UEFIStoreRecord {
|
||||
return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
|
||||
}
|
||||
|
||||
func writeBack() *uefistore.UEFIWriteBack {
|
||||
return &uefistore.UEFIWriteBack{
|
||||
Node: "node-2",
|
||||
Key: uefistore.KeyPrefix(instanceID) + "b",
|
||||
SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
|
||||
Since: time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
|
||||
}
|
||||
}
|
||||
|
||||
func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
|
||||
t.Helper()
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), key, []byte(body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// The records carry obligations past the 1 h terminated-record TTL, so the
|
||||
// bucket must not expire them, and it is stamped so a newer schema is refused.
|
||||
func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
|
||||
_, js := openStore(t)
|
||||
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err)
|
||||
status, err := kv.Status(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(1), status.History())
|
||||
assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
|
||||
|
||||
v, err := kvutil.ReadVersion(t.Context(), kv)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uefistore.BucketVersion, v)
|
||||
}
|
||||
|
||||
func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
|
||||
rev, err := s.Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, gotRev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev)
|
||||
assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
|
||||
assert.Equal(t, rec, got)
|
||||
}
|
||||
|
||||
// The wire names are what every release reads, so they are pinned.
|
||||
func TestRecord_WireShape(t *testing.T) {
|
||||
s, js := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err)
|
||||
entry, err := kv.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{
|
||||
"schema_version": 1,
|
||||
"instance_id": "i-0123456789abcdef0",
|
||||
"account_id": "111122223333",
|
||||
"state": "live",
|
||||
"write_back": {
|
||||
"node": "node-2",
|
||||
"key": "uefi/i-0123456789abcdef0/b",
|
||||
"seeded_from": "uefi/i-0123456789abcdef0/a",
|
||||
"since": "2026-11-03T01:02:03Z"
|
||||
}
|
||||
}`, string(entry.Value()))
|
||||
}
|
||||
|
||||
func TestGet_MissingIsNotFound(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, _, err := s.Get(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound)
|
||||
}
|
||||
|
||||
func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
dup := liveRecord()
|
||||
dup.AccountID = "999999999999"
|
||||
_, err = s.Create(t.Context(), dup)
|
||||
require.ErrorIs(t, err, kvstore.ErrExists)
|
||||
|
||||
got, _, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
|
||||
}
|
||||
|
||||
// A store is recorded live before its first object, so a record that starts
|
||||
// out deleting has no object it could be describing.
|
||||
func TestCreate_RefusesDeleting(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rec := liveRecord()
|
||||
rec.State = uefistore.StateDeleting
|
||||
|
||||
_, err := s.Create(t.Context(), rec)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
_, _, err = s.Get(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound)
|
||||
}
|
||||
|
||||
func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
first, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
moved := liveRecord()
|
||||
moved.WriteBack = writeBack()
|
||||
second, err := s.CompareAndSet(t.Context(), moved, first)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, second, first)
|
||||
|
||||
stale := liveRecord()
|
||||
stale.State = uefistore.StateDeleting
|
||||
_, err = s.CompareAndSet(t.Context(), stale, first)
|
||||
require.ErrorIs(t, err, kvstore.ErrConflict)
|
||||
|
||||
got, rev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, second, rev)
|
||||
assert.Equal(t, uefistore.StateLive, got.State)
|
||||
assert.Equal(t, writeBack(), got.WriteBack)
|
||||
}
|
||||
|
||||
// Two actors that read the same revision race to change it; exactly one may
|
||||
// win, so neither can overwrite an obligation the other recorded.
|
||||
func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
_, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
_, rev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
|
||||
const writers = 8
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
wins []string
|
||||
)
|
||||
for i := range writers {
|
||||
wg.Go(func() {
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
|
||||
if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
|
||||
mu.Lock()
|
||||
wins = append(wins, rec.WriteBack.Node)
|
||||
mu.Unlock()
|
||||
} else {
|
||||
assert.ErrorIs(t, err, kvstore.ErrConflict)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Len(t, wins, 1)
|
||||
got, _, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wins[0], got.WriteBack.Node)
|
||||
}
|
||||
|
||||
func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
first, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
deleting := liveRecord()
|
||||
deleting.State = uefistore.StateDeleting
|
||||
second, err := s.CompareAndSet(t.Context(), deleting, first)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
|
||||
_, _, err = s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
|
||||
|
||||
require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
|
||||
_, _, err = s.Get(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, kvstore.ErrNotFound)
|
||||
}
|
||||
|
||||
// A superseded obligation deletes its recorded key, so a key outside the
|
||||
// instance's own prefix must never reach the bucket.
|
||||
func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
|
||||
cases := map[string]func(*uefistore.UEFIStoreRecord){
|
||||
"unknown state": func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
|
||||
"not an instance ID": func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
|
||||
"write-back with no node": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
|
||||
"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
|
||||
"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
|
||||
},
|
||||
"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
|
||||
},
|
||||
"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
|
||||
},
|
||||
"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
|
||||
},
|
||||
"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
|
||||
r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
|
||||
},
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, _ := openStore(t)
|
||||
rev, err := s.Create(t.Context(), liveRecord())
|
||||
require.NoError(t, err)
|
||||
|
||||
rec := liveRecord()
|
||||
rec.WriteBack = writeBack()
|
||||
mutate(rec)
|
||||
_, err = s.CompareAndSet(t.Context(), rec, rev)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
|
||||
got, gotRev, err := s.Get(t.Context(), instanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev, "an invalid record must not be written")
|
||||
assert.Nil(t, got.WriteBack)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A record a newer release wrote, or one that does not describe its own key,
|
||||
// is reported rather than decoded into something a reaper would act on.
|
||||
func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"newer schema": `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"no schema": `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
|
||||
"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
|
||||
"unknown state": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
|
||||
"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
|
||||
}
|
||||
for name, body := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, js := openStore(t)
|
||||
require.True(t, json.Valid([]byte(body)))
|
||||
rawPut(t, js, instanceID, body)
|
||||
|
||||
_, _, err := s.Get(t.Context(), instanceID)
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type staticInstanceIndex struct {
|
||||
ids map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (s staticInstanceIndex) InstanceIDs(context.Context) (map[string]bool, error) {
|
||||
return s.ids, s.err
|
||||
}
|
||||
|
||||
type uefiOrphanFixture struct {
|
||||
objects *objectstore.MemoryObjectStore
|
||||
metadata *ebsmetadata.Store
|
||||
stores *uefistore.Store
|
||||
index staticInstanceIndex
|
||||
}
|
||||
|
||||
func newUEFIOrphanFixture(t *testing.T) *uefiOrphanFixture {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
stores := uefistore.NewStore(js)
|
||||
require.NoError(t, stores.Open(t.Context()))
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
return &uefiOrphanFixture{
|
||||
objects: objects,
|
||||
metadata: ebsmetadata.NewStore(objects, orphanTestBucket),
|
||||
stores: stores,
|
||||
index: staticInstanceIndex{ids: map[string]bool{}},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *uefiOrphanFixture) putStoreObject(t *testing.T, owner string) {
|
||||
t.Helper()
|
||||
_, err := f.objects.PutObject(t.Context(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(orphanTestBucket),
|
||||
Key: aws.String(uefistore.KeyPrefix(owner) + "0f1e2d3c"),
|
||||
Body: bytes.NewReader([]byte("vars")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (f *uefiOrphanFixture) record(t *testing.T, id string, deleting bool) {
|
||||
t.Helper()
|
||||
_, err := f.stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: id, AccountID: "111122223333", State: uefistore.StateLive})
|
||||
require.NoError(t, err)
|
||||
if deleting {
|
||||
_, err = f.stores.MarkDeleting(t.Context(), id)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *uefiOrphanFixture) scan(t *testing.T) (*OrphanUEFIStores, error) {
|
||||
t.Helper()
|
||||
return FindOrphanUEFIStores(t.Context(), f.objects, orphanTestBucket, f.metadata, f.stores, f.index)
|
||||
}
|
||||
|
||||
// seedEveryCase plants one store of each shape the report must tell apart.
|
||||
func (f *uefiOrphanFixture) seedEveryCase(t *testing.T) {
|
||||
t.Helper()
|
||||
// Owned: an instance record, a store record, and a registered image.
|
||||
f.index.ids["i-00000000000000live"] = true
|
||||
f.record(t, "i-00000000000000live", false)
|
||||
f.putStoreObject(t, "i-00000000000000live")
|
||||
f.record(t, "i-000000000recordonly", false)
|
||||
f.putStoreObject(t, "i-000000000recordonly")
|
||||
f.record(t, "i-0000000000deleting", true)
|
||||
f.putStoreObject(t, "i-0000000000deleting")
|
||||
require.NoError(t, f.metadata.PutAMI(t.Context(), ebsmetadata.AMI{ImageID: "ami-0000000000000live", Name: "img", State: "available"}))
|
||||
f.putStoreObject(t, "ami-0000000000000live")
|
||||
|
||||
// Unowned.
|
||||
f.putStoreObject(t, "i-00000000000noowner")
|
||||
f.putStoreObject(t, "ami-000000000noimage")
|
||||
f.putStoreObject(t, "vol-00000000000other")
|
||||
}
|
||||
|
||||
func TestFindOrphanUEFIStores_ListsEachUnownedShape(t *testing.T) {
|
||||
f := newUEFIOrphanFixture(t)
|
||||
f.seedEveryCase(t)
|
||||
|
||||
orphans, err := f.scan(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []string{"uefi/i-00000000000noowner/"}, orphans.InstancePrefixes,
|
||||
"only a prefix with neither an instance record nor a store record is unowned")
|
||||
assert.Equal(t, []string{"uefi/ami-000000000noimage/"}, orphans.ImagePrefixes)
|
||||
assert.Equal(t, []string{"uefi/vol-00000000000other/"}, orphans.OtherPrefixes)
|
||||
assert.Equal(t, []OrphanUEFIStoreRecord{{InstanceID: "i-000000000recordonly", AccountID: "111122223333", State: uefistore.StateLive}},
|
||||
orphans.Records, "a deleting record is already on its way out, and the reaper owns it")
|
||||
assert.False(t, orphans.Empty())
|
||||
}
|
||||
|
||||
// TestS7_UEFIOrphanReportDeletesNothing enforces ADR-0003:S7 for the report:
|
||||
// an absent instance record is not a terminate, so a scan that lists every
|
||||
// unowned store leaves every object and every record as it found them.
|
||||
func TestS7_UEFIOrphanReportDeletesNothing(t *testing.T) {
|
||||
f := newUEFIOrphanFixture(t)
|
||||
f.seedEveryCase(t)
|
||||
objectsBefore := f.objects.Count()
|
||||
ids, err := f.stores.InstanceIDs(t.Context())
|
||||
require.NoError(t, err)
|
||||
revs := map[string]uint64{}
|
||||
for _, id := range ids {
|
||||
_, rev, err := f.stores.Get(t.Context(), id)
|
||||
require.NoError(t, err)
|
||||
revs[id] = rev
|
||||
}
|
||||
|
||||
_, err = f.scan(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, objectsBefore, f.objects.Count(), "ADR-0003:S7: the orphan report must delete no object")
|
||||
for id, rev := range revs {
|
||||
rec, got, err := f.stores.Get(t.Context(), id)
|
||||
require.NoError(t, err, "ADR-0003:S7: the orphan report removed the store record %s", id)
|
||||
assert.Equalf(t, rev, got, "ADR-0003:S7: the orphan report changed the store record %s", id)
|
||||
assert.NotEmpty(t, rec.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindOrphanUEFIStores_EmptyClusterReportsNothing(t *testing.T) {
|
||||
f := newUEFIOrphanFixture(t)
|
||||
orphans, err := f.scan(t)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, orphans.Empty())
|
||||
}
|
||||
|
||||
// Reading an unreadable record or a failed index as absent would point an
|
||||
// operator at live data, so either aborts the scan.
|
||||
func TestFindOrphanUEFIStores_UnreadableInputsAbortTheScan(t *testing.T) {
|
||||
t.Run("store record", func(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
stores := uefistore.NewStore(js)
|
||||
require.NoError(t, stores.Open(t.Context()))
|
||||
kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err)
|
||||
_, err = kv.Put(t.Context(), "i-0123456789abcdef0", []byte(`{"schema_version":9,"instance_id":"i-0123456789abcdef0","state":"live"}`))
|
||||
require.NoError(t, err)
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
|
||||
_, err = FindOrphanUEFIStores(t.Context(), objects, orphanTestBucket, ebsmetadata.NewStore(objects, orphanTestBucket), stores, staticInstanceIndex{ids: map[string]bool{}})
|
||||
require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
|
||||
})
|
||||
t.Run("instance index", func(t *testing.T) {
|
||||
f := newUEFIOrphanFixture(t)
|
||||
f.index.err = errors.New("kv unavailable")
|
||||
_, err := f.scan(t)
|
||||
require.Error(t, err)
|
||||
})
|
||||
t.Run("AMI document", func(t *testing.T) {
|
||||
f := newUEFIOrphanFixture(t)
|
||||
putRawAMIDocument(t, f.objects, "ami-corrupted000", []byte("{not json"))
|
||||
_, err := f.scan(t)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
@@ -2,12 +2,18 @@ package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
)
|
||||
|
||||
// efiVolumeSuffix marks a volume derived from another one. An EFI variable
|
||||
@@ -115,3 +121,119 @@ func listProviderVolumes(ctx context.Context, provider ebsprovider.EBSProvider)
|
||||
token = resp.NextToken
|
||||
}
|
||||
}
|
||||
|
||||
// UEFIStoreRecords is the read side of the UEFI store-record bucket.
|
||||
type UEFIStoreRecords interface {
|
||||
InstanceIDs(ctx context.Context) ([]string, error)
|
||||
Get(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
|
||||
}
|
||||
|
||||
var _ UEFIStoreRecords = (*uefistore.Store)(nil)
|
||||
|
||||
// InstanceIndex reports every instance ID the control plane holds a record
|
||||
// for, live or terminated. A record it cannot decode still counts.
|
||||
type InstanceIndex interface {
|
||||
InstanceIDs(ctx context.Context) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// OrphanUEFIStoreRecord is a store record that is not deleting and whose
|
||||
// instance has no record, so nothing will ever delete its objects.
|
||||
type OrphanUEFIStoreRecord struct {
|
||||
InstanceID string
|
||||
AccountID string
|
||||
State string
|
||||
}
|
||||
|
||||
// OrphanUEFIStores is what the object store and the store records hold for
|
||||
// UEFI variable stores that nothing owns.
|
||||
type OrphanUEFIStores struct {
|
||||
// InstancePrefixes are uefi/i-*/ prefixes with no instance record and no
|
||||
// store record.
|
||||
InstancePrefixes []string
|
||||
// ImagePrefixes are uefi/ami-*/ prefixes with no AMI document.
|
||||
ImagePrefixes []string
|
||||
// OtherPrefixes are prefixes named for neither an instance nor an image,
|
||||
// which no owner can ever claim.
|
||||
OtherPrefixes []string
|
||||
Records []OrphanUEFIStoreRecord
|
||||
}
|
||||
|
||||
// Empty reports whether the scan found nothing.
|
||||
func (o *OrphanUEFIStores) Empty() bool {
|
||||
return len(o.InstancePrefixes) == 0 && len(o.ImagePrefixes) == 0 && len(o.OtherPrefixes) == 0 && len(o.Records) == 0
|
||||
}
|
||||
|
||||
// FindOrphanUEFIStores reports UEFI variable stores that nothing owns. Like
|
||||
// FindOrphanVolumes it never deletes: an absent instance record is not proof
|
||||
// of a terminate, so settling each one is an operator's decision.
|
||||
//
|
||||
// Every read is strict. A record or document this cannot read aborts the scan
|
||||
// rather than reading as absent, which would point an operator at live data.
|
||||
func FindOrphanUEFIStores(ctx context.Context, objects objectstore.ObjectStore, bucket string,
|
||||
metadata *ebsmetadata.Store, stores UEFIStoreRecords, instances InstanceIndex) (*OrphanUEFIStores, error) {
|
||||
if objects == nil || bucket == "" || metadata == nil || stores == nil || instances == nil {
|
||||
return nil, errors.New("uefi orphan scan: object store, bucket, metadata, store records and instance index are all required")
|
||||
}
|
||||
|
||||
known, err := instances.InstanceIDs(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefi orphan scan: list instance records: %w", err)
|
||||
}
|
||||
images, err := metadata.ListAMIsStrict(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefi orphan scan: list control-plane images: %w", err)
|
||||
}
|
||||
imageIDs := make(map[string]bool, len(images))
|
||||
for _, image := range images {
|
||||
imageIDs[image.ImageID] = true
|
||||
}
|
||||
|
||||
out := &OrphanUEFIStores{}
|
||||
ids, err := stores.InstanceIDs(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefi orphan scan: list store records: %w", err)
|
||||
}
|
||||
recorded := make(map[string]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
recorded[id] = true
|
||||
rec, _, err := stores.Get(ctx, id)
|
||||
if errors.Is(err, kvstore.ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefi orphan scan: read store record %s: %w", id, err)
|
||||
}
|
||||
if rec.State != uefistore.StateDeleting && !known[id] {
|
||||
out.Records = append(out.Records, OrphanUEFIStoreRecord{InstanceID: id, AccountID: rec.AccountID, State: rec.State})
|
||||
}
|
||||
}
|
||||
|
||||
_, prefixes, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(uefistore.ObjectRoot),
|
||||
Delimiter: aws.String("/"),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("uefi orphan scan: list store objects: %w", err)
|
||||
}
|
||||
for _, p := range prefixes {
|
||||
prefix := aws.StringValue(p.Prefix)
|
||||
owner := strings.TrimSuffix(strings.TrimPrefix(prefix, uefistore.ObjectRoot), "/")
|
||||
switch {
|
||||
case strings.HasPrefix(owner, "i-"):
|
||||
if !known[owner] && !recorded[owner] {
|
||||
out.InstancePrefixes = append(out.InstancePrefixes, prefix)
|
||||
}
|
||||
case strings.HasPrefix(owner, "ami-"):
|
||||
if !imageIDs[owner] {
|
||||
out.ImagePrefixes = append(out.ImagePrefixes, prefix)
|
||||
}
|
||||
default:
|
||||
out.OtherPrefixes = append(out.OtherPrefixes, prefix)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.InstancePrefixes)
|
||||
sort.Strings(out.ImagePrefixes)
|
||||
sort.Strings(out.OtherPrefixes)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/host"
|
||||
@@ -45,6 +46,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs/viperblock/dirty"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/formation"
|
||||
hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/dns"
|
||||
@@ -952,6 +954,13 @@ func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
|
||||
fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// The variable stores are reported first, so the volume report's early
|
||||
// return on a clean scan cannot hide them.
|
||||
if err := reportUEFIStoreOrphans(context.Background(), nc, store, node.Predastore.Bucket); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "UEFI store orphan scan failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println()
|
||||
if len(orphans) == 0 {
|
||||
fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
|
||||
return
|
||||
@@ -970,6 +979,53 @@ func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
|
||||
fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
|
||||
}
|
||||
|
||||
// reportUEFIStoreOrphans prints the UEFI variable stores that nothing owns.
|
||||
// It reads only, and attaches to the buckets rather than creating them.
|
||||
func reportUEFIStoreOrphans(ctx context.Context, nc *nats.Conn, store objectstore.ObjectStore, bucket string) error {
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("jetstream: %w", err)
|
||||
}
|
||||
orphans, err := admin.FindOrphanUEFIStores(ctx, store, bucket, ebsmetadata.NewStore(store, bucket),
|
||||
uefistore.NewReader(js), kvInstanceIndex{js: js})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if orphans.Empty() {
|
||||
fmt.Println("No orphaned UEFI variable stores.")
|
||||
return nil
|
||||
}
|
||||
fmt.Println("Orphaned UEFI variable stores:")
|
||||
for _, p := range orphans.InstancePrefixes {
|
||||
fmt.Printf(" %s (no instance record and no store record)\n", p)
|
||||
}
|
||||
for _, p := range orphans.ImagePrefixes {
|
||||
fmt.Printf(" %s (no AMI document)\n", p)
|
||||
}
|
||||
for _, p := range orphans.OtherPrefixes {
|
||||
fmt.Printf(" %s (names neither an instance nor an image)\n", p)
|
||||
}
|
||||
for _, r := range orphans.Records {
|
||||
fmt.Printf(" store record %s (account %s, %s, no instance record)\n", r.InstanceID, r.AccountID, r.State)
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Println("Nothing was deleted. An absent instance record is not proof of a terminate,")
|
||||
fmt.Println("so settling each of these is an operator decision.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// kvInstanceIndex is the instance domain's index of every instance ID it holds
|
||||
// a record for.
|
||||
type kvInstanceIndex struct {
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
var _ admin.InstanceIndex = kvInstanceIndex{}
|
||||
|
||||
func (k kvInstanceIndex) InstanceIDs(ctx context.Context) (map[string]bool, error) {
|
||||
return daemon.KnownInstanceIDs(ctx, k.js)
|
||||
}
|
||||
|
||||
func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
|
||||
imageID, _ := cmd.Flags().GetString("image-id")
|
||||
force, _ := cmd.Flags().GetBool("force")
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
)
|
||||
|
||||
func putKeys(t *testing.T, js jetstream.JetStream, bucket string, keys ...string) {
|
||||
t.Helper()
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket})
|
||||
require.NoError(t, err)
|
||||
for _, key := range keys {
|
||||
_, err := kv.Put(t.Context(), key, []byte("not decoded"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Every key space an instance record has lived in counts, and the node
|
||||
// blobs and presence markers name nodes rather than instances.
|
||||
func TestKVInstanceIndex_CountsEveryInstanceKeySpace(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
putKeys(t, js, daemon.InstanceStateBucket, "i.i-running", "instance.i-frozenstopped", "node.node-1", "nodepresence.node-1")
|
||||
putKeys(t, js, daemon.TerminatedInstanceBucket, "i.i-terminated", "terminated.i-frozenterminated")
|
||||
|
||||
ids, err := kvInstanceIndex{js: js}.InstanceIDs(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]bool{
|
||||
"i-running": true, "i-frozenstopped": true, "i-terminated": true, "i-frozenterminated": true,
|
||||
}, ids)
|
||||
}
|
||||
|
||||
// The report attaches to the buckets rather than creating them, so a cluster
|
||||
// missing one fails the scan instead of reporting every store as unowned.
|
||||
func TestReportUEFIStoreOrphans_FailsWhenABucketIsMissing(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
|
||||
require.Error(t, reportUEFIStoreOrphans(t.Context(), nc, objects, "spinifex"))
|
||||
}
|
||||
|
||||
func TestReportUEFIStoreOrphans_PrintsAndDeletesNothing(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
putKeys(t, js, daemon.InstanceStateBucket)
|
||||
putKeys(t, js, daemon.TerminatedInstanceBucket)
|
||||
stores := uefistore.NewStore(js)
|
||||
require.NoError(t, stores.Open(t.Context()))
|
||||
_, err := stores.Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: "i-0000000000recorded", AccountID: "111122223333", State: uefistore.StateLive})
|
||||
require.NoError(t, err)
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
for _, owner := range []string{"i-00000000000noowner", "ami-000000000noimage", "vol-00000000000other"} {
|
||||
_, err := objects.PutObject(t.Context(), &s3.PutObjectInput{
|
||||
Bucket: aws.String("spinifex"), Key: aws.String(uefistore.KeyPrefix(owner) + "a"), Body: bytes.NewReader([]byte("vars")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
out := captureStdout(t, func() {
|
||||
require.NoError(t, reportUEFIStoreOrphans(t.Context(), nc, objects, "spinifex"))
|
||||
})
|
||||
|
||||
assert.Contains(t, out, "uefi/i-00000000000noowner/")
|
||||
assert.Contains(t, out, "uefi/ami-000000000noimage/")
|
||||
assert.Contains(t, out, "uefi/vol-00000000000other/")
|
||||
assert.Contains(t, out, "store record i-0000000000recorded")
|
||||
assert.Contains(t, out, "Nothing was deleted")
|
||||
assert.Equal(t, 3, objects.Count())
|
||||
}
|
||||
|
||||
func TestReportUEFIStoreOrphans_CleanCluster(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
putKeys(t, js, daemon.InstanceStateBucket)
|
||||
putKeys(t, js, daemon.TerminatedInstanceBucket)
|
||||
require.NoError(t, uefistore.NewStore(js).Open(t.Context()))
|
||||
|
||||
out := captureStdout(t, func() {
|
||||
require.NoError(t, reportUEFIStoreOrphans(t.Context(), nc, objectstore.NewMemoryObjectStore(), "spinifex"))
|
||||
})
|
||||
assert.Contains(t, out, "No orphaned UEFI variable stores.")
|
||||
}
|
||||
@@ -23,6 +23,7 @@ Single source of truth for the current schema version of every config file Spini
|
||||
| `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
|
||||
| `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
|
||||
| `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
|
||||
| `spinifex-uefi-stores` | `1` | `uefistore.BucketVersion` |
|
||||
|
||||
## Registered migrations
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ type Deps struct {
|
||||
Resources ResourceController
|
||||
VolumeStateUpdater VolumeStateUpdater
|
||||
InstanceCleaner InstanceCleaner
|
||||
UEFIStores UEFIStoreTerminator
|
||||
Hooks ManagerHooks
|
||||
|
||||
// ShutdownSignal returns true once the daemon has begun coordinated
|
||||
|
||||
@@ -106,6 +106,10 @@ type InstanceStatus struct {
|
||||
EBSRequests []EBSRequest `json:"ebs_requests,omitempty"`
|
||||
ENIAvailableSlots []int `json:"eni_available_slots,omitempty"`
|
||||
ENIAttachedByID map[string]int `json:"eni_attached_by_id,omitempty"`
|
||||
|
||||
// UEFIVarStore is optional, so records written without it still decode
|
||||
// and the bucket schema does not change.
|
||||
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
|
||||
}
|
||||
|
||||
// Record projects a VM onto the persisted record. QMPClient and the request
|
||||
@@ -166,6 +170,7 @@ func (v *VM) Record() *InstanceRecord {
|
||||
EBSRequests: v.EBSRequests.Requests,
|
||||
ENIAvailableSlots: v.ENIRequests.AvailableSlots,
|
||||
ENIAttachedByID: v.ENIRequests.AttachedByENIID,
|
||||
UEFIVarStore: v.UEFIVarStore,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -222,6 +227,7 @@ func VMFromRecord(r *InstanceRecord) *VM {
|
||||
TerminatedAt: r.Status.TerminatedAt,
|
||||
ShuttingDownAt: r.Status.ShuttingDownAt,
|
||||
FirstLaunch: r.Status.FirstLaunch,
|
||||
UEFIVarStore: r.Status.UEFIVarStore,
|
||||
}
|
||||
v.EBSRequests.Requests = r.Status.EBSRequests
|
||||
v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
@@ -114,6 +115,37 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
|
||||
assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
|
||||
}
|
||||
|
||||
// The pointer is optional: a record written before it existed decodes with
|
||||
// none, and a record without one writes no key, so the bucket schema holds.
|
||||
func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
|
||||
var decoded vm.InstanceRecord
|
||||
require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
|
||||
assert.Nil(t, decoded.Status.UEFIVarStore)
|
||||
|
||||
encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(encoded), "uefi_var_store")
|
||||
}
|
||||
|
||||
// The pointer's wire names are what the record carries across releases, so
|
||||
// they are pinned rather than left to follow the Go field names.
|
||||
func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
|
||||
encoded, err := json.Marshal(populatedVM().Record())
|
||||
require.NoError(t, err)
|
||||
|
||||
var wire struct {
|
||||
Status struct {
|
||||
UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
|
||||
} `json:"status"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(encoded, &wire))
|
||||
ref := wire.Status.UEFIVarStore
|
||||
assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
|
||||
assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
|
||||
assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
|
||||
assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
|
||||
}
|
||||
|
||||
// The mutexes are dropped on the way out, so the rebuilt instance must arrive
|
||||
// with usable zero-value locks rather than ones somebody else left held.
|
||||
func TestVMFromRecord_RequestWrappersAreUsable(t *testing.T) {
|
||||
@@ -206,6 +238,12 @@ func populatedVM() *vm.VM {
|
||||
CapacityReservationId: "cr-1",
|
||||
InstanceLifecycle: "spot",
|
||||
SpotInstanceRequestId: "sir-1",
|
||||
UEFIVarStore: &vm.UEFIVarStoreRef{
|
||||
Key: "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
|
||||
Firmware: uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
|
||||
WriterNode: "node-3",
|
||||
WrittenAt: time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
|
||||
},
|
||||
|
||||
Status: vm.StateRunning,
|
||||
Instance: &ec2.Instance{InstanceId: aws.String("i-round-trip")},
|
||||
|
||||
@@ -433,6 +433,7 @@ func (m *Manager) finalizeTerminated(instance *VM) error {
|
||||
// already spent and terminated is terminal.
|
||||
func (m *Manager) reconcileVanishedQEMU(instance *VM) error {
|
||||
m.markTeardown(instance, TeardownQEMU, TeardownDone)
|
||||
m.markUEFIStoreForDeletion(instance)
|
||||
m.stampOutstandingTeardownFailed(instance)
|
||||
return m.finalizeTerminated(instance)
|
||||
}
|
||||
@@ -460,6 +461,7 @@ func (m *Manager) forceFinalizeStuckTerminate(instance *VM) error {
|
||||
if m.deps.InstanceCleaner != nil {
|
||||
m.markTeardownResult(instance, TeardownVolumes, m.deps.InstanceCleaner.DeleteVolumes(instance))
|
||||
}
|
||||
m.markUEFIStoreForDeletion(instance)
|
||||
m.stampOutstandingTeardownFailed(instance)
|
||||
return m.finalizeTerminated(instance)
|
||||
}
|
||||
@@ -548,8 +550,8 @@ func (m *Manager) releaseAutoAssignedPublicIP(instance *VM) {
|
||||
}
|
||||
|
||||
// terminateCleanup is stopCleanup plus the AWS-resource cleanup that
|
||||
// only applies on terminate: volume deletion, public IP release, ENI
|
||||
// deletion, placement-group removal.
|
||||
// only applies on terminate: volume deletion, the UEFI store's deletion
|
||||
// record, public IP release, ENI deletion, placement-group removal.
|
||||
func (m *Manager) terminateCleanup(instance *VM) {
|
||||
// Terminate deletes the volumes next, so an unsealed block map loses
|
||||
// nothing a caller can still ask for. Tolerated to keep terminate
|
||||
@@ -560,6 +562,7 @@ func (m *Manager) terminateCleanup(instance *VM) {
|
||||
if m.deps.InstanceCleaner != nil {
|
||||
m.markTeardownResult(instance, TeardownVolumes, m.deps.InstanceCleaner.DeleteVolumes(instance))
|
||||
}
|
||||
m.markUEFIStoreForDeletion(instance)
|
||||
|
||||
m.cleanupTapDevices(instance)
|
||||
m.markTeardown(instance, TeardownTap, TeardownDone)
|
||||
|
||||
@@ -23,6 +23,9 @@ const (
|
||||
TeardownENI = "eni"
|
||||
TeardownOVN = "ovn"
|
||||
TeardownPlacement = "placement"
|
||||
// TeardownUEFI is pending once the variable store is recorded for
|
||||
// deletion; the cluster-wide store reaper marks it done.
|
||||
TeardownUEFI = "uefi"
|
||||
)
|
||||
|
||||
// markTeardown records dep → state on the VM under the manager lock. Inspect
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package vm
|
||||
|
||||
import "log/slog"
|
||||
|
||||
// UEFIStoreTerminator settles a terminated instance's UEFI variable store. The
|
||||
// objects are deleted by a cluster-wide reaper, so terminate only records the
|
||||
// obligation and removes this node's local copy.
|
||||
type UEFIStoreTerminator interface {
|
||||
// MarkUEFIStoreDeleting moves the instance's store record to deleting. It
|
||||
// reports false with no error when the instance has no store.
|
||||
MarkUEFIStoreDeleting(instanceID string) (bool, error)
|
||||
// RemoveLocalUEFIStore removes this node's copy of the store. Idempotent.
|
||||
RemoveLocalUEFIStore(instanceID string) error
|
||||
}
|
||||
|
||||
// markUEFIStoreForDeletion records a terminating instance's store for the
|
||||
// reaper. A failed mark is stamped failed, which the reaper completes from the
|
||||
// terminated record; an instance with no store gets no teardown entry.
|
||||
func (m *Manager) markUEFIStoreForDeletion(instance *VM) {
|
||||
if m.deps.UEFIStores == nil {
|
||||
return
|
||||
}
|
||||
has, err := m.deps.UEFIStores.MarkUEFIStoreDeleting(instance.ID)
|
||||
switch {
|
||||
case err != nil:
|
||||
slog.Warn("Failed to record UEFI variable store for deletion; the store reaper completes it",
|
||||
"instanceId", instance.ID, "err", err)
|
||||
m.markTeardown(instance, TeardownUEFI, TeardownFailed)
|
||||
case has:
|
||||
m.markTeardown(instance, TeardownUEFI, TeardownPending)
|
||||
}
|
||||
if err := m.deps.UEFIStores.RemoveLocalUEFIStore(instance.ID); err != nil {
|
||||
slog.Warn("Failed to remove local UEFI variable store", "instanceId", instance.ID, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeUEFIStores records each terminator call; has and markErr shape the mark.
|
||||
type fakeUEFIStores struct {
|
||||
mu sync.Mutex
|
||||
has bool
|
||||
markErr error
|
||||
marked []string
|
||||
removed []string
|
||||
}
|
||||
|
||||
var _ UEFIStoreTerminator = (*fakeUEFIStores)(nil)
|
||||
|
||||
func (f *fakeUEFIStores) MarkUEFIStoreDeleting(id string) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.marked = append(f.marked, id)
|
||||
return f.has && f.markErr == nil, f.markErr
|
||||
}
|
||||
|
||||
func (f *fakeUEFIStores) RemoveLocalUEFIStore(id string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.removed = append(f.removed, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeUEFIStores) calls() (marked, removed []string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.marked...), append([]string(nil), f.removed...)
|
||||
}
|
||||
|
||||
func withUEFIStores(m *Manager, f *fakeUEFIStores) {
|
||||
deps := m.deps
|
||||
deps.UEFIStores = f
|
||||
m.SetDeps(deps)
|
||||
}
|
||||
|
||||
func runningUEFIInstance(id string) *VM {
|
||||
return &VM{ID: id, Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
|
||||
}
|
||||
|
||||
// TestS8_TerminateRecordsTheUEFIStoreObligationAndReturns enforces ADR-0003:S8
|
||||
// for the store: terminate deletes no object, it records the obligation and
|
||||
// returns, and the terminated record carries it for the reaper to converge.
|
||||
func TestS8_TerminateRecordsTheUEFIStoreObligationAndReturns(t *testing.T) {
|
||||
store := newFakeStateStore()
|
||||
m, _, _, _, _ := terminateTestManager(t, store)
|
||||
stores := &fakeUEFIStores{has: true}
|
||||
withUEFIStores(m, stores)
|
||||
v := runningUEFIInstance("i-uefi-terminate")
|
||||
m.Insert(v)
|
||||
|
||||
require.NoError(t, m.Terminate(v.ID))
|
||||
|
||||
marked, removed := stores.calls()
|
||||
assert.Equal(t, []string{v.ID}, marked, "terminate must record the store for deletion")
|
||||
assert.Equal(t, []string{v.ID}, removed, "the node that ran the instance removes its local copy")
|
||||
terminated := store.terminated[v.ID]
|
||||
require.NotNil(t, terminated, "terminate must write the terminated record")
|
||||
assert.Equal(t, string(TeardownPending), terminated.Teardown[TeardownUEFI],
|
||||
"ADR-0003:S8: the terminated record must carry the store obligation while the reaper converges it")
|
||||
assert.False(t, terminated.TeardownComplete(), "a pending store keeps the record from early purge")
|
||||
}
|
||||
|
||||
func TestTerminate_InstanceWithNoUEFIStoreStampsNoEntry(t *testing.T) {
|
||||
store := newFakeStateStore()
|
||||
m, _, _, _, _ := terminateTestManager(t, store)
|
||||
withUEFIStores(m, &fakeUEFIStores{has: false})
|
||||
v := runningUEFIInstance("i-bios")
|
||||
m.Insert(v)
|
||||
|
||||
require.NoError(t, m.Terminate(v.ID))
|
||||
|
||||
_, stamped := store.terminated[v.ID].Teardown[TeardownUEFI]
|
||||
assert.False(t, stamped, "a BIOS instance has no store, so nothing is owed")
|
||||
}
|
||||
|
||||
// A mark that fails is stamped failed rather than dropped, so the reaper can
|
||||
// complete it from the terminated record.
|
||||
func TestTerminate_FailedUEFIMarkIsStampedFailed(t *testing.T) {
|
||||
store := newFakeStateStore()
|
||||
m, _, _, _, _ := terminateTestManager(t, store)
|
||||
withUEFIStores(m, &fakeUEFIStores{has: true, markErr: errors.New("kv unavailable")})
|
||||
v := runningUEFIInstance("i-uefi-mark-fails")
|
||||
m.Insert(v)
|
||||
|
||||
require.NoError(t, m.Terminate(v.ID), "a store mark never fails the terminate")
|
||||
|
||||
assert.Equal(t, string(TeardownFailed), store.terminated[v.ID].Teardown[TeardownUEFI])
|
||||
}
|
||||
|
||||
// A terminate that wedged before its cleanup still records the store when the
|
||||
// backstop finalises it.
|
||||
func TestReconcileVanishedQEMU_RecordsTheUEFIStoreForDeletion(t *testing.T) {
|
||||
store := newFakeStateStore()
|
||||
m, _, _, _, _ := terminateTestManager(t, store)
|
||||
stores := &fakeUEFIStores{has: true}
|
||||
withUEFIStores(m, stores)
|
||||
v := runningUEFIInstance("i-uefi-wedged")
|
||||
v.Status = StateShuttingDown
|
||||
m.Insert(v)
|
||||
|
||||
require.NoError(t, m.reconcileVanishedQEMU(v))
|
||||
|
||||
marked, _ := stores.calls()
|
||||
assert.Equal(t, []string{v.ID}, marked)
|
||||
assert.Equal(t, string(TeardownPending), store.terminated[v.ID].Teardown[TeardownUEFI])
|
||||
}
|
||||
|
||||
// TestS7_OnlyTerminateRecordsTheUEFIStoreForDeletion enforces ADR-0003:S7 at
|
||||
// the node: a stop, a node-wide stop and a failed recovery of an instance
|
||||
// that has run never move its store toward deletion or remove its local copy.
|
||||
func TestS7_OnlyTerminateRecordsTheUEFIStoreForDeletion(t *testing.T) {
|
||||
m, _, _, _, rt := shutdownTestManager(t)
|
||||
stores := &fakeUEFIStores{has: true}
|
||||
withUEFIStores(m, stores)
|
||||
|
||||
stopped := runningUEFIInstance("i-uefi-stop")
|
||||
m.Insert(stopped)
|
||||
require.NoError(t, m.Stop(stopped.ID))
|
||||
|
||||
drained := runningUEFIInstance("i-uefi-drain")
|
||||
m.Insert(drained)
|
||||
require.NoError(t, m.StopAll())
|
||||
|
||||
failed := &VM{ID: "i-uefi-has-run", Status: StatePending, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
|
||||
m.Insert(failed)
|
||||
errored := rt.waitFor(failed.ID, StateError)
|
||||
m.MarkFailed(t.Context(), failed, "relaunch_timeout")
|
||||
select {
|
||||
case <-errored:
|
||||
case <-time.After(markFailedDeadline):
|
||||
t.Fatalf("MarkFailed on an instance that has run did not reach error within %s", markFailedDeadline)
|
||||
}
|
||||
m.goroutineWg.Wait()
|
||||
|
||||
marked, removed := stores.calls()
|
||||
assert.Empty(t, marked, "ADR-0003:S7: only a decided terminate may record a store for deletion")
|
||||
assert.Empty(t, removed, "ADR-0003:S7: a stop or failure keeps the node's copy of the store")
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
@@ -308,6 +309,20 @@ type VM struct {
|
||||
// tolerated by migrate.
|
||||
InstanceLifecycle string `json:"instance_lifecycle,omitempty"`
|
||||
SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
|
||||
|
||||
// UEFIVarStore points at the object holding this instance's UEFI variable
|
||||
// store. Status, so written only by the node; nil for a BIOS instance.
|
||||
UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
|
||||
}
|
||||
|
||||
// UEFIVarStoreRef names the store object an instance launches from, with the
|
||||
// firmware it was written under, so a launch can check it without reading the
|
||||
// object's metadata.
|
||||
type UEFIVarStoreRef struct {
|
||||
Key string `json:"key"`
|
||||
Firmware uefistore.FirmwareIdentity `json:"firmware"`
|
||||
WriterNode string `json:"writer_node"`
|
||||
WrittenAt time.Time `json:"written_at"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON folds the fields of legacy records that mixed a request with an
|
||||
|
||||
@@ -2211,6 +2211,9 @@ func (d *Daemon) startCluster() error {
|
||||
}
|
||||
|
||||
d.instanceService.SetTerminationDeps(d.volumeService, d.vpcService, ipReleaser, d.tagsService)
|
||||
if d.jsManager != nil && d.jsManager.UEFIStores() != nil {
|
||||
d.instanceService.SetUEFIStores(d.jsManager.UEFIStores())
|
||||
}
|
||||
d.instanceService.SetVolumeCreator(d.volumeService)
|
||||
d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, ipAllocator)
|
||||
|
||||
@@ -2581,6 +2584,9 @@ func (d *Daemon) startCluster() error {
|
||||
if gpuRec := d.newGPUPoolReconciler(); gpuRec != nil {
|
||||
reapers = append(reapers, gpuRec)
|
||||
}
|
||||
if uefiStores := d.newUEFIStoreReaper(); uefiStores != nil {
|
||||
reapers = append(reapers, uefiStores)
|
||||
}
|
||||
if d.volumeService != nil {
|
||||
reapers = append(reapers, d.volumeService.NewVolumeLeakReaper(d.leakedVolumeInstances))
|
||||
reapers = append(reapers, d.volumeService.NewVolumeReclaimReaper())
|
||||
@@ -2754,6 +2760,10 @@ func (d *Daemon) initJetStream() error {
|
||||
err = d.jsManager.InitClusterConfigBucket()
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = d.jsManager.InitUEFIStoreBucket()
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
d.jsManager.SetSyncObserver(d)
|
||||
// Replicas cannot be undeclared here: bucket creation refuses
|
||||
|
||||
@@ -3,10 +3,13 @@ package daemon
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// InstanceRecordPrefix is the key prefix for the per-resource instance record
|
||||
@@ -215,3 +218,99 @@ func loadRecord[T any](store *kvstore.Store[T], key string) (*T, error) {
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// TerminateDecided reports whether instanceID is terminated or terminating, or
|
||||
// has no instance record at all, read from the KV store at call time. A read
|
||||
// error is returned rather than guessed, so a partition never reads as absent.
|
||||
func (m *JetStreamManager) TerminateDecided(instanceID string) (bool, error) {
|
||||
terminated, err := m.LoadTerminatedInstance(instanceID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if terminated != nil {
|
||||
return true, nil
|
||||
}
|
||||
record, err := m.LoadInstanceRecord(instanceID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if record == nil {
|
||||
return true, nil
|
||||
}
|
||||
switch record.Status.Status {
|
||||
case vm.StateShuttingDown, vm.StateTerminated:
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// OpenTeardowns lists the terminated instances whose dependency entry dep is
|
||||
// recorded and not yet done. A record that has expired is not listed, and a
|
||||
// failed listing is returned whole rather than as a partial set.
|
||||
func (m *JetStreamManager) OpenTeardowns(dep string) ([]string, error) {
|
||||
terminated, err := m.ListTerminatedInstances()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for _, v := range terminated {
|
||||
state, ok := v.Teardown[dep]
|
||||
if ok && vm.TeardownState(state) != vm.TeardownDone && v.Status == vm.StateTerminated {
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// MarkTeardownDone sets the terminated record's dependency entry dep done. An
|
||||
// absent or expired record is not an error: there is no entry left to close.
|
||||
func (m *JetStreamManager) MarkTeardownDone(instanceID, dep string) error {
|
||||
terminated, err := m.LoadTerminatedInstance(instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if terminated == nil || vm.TeardownState(terminated.Teardown[dep]) == vm.TeardownDone {
|
||||
return nil
|
||||
}
|
||||
_, err = m.UpdateTerminatedInstance(instanceID, func(v *vm.VM) {
|
||||
if v.Teardown == nil {
|
||||
v.Teardown = make(map[string]string)
|
||||
}
|
||||
v.Teardown[dep] = string(vm.TeardownDone)
|
||||
})
|
||||
if errors.Is(err, kvstore.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// KnownInstanceIDs lists every instance ID the instance-state and terminated
|
||||
// buckets hold a record for, in any key space, by key alone so an undecodable
|
||||
// record still counts. It attaches to the buckets and fails if one is missing.
|
||||
func KnownInstanceIDs(ctx context.Context, js jetstream.JetStream) (map[string]bool, error) {
|
||||
ids := make(map[string]bool)
|
||||
buckets := map[string][]string{
|
||||
InstanceStateBucket: {InstanceRecordPrefix, StoppedInstancePrefix},
|
||||
TerminatedInstanceBucket: {InstanceRecordPrefix, TerminatedInstancePrefix},
|
||||
}
|
||||
for name, prefixes := range buckets {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open %s: %w", name, err)
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s: %w", name, err)
|
||||
}
|
||||
for key := range lister.Keys() {
|
||||
for _, prefix := range prefixes {
|
||||
if id, ok := strings.CutPrefix(key, prefix); ok {
|
||||
ids[id] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
// The lister closes itself once drained, so Stop only reports that.
|
||||
_ = lister.Stop()
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
|
||||
@@ -88,6 +89,7 @@ type JetStreamManager struct {
|
||||
termRecords *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
|
||||
clusterKV jetstream.KeyValue // spinifex-cluster-state
|
||||
clusterConfig *kvstore.Bucket // spinifex-cluster-config
|
||||
uefiStores *uefistore.Store // spinifex-uefi-stores
|
||||
obs KVSyncObserver
|
||||
running runningSetState
|
||||
}
|
||||
@@ -232,6 +234,18 @@ func (m *JetStreamManager) InitClusterConfigBucket() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// InitUEFIStoreBucket opens the UEFI store-record bucket, creating it if it
|
||||
// doesn't exist, so a bucket that cannot be created fails startup.
|
||||
func (m *JetStreamManager) InitUEFIStoreBucket() error {
|
||||
m.uefiStores = uefistore.NewStore(m.js)
|
||||
return m.uefiStores.Open(context.Background())
|
||||
}
|
||||
|
||||
// UEFIStores returns the store-record bucket opened by InitUEFIStoreBucket.
|
||||
func (m *JetStreamManager) UEFIStores() *uefistore.Store {
|
||||
return m.uefiStores
|
||||
}
|
||||
|
||||
// InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
|
||||
// JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
|
||||
func (m *JetStreamManager) InitTerminatedInstanceBucket() error {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package daemon_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The daemon creates the store-record bucket at startup and hands out the
|
||||
// store it opened, so later readers share one handle.
|
||||
func TestInitUEFIStoreBucket_CreatesBucketAndServesTheStore(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, m.InitUEFIStoreBucket())
|
||||
|
||||
_, err = js.KeyValue(t.Context(), uefistore.BucketName)
|
||||
require.NoError(t, err, "init must create the bucket rather than leave it to first use")
|
||||
|
||||
rec := &uefistore.UEFIStoreRecord{InstanceID: "i-0123456789abcdef0", State: uefistore.StateLive}
|
||||
rev, err := m.UEFIStores().Create(t.Context(), rec)
|
||||
require.NoError(t, err)
|
||||
got, gotRev, err := m.UEFIStores().Get(t.Context(), rec.InstanceID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, gotRev)
|
||||
assert.Equal(t, rec, got)
|
||||
|
||||
_, err = m.UEFIStores().Create(t.Context(), rec)
|
||||
require.ErrorIs(t, err, kvstore.ErrExists)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
// uefiStoreKVTimeout bounds the one record read and write a terminate makes.
|
||||
const uefiStoreKVTimeout = 10 * time.Second
|
||||
|
||||
// uefiLocalDirName is the directory under the daemon's data directory that
|
||||
// holds each instance's local copy of its variable store.
|
||||
const uefiLocalDirName = "uefi"
|
||||
|
||||
// uefiLocalDir is the directory holding instanceID's local variable store.
|
||||
func uefiLocalDir(dataDir, instanceID string) string {
|
||||
if dataDir == "" {
|
||||
dataDir = DefaultDataDir
|
||||
}
|
||||
return filepath.Join(dataDir, uefiLocalDirName, instanceID)
|
||||
}
|
||||
|
||||
// uefiStoreTerminator satisfies vm.UEFIStoreTerminator over the store-record
|
||||
// bucket and this node's data directory.
|
||||
type uefiStoreTerminator struct {
|
||||
stores *uefistore.Store
|
||||
dataDir string
|
||||
}
|
||||
|
||||
var _ vm.UEFIStoreTerminator = (*uefiStoreTerminator)(nil)
|
||||
|
||||
// newUEFIStoreTerminator returns nil, not a typed nil, when the bucket is not
|
||||
// open, so the manager sees no terminator rather than one that panics.
|
||||
func (d *Daemon) newUEFIStoreTerminator() vm.UEFIStoreTerminator {
|
||||
if d.jsManager == nil || d.jsManager.UEFIStores() == nil {
|
||||
return nil
|
||||
}
|
||||
return &uefiStoreTerminator{stores: d.jsManager.UEFIStores(), dataDir: d.config.DataDir}
|
||||
}
|
||||
|
||||
func (t *uefiStoreTerminator) MarkUEFIStoreDeleting(instanceID string) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), uefiStoreKVTimeout)
|
||||
defer cancel()
|
||||
return t.stores.MarkDeleting(ctx, instanceID)
|
||||
}
|
||||
|
||||
// RemoveLocalUEFIStore refuses anything but a bare instance ID, so a malformed
|
||||
// ID cannot point the removal outside the instance's own directory.
|
||||
func (t *uefiStoreTerminator) RemoveLocalUEFIStore(instanceID string) error {
|
||||
if !strings.HasPrefix(instanceID, "i-") || strings.ContainsAny(instanceID, `/\`) || strings.Contains(instanceID, "..") {
|
||||
return fmt.Errorf("refusing to remove a local UEFI store for %q", instanceID)
|
||||
}
|
||||
return os.RemoveAll(uefiLocalDir(t.dataDir, instanceID))
|
||||
}
|
||||
|
||||
// uefiInstanceRecords is what the store reaper asks of the instance domain:
|
||||
// whether a terminate is decided, and the terminated records' uefi entries.
|
||||
type uefiInstanceRecords interface {
|
||||
TerminateDecided(instanceID string) (bool, error)
|
||||
OpenTeardowns(dep string) ([]string, error)
|
||||
MarkTeardownDone(instanceID, dep string) error
|
||||
}
|
||||
|
||||
var _ uefiInstanceRecords = (*JetStreamManager)(nil)
|
||||
|
||||
// uefiStoreReaper deletes the variable stores whose record is deleting: each
|
||||
// object under the instance's prefix, then the record. Terminate, or an
|
||||
// operator's forget, sets deleting and returns, so this decides nothing.
|
||||
//
|
||||
// It also completes a terminate whose mark failed, from the terminated
|
||||
// record's pending or failed uefi entry. That record is the terminate itself,
|
||||
// so moving the store record to deleting there is still not the reaper's call.
|
||||
//
|
||||
// Cluster-wide: the store outlives the node that ran the instance, and one
|
||||
// sweeper keeps two deletes off one prefix.
|
||||
type uefiStoreReaper struct {
|
||||
stores *uefistore.Store
|
||||
objects objectstore.ObjectStore
|
||||
bucket string
|
||||
instances uefiInstanceRecords
|
||||
}
|
||||
|
||||
var _ vm.Reaper = (*uefiStoreReaper)(nil)
|
||||
|
||||
// newUEFIStoreReaper builds the reaper, or nil when this node has no store
|
||||
// bucket or object store to reap with.
|
||||
func (d *Daemon) newUEFIStoreReaper() *uefiStoreReaper {
|
||||
if d.jsManager == nil || d.jsManager.UEFIStores() == nil || d.config == nil || d.config.Predastore.Bucket == "" {
|
||||
return nil
|
||||
}
|
||||
return &uefiStoreReaper{
|
||||
stores: d.jsManager.UEFIStores(),
|
||||
objects: objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(d.config.Predastore.Host),
|
||||
d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
|
||||
bucket: d.config.Predastore.Bucket,
|
||||
instances: d.jsManager,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *uefiStoreReaper) Class() string { return "uefi-store" }
|
||||
func (r *uefiStoreReaper) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
|
||||
|
||||
// Sweep deletes every deleting store it can and reports how many records it
|
||||
// retired. A store whose delete fails keeps its record, which has no TTL, and
|
||||
// is retried next sweep however long predastore is away.
|
||||
func (r *uefiStoreReaper) Sweep(ctx context.Context) (int, error) {
|
||||
r.completeTerminatedMarks(ctx)
|
||||
|
||||
ids, err := r.stores.InstanceIDs(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list UEFI store records: %w", err)
|
||||
}
|
||||
reaped := 0
|
||||
for _, id := range ids {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return reaped, err
|
||||
}
|
||||
if r.reap(ctx, id) {
|
||||
reaped++
|
||||
}
|
||||
}
|
||||
return reaped, nil
|
||||
}
|
||||
|
||||
// reap deletes one store if its record is deleting, and reports whether the
|
||||
// record was retired.
|
||||
func (r *uefiStoreReaper) reap(ctx context.Context, id string) bool {
|
||||
rec, rev, err := r.stores.Get(ctx, id)
|
||||
if err != nil {
|
||||
if !errors.Is(err, kvstore.ErrNotFound) {
|
||||
slog.WarnContext(ctx, "uefi-store: unreadable store record, leaving it", "instanceId", id, "err", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if rec.State != uefistore.StateDeleting {
|
||||
return false
|
||||
}
|
||||
|
||||
ok, err := r.instances.TerminateDecided(id)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: cannot read the instance record, retrying next sweep", "instanceId", id, "err", err)
|
||||
return false
|
||||
}
|
||||
if !ok {
|
||||
// Deleting is only ever set by a decided terminate or a forget, so a
|
||||
// live instance here is a defect; its store is kept for an operator.
|
||||
slog.ErrorContext(ctx, "uefi-store: refusing to delete the store of an instance that is not terminating",
|
||||
"instanceId", id, "accountId", rec.AccountID)
|
||||
return false
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
deleted, err := uefistore.DeleteObjects(ctx, r.objects, r.bucket, id)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: delete incomplete, retrying next sweep",
|
||||
"instanceId", id, "deleted", deleted, "err", err)
|
||||
return false
|
||||
}
|
||||
if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record, retrying next sweep", "instanceId", id, "err", err)
|
||||
return false
|
||||
}
|
||||
// Removed last: while it exists the obligation is readable, so a sweep
|
||||
// interrupted above resumes rather than leaving objects nobody owns.
|
||||
if err := r.stores.CompareAndDelete(ctx, id, rev); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to remove the store record, retrying next sweep", "instanceId", id, "err", err)
|
||||
return false
|
||||
}
|
||||
slog.InfoContext(ctx, "uefi-store: store deleted", "instanceId", id, "accountId", rec.AccountID,
|
||||
"objects", deleted, "elapsed_ms", otelsetup.Millis(time.Since(start)))
|
||||
return true
|
||||
}
|
||||
|
||||
// completeTerminatedMarks finishes the mark of each terminate that recorded a
|
||||
// uefi entry but did not land its store record in deleting.
|
||||
func (r *uefiStoreReaper) completeTerminatedMarks(ctx context.Context) {
|
||||
ids, err := r.instances.OpenTeardowns(vm.TeardownUEFI)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: cannot list terminated instances", "err", err)
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
rec, _, err := r.stores.Get(ctx, id)
|
||||
switch {
|
||||
case errors.Is(err, kvstore.ErrNotFound):
|
||||
// Nothing left to delete: the store is gone or never existed.
|
||||
if err := r.instances.MarkTeardownDone(id, vm.TeardownUEFI); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to mark the terminated record", "instanceId", id, "err", err)
|
||||
}
|
||||
case err != nil:
|
||||
slog.WarnContext(ctx, "uefi-store: unreadable store record", "instanceId", id, "err", err)
|
||||
case rec.State == uefistore.StateLive:
|
||||
if _, err := r.stores.MarkDeleting(ctx, id); err != nil {
|
||||
slog.WarnContext(ctx, "uefi-store: failed to record a terminated instance's store for deletion",
|
||||
"instanceId", id, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
//test:in-package — the reaper and the terminator adapter are unexported daemon types
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const uefiTestBucket = "spinifex"
|
||||
|
||||
type uefiReaperFixture struct {
|
||||
m *JetStreamManager
|
||||
objects *objectstore.MemoryObjectStore
|
||||
r *uefiStoreReaper
|
||||
}
|
||||
|
||||
func newUEFIReaperFixture(t *testing.T) *uefiReaperFixture {
|
||||
t.Helper()
|
||||
m := newRecordManagerInPackage(t)
|
||||
require.NoError(t, m.InitUEFIStoreBucket())
|
||||
objects := objectstore.NewMemoryObjectStore()
|
||||
return &uefiReaperFixture{
|
||||
m: m,
|
||||
objects: objects,
|
||||
r: &uefiStoreReaper{stores: m.UEFIStores(), objects: objects, bucket: uefiTestBucket, instances: m},
|
||||
}
|
||||
}
|
||||
|
||||
// seedStore writes a live store record and n objects under its prefix, the
|
||||
// state a launch leaves, then marks it deleting if asked, as terminate does.
|
||||
func (f *uefiReaperFixture) seedStore(t *testing.T, id string, n int, deleting bool) {
|
||||
t.Helper()
|
||||
_, err := f.m.UEFIStores().Create(t.Context(), &uefistore.UEFIStoreRecord{InstanceID: id, AccountID: "111122223333", State: uefistore.StateLive})
|
||||
require.NoError(t, err)
|
||||
for i := range n {
|
||||
_, err := f.objects.PutObject(t.Context(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(uefiTestBucket),
|
||||
Key: aws.String(uefistore.KeyPrefix(id) + string(rune('a'+i))),
|
||||
Body: bytes.NewReader([]byte("vars")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
if deleting {
|
||||
_, err := f.m.UEFIStores().MarkDeleting(t.Context(), id)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *uefiReaperFixture) writeInstance(t *testing.T, id string, status vm.InstanceState) {
|
||||
t.Helper()
|
||||
v := &vm.VM{ID: id, Status: status, AccountID: "111122223333", Instance: &ec2.Instance{InstanceId: aws.String(id)}}
|
||||
require.NoError(t, f.m.WriteInstanceRecord(id, v.Record()))
|
||||
}
|
||||
|
||||
func (f *uefiReaperFixture) writeTerminated(t *testing.T, id string, uefi vm.TeardownState) {
|
||||
t.Helper()
|
||||
v := &vm.VM{ID: id, Status: vm.StateTerminated, AccountID: "111122223333", Instance: &ec2.Instance{InstanceId: aws.String(id)},
|
||||
Teardown: map[string]string{vm.TeardownUEFI: string(uefi)}}
|
||||
require.NoError(t, f.m.WriteTerminatedInstance(id, v))
|
||||
}
|
||||
|
||||
func (f *uefiReaperFixture) objectCount(t *testing.T, id string) int {
|
||||
t.Helper()
|
||||
keys, err := uefistore.ListObjects(t.Context(), f.objects, uefiTestBucket, id)
|
||||
require.NoError(t, err)
|
||||
return len(keys)
|
||||
}
|
||||
|
||||
func (f *uefiReaperFixture) recordState(t *testing.T, id string) string {
|
||||
t.Helper()
|
||||
rec, _, err := f.m.UEFIStores().Get(t.Context(), id)
|
||||
if errors.Is(err, kvstore.ErrNotFound) {
|
||||
return "absent"
|
||||
}
|
||||
require.NoError(t, err)
|
||||
return rec.State
|
||||
}
|
||||
|
||||
func (f *uefiReaperFixture) terminatedUEFIMark(t *testing.T, id string) string {
|
||||
t.Helper()
|
||||
v, err := f.m.LoadTerminatedInstance(id)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, v)
|
||||
return v.Teardown[vm.TeardownUEFI]
|
||||
}
|
||||
|
||||
// TestS7_UEFIStoreReaperDeletesOnlyAStoreATerminateMarkedDeleting enforces
|
||||
// ADR-0003:S7 for the store reaper: it deletes the objects and record of a
|
||||
// store a terminate set deleting, and never a live store, whether its
|
||||
// instance runs, is stopped, or has no record the reaper could judge it by.
|
||||
func TestS7_UEFIStoreReaperDeletesOnlyAStoreATerminateMarkedDeleting(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const (
|
||||
running = "i-0000000000000running"
|
||||
stopped = "i-0000000000000stopped"
|
||||
unowned = "i-0000000000000unowned"
|
||||
terminated = "i-000000000terminated"
|
||||
terminating = "i-00000000terminating"
|
||||
)
|
||||
f.writeInstance(t, running, vm.StateRunning)
|
||||
f.seedStore(t, running, 3, false)
|
||||
f.writeInstance(t, stopped, vm.StateStopped)
|
||||
f.seedStore(t, stopped, 3, false)
|
||||
f.seedStore(t, unowned, 3, false)
|
||||
f.writeTerminated(t, terminated, vm.TeardownPending)
|
||||
f.seedStore(t, terminated, 3, true)
|
||||
f.writeInstance(t, terminating, vm.StateShuttingDown)
|
||||
f.seedStore(t, terminating, 2, true)
|
||||
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, reaped)
|
||||
|
||||
for _, id := range []string{running, stopped, unowned} {
|
||||
assert.Equalf(t, 3, f.objectCount(t, id), "ADR-0003:S7: the reaper deleted objects of the live store %s", id)
|
||||
assert.Equalf(t, uefistore.StateLive, f.recordState(t, id), "ADR-0003:S7: the reaper changed the live store record %s", id)
|
||||
}
|
||||
for _, id := range []string{terminated, terminating} {
|
||||
assert.Zerof(t, f.objectCount(t, id), "a store a terminate marked deleting must be deleted: %s", id)
|
||||
assert.Equalf(t, "absent", f.recordState(t, id), "the record goes once its prefix is gone: %s", id)
|
||||
}
|
||||
assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, terminated))
|
||||
}
|
||||
|
||||
// TestS7_UEFIStoreReaperRefusesADeletingStoreOfALiveInstance enforces
|
||||
// ADR-0003:S7 when the record and the instance disagree: a deleting record
|
||||
// whose instance still runs or is stopped is kept, objects and all.
|
||||
func TestS7_UEFIStoreReaperRefusesADeletingStoreOfALiveInstance(t *testing.T) {
|
||||
for _, status := range []vm.InstanceState{vm.StateRunning, vm.StateStopped, vm.StatePending} {
|
||||
t.Run(string(status), func(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.writeInstance(t, id, status)
|
||||
f.seedStore(t, id, 3, true)
|
||||
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reaped)
|
||||
assert.Equal(t, 3, f.objectCount(t, id), "ADR-0003:S7: a live instance's store must survive a deleting record")
|
||||
assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// failingObjects stands in for predastore while it is down: every list and
|
||||
// delete fails.
|
||||
type failingObjects struct {
|
||||
objectstore.ObjectStore
|
||||
}
|
||||
|
||||
func (failingObjects) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return nil, errors.New("predastore: connection refused")
|
||||
}
|
||||
|
||||
func (failingObjects) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return nil, errors.New("predastore: connection refused")
|
||||
}
|
||||
|
||||
// The obligation lives in the store record, which has no TTL, so a terminate
|
||||
// still completes when predastore is down for longer than the terminated
|
||||
// record lives.
|
||||
func TestUEFIStoreReaper_CompletesAfterTheTerminatedRecordExpires(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.writeTerminated(t, id, vm.TeardownPending)
|
||||
f.seedStore(t, id, 3, true)
|
||||
|
||||
f.r.objects = failingObjects{f.objects}
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, reaped)
|
||||
assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id), "a failed delete keeps the obligation")
|
||||
assert.Equal(t, string(vm.TeardownPending), f.terminatedUEFIMark(t, id))
|
||||
|
||||
// The bucket's TTL expires the terminated record while predastore is away.
|
||||
require.NoError(t, f.m.DeleteTerminatedInstance(id))
|
||||
f.r.objects = f.objects
|
||||
|
||||
reaped, err = f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
assert.Zero(t, f.objectCount(t, id))
|
||||
assert.Equal(t, "absent", f.recordState(t, id))
|
||||
}
|
||||
|
||||
// A terminate whose mark failed left the store live and the terminated
|
||||
// record's entry failed; the reaper completes that terminate.
|
||||
func TestUEFIStoreReaper_CompletesATerminateWhoseMarkFailed(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.seedStore(t, id, 3, false)
|
||||
f.writeTerminated(t, id, vm.TeardownFailed)
|
||||
|
||||
reaped, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reaped)
|
||||
assert.Zero(t, f.objectCount(t, id))
|
||||
assert.Equal(t, "absent", f.recordState(t, id))
|
||||
assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, id))
|
||||
}
|
||||
|
||||
// A pending entry whose store record is already gone has nothing left to
|
||||
// delete, so the entry is closed and the terminated record can be purged.
|
||||
func TestUEFIStoreReaper_ClosesAPendingEntryWithNoStore(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.writeTerminated(t, id, vm.TeardownPending)
|
||||
|
||||
_, err := f.r.Sweep(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(vm.TeardownDone), f.terminatedUEFIMark(t, id))
|
||||
}
|
||||
|
||||
func TestUEFIStoreTerminator_MarksTheRecordAndRemovesOnlyTheInstanceDirectory(t *testing.T) {
|
||||
f := newUEFIReaperFixture(t)
|
||||
const id = "i-0000000000000000a"
|
||||
f.seedStore(t, id, 1, false)
|
||||
dataDir := t.TempDir()
|
||||
mine := uefiLocalDir(dataDir, id)
|
||||
other := uefiLocalDir(dataDir, "i-0000000000000000b")
|
||||
for _, dir := range []string{mine, other} {
|
||||
require.NoError(t, os.MkdirAll(dir, 0o750))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "vars.fd"), []byte("vars"), 0o600))
|
||||
}
|
||||
term := &uefiStoreTerminator{stores: f.m.UEFIStores(), dataDir: dataDir}
|
||||
|
||||
has, err := term.MarkUEFIStoreDeleting(id)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has)
|
||||
assert.Equal(t, uefistore.StateDeleting, f.recordState(t, id))
|
||||
|
||||
require.NoError(t, term.RemoveLocalUEFIStore(id))
|
||||
assert.NoDirExists(t, mine)
|
||||
assert.DirExists(t, other)
|
||||
require.NoError(t, term.RemoveLocalUEFIStore(id), "removal is idempotent")
|
||||
|
||||
for _, bad := range []string{"", "..", "i-0/../..", "i-0000000000000000b/vars.fd", "vol-0123"} {
|
||||
require.Errorf(t, term.RemoveLocalUEFIStore(bad), "%q", bad)
|
||||
}
|
||||
assert.FileExists(t, filepath.Join(other, "vars.fd"))
|
||||
}
|
||||
@@ -679,6 +679,7 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
|
||||
Resources: newResourceControllerAdapter(d.resourceMgr),
|
||||
VolumeStateUpdater: d.volumeService,
|
||||
InstanceCleaner: newInstanceCleanerAdapter(d),
|
||||
UEFIStores: d.newUEFIStoreTerminator(),
|
||||
Hooks: vm.ManagerHooks{
|
||||
OnInstanceUp: d.onInstanceUpHook(),
|
||||
OnInstanceDown: d.onInstanceDownHook(),
|
||||
|
||||
Reference in New Issue
Block a user