mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-04 10:13:43 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f5e09923c1 |
@@ -144,7 +144,7 @@ type VolumeDeleter interface {
|
||||
// terminate without deleting it, matching AWS semantics for a
|
||||
// DeleteOnTermination=false volume: terminate still implies detach, it
|
||||
// just leaves the volume behind as available rather than deleting it.
|
||||
DetachVolumeOnTerminate(ctx context.Context, volumeID, instanceID, accountID string) error
|
||||
DetachVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error
|
||||
}
|
||||
|
||||
// VolumeCreator creates the volumes a launch names beyond the root device, and
|
||||
|
||||
@@ -3002,7 +3002,7 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
|
||||
continue
|
||||
}
|
||||
// A volume already gone is the detach's own goal, not a failure.
|
||||
if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instanceID, instance.AccountID); err != nil &&
|
||||
if err := s.volumeDeleter.DetachVolumeOnTerminate(ctx, ebsRequest.Name, instance.AccountID); err != nil &&
|
||||
!awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
|
||||
slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to detach volume", "name", ebsRequest.Name, "err", err)
|
||||
lastErr = err
|
||||
|
||||
@@ -1761,7 +1761,7 @@ func (f *fakeVolumeDeleter) DeleteVolumeOnTerminate(_ context.Context, volumeID,
|
||||
// DetachVolumeOnTerminate mirrors DeleteVolumeOnTerminate's call/error
|
||||
// bookkeeping but records into detached instead of deleted; the non-DoT
|
||||
// Boot-volume branch calls this instead of deleting.
|
||||
func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _, _ string) error {
|
||||
func (f *fakeVolumeDeleter) DetachVolumeOnTerminate(_ context.Context, volumeID, _ string) error {
|
||||
f.calls = append(f.calls, volumeID)
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
|
||||
@@ -995,23 +995,8 @@ func (s *VolumeServiceImpl) UpdateVolumeState(accountID, volumeID, state, attach
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModifyVolume modifies an EBS volume (grow-only, refused while attached).
|
||||
// ModifyVolume modifies an EBS volume (grow-only, requires stopped instance).
|
||||
func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error) {
|
||||
return s.modifyVolume(ctx, input, accountID, "")
|
||||
}
|
||||
|
||||
// ExpandAttachedToStopped grows a volume still attached to instanceID, which the
|
||||
// caller has stopped and holds exclusively. Not reachable from the AWS API.
|
||||
func (s *VolumeServiceImpl) ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
|
||||
if instanceID == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
return s.modifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(volumeID), Size: aws.Int64(sizeGiB)}, accountID, instanceID)
|
||||
}
|
||||
|
||||
// modifyVolume grows a volume. An empty stoppedOwner refuses any attached
|
||||
// volume; otherwise only a volume attached to that instance is accepted.
|
||||
func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID, stoppedOwner string) (*ec2.ModifyVolumeOutput, error) {
|
||||
if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
|
||||
}
|
||||
@@ -1031,9 +1016,7 @@ func (s *VolumeServiceImpl) modifyVolume(ctx context.Context, input *ec2.ModifyV
|
||||
if input.Size == nil || *input.Size <= originalSize {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
attached := meta.AttachedInstance != "" && meta.State == "in-use"
|
||||
if meta.State == string(ebsprovider.VolumeStateCreating) || attached != (stoppedOwner != "") ||
|
||||
(attached && meta.AttachedInstance != stoppedOwner) {
|
||||
if (meta.AttachedInstance != "" && meta.State == "in-use") || meta.State == string(ebsprovider.VolumeStateCreating) {
|
||||
return nil, errors.New(awserrors.ErrorIncorrectState)
|
||||
}
|
||||
originalType := meta.VolumeType
|
||||
@@ -1126,24 +1109,7 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
|
||||
// available rather than deleting it. Metadata-only, no QMP — same rationale
|
||||
// as DeleteVolumeOnTerminate: there is no live QEMU to hot-unplug on either
|
||||
// terminate path.
|
||||
//
|
||||
// A volume since attached to another instance is left alone, so a delayed
|
||||
// teardown retry never clears a newer attachment.
|
||||
func (s *VolumeServiceImpl) DetachVolumeOnTerminate(ctx context.Context, volumeID, instanceID, accountID string) error {
|
||||
if s.metadata != nil && instanceID != "" {
|
||||
meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
|
||||
if err != nil {
|
||||
if objectstore.IsNoSuchKeyError(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if meta.AttachedInstance != "" && meta.AttachedInstance != instanceID {
|
||||
slog.InfoContext(ctx, "DetachVolumeOnTerminate: volume is attached elsewhere; leaving it",
|
||||
"volumeId", volumeID, "terminated", instanceID, "attachedTo", meta.AttachedInstance)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, accountID string) error {
|
||||
if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
|
||||
// No metadata doc means the volume is already gone, per GetVolume's own
|
||||
// contract, so absence here is success, not a teardown failure.
|
||||
|
||||
@@ -1468,60 +1468,6 @@ func TestModifyVolume_AttachedInUse(t *testing.T) {
|
||||
assert.Equal(t, awserrors.ErrorIncorrectState, err.Error())
|
||||
}
|
||||
|
||||
func TestExpandAttachedToStopped_GrowsTheNamedInstancesVolume(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
createVolumeInStoreWithMeta(t, svc, store, "vol-stopped", ebsmetadata.Volume{
|
||||
VolumeID: "vol-stopped",
|
||||
CapacityGiB: 20,
|
||||
State: "in-use",
|
||||
AttachedInstance: "i-owner",
|
||||
DeviceName: "/dev/sdf",
|
||||
})
|
||||
|
||||
out, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-stopped", "i-owner", 30)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(30), aws.Int64Value(out.VolumeModification.TargetSize))
|
||||
|
||||
meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-stopped")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(30), meta.CapacityGiB)
|
||||
assert.Equal(t, "in-use", meta.State, "the grow must not change the attachment")
|
||||
assert.Equal(t, "i-owner", meta.AttachedInstance)
|
||||
assert.Equal(t, "/dev/sdf", meta.DeviceName)
|
||||
}
|
||||
|
||||
func TestExpandAttachedToStopped_Refusals(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
meta ebsmetadata.Volume
|
||||
instance string
|
||||
want string
|
||||
}{
|
||||
{"attached to another instance", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-other"}, "i-owner", awserrors.ErrorIncorrectState},
|
||||
{"not attached", ebsmetadata.Volume{State: "available"}, "i-owner", awserrors.ErrorIncorrectState},
|
||||
{"still creating", ebsmetadata.Volume{State: string(ebsprovider.VolumeStateCreating)}, "i-owner", awserrors.ErrorIncorrectState},
|
||||
{"no instance named", ebsmetadata.Volume{State: "in-use", AttachedInstance: "i-owner"}, "", awserrors.ErrorMissingParameter},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
tc.meta.VolumeID = "vol-x"
|
||||
tc.meta.CapacityGiB = 20
|
||||
createVolumeInStoreWithMeta(t, svc, store, "vol-x", tc.meta)
|
||||
|
||||
_, err := svc.ExpandAttachedToStopped(context.Background(), testVolAccountID, "vol-x", tc.instance, 30)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tc.want, err.Error())
|
||||
|
||||
meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-x")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(20), meta.CapacityGiB, "a refused grow must leave the size unchanged")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModifyVolume_SuccessfulGrow(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
@@ -1883,36 +1829,10 @@ func TestDetachVolumeOnTerminate_AlreadyGoneMetadataIsSuccess(t *testing.T) {
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
svc.snapshotKV = kv
|
||||
|
||||
err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", "i-gone", testVolAccountID)
|
||||
err := svc.DetachVolumeOnTerminate(context.Background(), "vol-already-gone", testVolAccountID)
|
||||
require.NoError(t, err, "a volume with no metadata document is already gone, not a teardown failure")
|
||||
}
|
||||
|
||||
// A teardown retried after the volume moved on must not clear the attachment it
|
||||
// now has, while the terminated instance's own attachment is still cleared.
|
||||
func TestDetachVolumeOnTerminate_LeavesAVolumeAttachedElsewhere(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
createVolumeInStoreWithMeta(t, svc, store, "vol-moved", ebsmetadata.Volume{
|
||||
VolumeID: "vol-moved", CapacityGiB: 20, State: "in-use", AttachedInstance: "i-new", DeviceName: "/dev/sdf",
|
||||
})
|
||||
createVolumeInStoreWithMeta(t, svc, store, "vol-own", ebsmetadata.Volume{
|
||||
VolumeID: "vol-own", CapacityGiB: 20, State: "in-use", AttachedInstance: "i-old", DeviceName: "/dev/sdf",
|
||||
})
|
||||
|
||||
require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-moved", "i-old", testVolAccountID))
|
||||
require.NoError(t, svc.DetachVolumeOnTerminate(context.Background(), "vol-own", "i-old", testVolAccountID))
|
||||
|
||||
moved, err := svc.GetVolumeMetadata(testVolAccountID, "vol-moved")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "in-use", moved.State)
|
||||
assert.Equal(t, "i-new", moved.AttachedInstance)
|
||||
|
||||
own, err := svc.GetVolumeMetadata(testVolAccountID, "vol-own")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "available", own.State)
|
||||
assert.Empty(t, own.AttachedInstance)
|
||||
}
|
||||
|
||||
// failingDeleteProvider makes the provider's delete fail while leaving every
|
||||
// other operation intact, which is the shape of a blob node that is down.
|
||||
type failingDeleteProvider struct {
|
||||
|
||||
@@ -19,10 +19,6 @@ import (
|
||||
|
||||
var errFinalSnapshotInProgress = errors.New("rds: final DB snapshot creation is still in progress")
|
||||
|
||||
// errModifyInProgress defers a teardown until a modify that already holds the
|
||||
// lease, and may be growing the data volume, has let go of it.
|
||||
var errModifyInProgress = errors.New("rds: a modification of the DB instance is still in progress")
|
||||
|
||||
type finalSnapshotReservation struct {
|
||||
creator bool
|
||||
revision uint64
|
||||
@@ -97,10 +93,6 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
|
||||
}
|
||||
|
||||
if err := s.teardownDBInstance(ctx, kv, accountID, rec, reservation.creator); err != nil {
|
||||
// The reconciler finishes the delete once the modify lets go.
|
||||
if errors.Is(err, errModifyInProgress) {
|
||||
return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -185,14 +177,6 @@ func (s *Service) checkFinalSnapshotAvailable(ctx context.Context, kv *kvstore.B
|
||||
// any point it stopped at.
|
||||
func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, accountID string,
|
||||
rec *DBInstanceRecord, finalSnapshotCreator bool) error {
|
||||
current, _, err := s.getDBInstance(ctx, kv, rec.DBInstanceIdentifier)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.ModifyLease.live() {
|
||||
return errModifyInProgress
|
||||
}
|
||||
|
||||
// Asked to stop cleanly even though the VM is about to go: it is what makes
|
||||
// the final snapshot a clean checkpoint rather than one needing WAL replay.
|
||||
if rec.FinalSnapshotIdentifier != "" {
|
||||
@@ -202,9 +186,6 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
|
||||
if err := s.terminateInstanceVM(ctx, rec.InstanceID); err != nil {
|
||||
return fmt.Errorf("rds: terminate the VM behind %s: %w", rec.DBInstanceIdentifier, err)
|
||||
}
|
||||
if err := s.requireDataVolumeDetached(ctx, rec.DataVolumeID, rec.InstanceID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.takeFinalSnapshot(ctx, kv, accountID, rec, finalSnapshotCreator); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -243,8 +224,8 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
|
||||
return nil
|
||||
}
|
||||
|
||||
// A VM no node owns is either already gone or stopped, and a stopped one is
|
||||
// terminated from its shared record, which also detaches the data volume.
|
||||
// A VM no node owns is already terminated, which is the state this is trying to
|
||||
// reach.
|
||||
func (s *Service) terminateInstanceVM(ctx context.Context, instanceID string) error {
|
||||
if instanceID == "" {
|
||||
return nil
|
||||
@@ -253,53 +234,10 @@ func (s *Service) terminateInstanceVM(ctx context.Context, instanceID string) er
|
||||
if launcher == nil {
|
||||
return errors.New("rds: no system-instance launcher configured")
|
||||
}
|
||||
err := launcher.TerminateSystemInstance(instanceID)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, systeminstance.ErrSystemInstanceNotFound) {
|
||||
if err := launcher.TerminateSystemInstance(instanceID); err != nil &&
|
||||
!errors.Is(err, systeminstance.ErrSystemInstanceNotFound) {
|
||||
return err
|
||||
}
|
||||
if s.deps.Instances == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.deps.Instances.TerminateStoppedInstance(ctx, instanceID); err != nil {
|
||||
return fmt.Errorf("terminate the stopped VM %s: %w", instanceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A stopped-VM terminate records a failed detach without failing, and a volume
|
||||
// left attached would be retained as if a snapshot held it. The VM is gone by
|
||||
// now, so the detach is repeated here, on every retry, until it lands.
|
||||
func (s *Service) requireDataVolumeDetached(ctx context.Context, volumeID, instanceID string) error {
|
||||
if volumeID == "" || instanceID == "" || s.deps.Storage == nil {
|
||||
return nil
|
||||
}
|
||||
out, err := s.deps.Storage.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{
|
||||
VolumeIds: aws.StringSlice([]string{volumeID}),
|
||||
}, awsidentifiers.GlobalAccountID)
|
||||
if err != nil {
|
||||
if awserrors.IsErrorCode(err, awserrors.ErrorInvalidVolumeNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("rds: read the data volume %s: %w", volumeID, err)
|
||||
}
|
||||
for _, volume := range out.Volumes {
|
||||
if aws.StringValue(volume.VolumeId) != volumeID || aws.StringValue(volume.State) != ec2.VolumeStateInUse {
|
||||
continue
|
||||
}
|
||||
for _, a := range volume.Attachments {
|
||||
if aws.StringValue(a.InstanceId) != instanceID {
|
||||
continue
|
||||
}
|
||||
if err := s.deps.Storage.DetachVolumeOnTerminate(ctx, volumeID, instanceID, awsidentifiers.GlobalAccountID); err != nil {
|
||||
return fmt.Errorf("rds: detach the data volume %s from the terminated VM %s: %w", volumeID, instanceID, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "rds: detached the data volume the VM's terminate left attached",
|
||||
"volumeId", volumeID, "instanceId", instanceID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/rds"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -380,74 +379,3 @@ func TestReconciler_ResumesAnInterruptedStop(t *testing.T) {
|
||||
assert.Equal(t, []string{"stop:" + testInstance}, h.cmdr.calls)
|
||||
assert.Equal(t, StatusStopped, h.record(t).Status)
|
||||
}
|
||||
|
||||
// A stopped DB VM is held by no node, so the system terminate finds nothing; it
|
||||
// is terminated from its shared record instead, which releases the data volume.
|
||||
func TestDeleteDBInstance_TerminatesAStoppedVMFromItsRecord(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
|
||||
h.cmdr.onTerminateStopped = func(string) { delete(h.storage.attachedTo, "vol-rdsdata01") }
|
||||
seedInstance(t, h.svc, availableRecord())
|
||||
|
||||
_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Contains(t, h.cmdr.calls, "terminate-stopped:"+testInstance)
|
||||
assert.Equal(t, []string{"vol-rdsdata01"}, h.volumes.deleted)
|
||||
assert.False(t, h.recordExists(t, testDBID))
|
||||
}
|
||||
|
||||
// The stopped terminate reports success even when its detach failed, and a
|
||||
// retried teardown finds the record gone, so the teardown repeats the detach.
|
||||
func TestDeleteDBInstance_DetachesWhatTheStoppedTerminateLeftAttached(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
|
||||
seedInstance(t, h.svc, availableRecord())
|
||||
|
||||
_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []string{"vol-rdsdata01:" + testInstance}, h.storage.detached)
|
||||
assert.Equal(t, []string{"vol-rdsdata01"}, h.volumes.deleted)
|
||||
assert.False(t, h.recordExists(t, testDBID))
|
||||
}
|
||||
|
||||
// A detach that keeps failing must not let the release read the volume as
|
||||
// snapshot-held: the teardown stops with the record kept, and is retried.
|
||||
func TestDeleteDBInstance_StopsWhileTheDataVolumeCannotBeDetached(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.storage.attachedTo = map[string]string{"vol-rdsdata01": testInstance}
|
||||
h.storage.detachErr = errors.New("the volume store is unavailable")
|
||||
seedInstance(t, h.svc, availableRecord())
|
||||
|
||||
_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "detach the data volume")
|
||||
|
||||
assert.Empty(t, h.volumes.deleted)
|
||||
_, retained := h.retainedVolume(t, "vol-rdsdata01")
|
||||
assert.False(t, retained, "an attached volume must not be recorded as retained")
|
||||
assert.True(t, h.recordExists(t, testDBID), "the record stays so the teardown can be resumed")
|
||||
}
|
||||
|
||||
// A stopped VM that refuses the terminate is not gone, so the teardown must not
|
||||
// carry on as though it were.
|
||||
func TestDeleteDBInstance_ReturnsAStoppedTerminateRefusal(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
h.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.cmdr.terminateStoppedErr = errors.New(awserrors.ErrorIncorrectInstanceState)
|
||||
seedInstance(t, h.svc, availableRecord())
|
||||
|
||||
_, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorIncorrectInstanceState)
|
||||
assert.Empty(t, h.volumes.deleted)
|
||||
assert.True(t, h.recordExists(t, testDBID))
|
||||
}
|
||||
|
||||
@@ -34,9 +34,6 @@ type instanceCommander interface {
|
||||
// Relaunches a VM whose owning node no longer holds it, from its persisted
|
||||
// stopped-instance record.
|
||||
StartStoppedInstance(ctx context.Context, instanceID string) error
|
||||
// Terminates a VM no node holds from its stopped-instance record. A VM
|
||||
// already gone is not an error.
|
||||
TerminateStoppedInstance(ctx context.Context, instanceID string) error
|
||||
}
|
||||
|
||||
// ErrInstanceNotOnNode means no node subscribes ec2.cmd.{instanceID} for this VM,
|
||||
@@ -457,13 +454,3 @@ func (c *natsInstanceCommander) StartStoppedInstance(ctx context.Context, instan
|
||||
ec2instance.StartStoppedInstanceInput{InstanceID: instanceID}, c.timeout, awsidentifiers.GlobalAccountID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *natsInstanceCommander) TerminateStoppedInstance(ctx context.Context, instanceID string) error {
|
||||
_, err := natsmsg.NATSRequest[ec2instance.TerminateStoppedInstanceOutput](ctx, c.nc, "ec2.terminate",
|
||||
ec2instance.TerminateStoppedInstanceInput{InstanceID: instanceID}, c.timeout, awsidentifiers.GlobalAccountID)
|
||||
if err == nil || errors.Is(err, nats.ErrNoResponders) ||
|
||||
strings.Contains(err.Error(), awserrors.ErrorInvalidInstanceIDNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -16,10 +16,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/service/rds"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -55,10 +52,6 @@ type fakeInstanceCommander struct {
|
||||
// The VM state an accepted stop takes down. Nil models a node that accepts
|
||||
// the command and never lands it.
|
||||
vm *fakeInstanceState
|
||||
// terminateStoppedErr is what terminating a stopped VM returns; onTerminateStopped
|
||||
// runs first, so a test can model what the terminate does to the data volume.
|
||||
terminateStoppedErr error
|
||||
onTerminateStopped func(instanceID string)
|
||||
}
|
||||
|
||||
var _ instanceCommander = (*fakeInstanceCommander)(nil)
|
||||
@@ -98,14 +91,6 @@ func (f *fakeInstanceCommander) StartStoppedInstance(_ context.Context, instance
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeInstanceCommander) TerminateStoppedInstance(_ context.Context, instanceID string) error {
|
||||
f.calls = append(f.calls, "terminate-stopped:"+instanceID)
|
||||
if f.onTerminateStopped != nil {
|
||||
f.onTerminateStopped(instanceID)
|
||||
}
|
||||
return f.terminateStoppedErr
|
||||
}
|
||||
|
||||
// fakeSnapshots stands in for the EC2 snapshot service. holding is what
|
||||
// DescribeSnapshots reports against the data volume, which is what decides
|
||||
// between deleting the volume and retaining it.
|
||||
@@ -280,7 +265,6 @@ type lifecycleHarness struct {
|
||||
sysvpc *fakeSystemVPC
|
||||
launcher *fakeLauncher
|
||||
volumes *fakeVolumes
|
||||
storage *fakeVolumeResizer
|
||||
agent *stubAgent
|
||||
vmState *fakeInstanceState
|
||||
}
|
||||
@@ -302,7 +286,6 @@ func newLifecycleHarness(t *testing.T, agentFails bool) *lifecycleHarness {
|
||||
sysvpc: &fakeSystemVPC{},
|
||||
launcher: &fakeLauncher{},
|
||||
volumes: &fakeVolumes{},
|
||||
storage: newFakeVolumeResizer("vol-rdsdata01", 20),
|
||||
vmState: vmState,
|
||||
}
|
||||
h.agent = newStubAgent(t, nc, testAccountID, testDBID, agentFails)
|
||||
@@ -311,7 +294,6 @@ func newLifecycleHarness(t *testing.T, agentFails bool) *lifecycleHarness {
|
||||
MasterKey: testMasterKey,
|
||||
Instances: h.cmdr,
|
||||
Snapshots: h.snaps,
|
||||
Storage: h.storage,
|
||||
InstanceState: h.vmState,
|
||||
VMStopTimeout: testVMStopTimeout,
|
||||
ServingCertKeyBits: testServingCertKeyBits,
|
||||
@@ -874,52 +856,3 @@ func TestUncleanStopMessage_IsEngineAware(t *testing.T) {
|
||||
unknown := uncleanStopMessage(t.Context(), "oracle", "stopping the instance")
|
||||
assert.Equal(t, "The database engine could not be shut down cleanly before stopping the instance.", unknown)
|
||||
}
|
||||
|
||||
// A stopped DB VM is terminated over ec2.terminate as the system account. Gone
|
||||
// in either form is done; a refusal is returned so the teardown retries.
|
||||
func TestNATSInstanceCommander_TerminateStoppedInstance(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
reply []byte
|
||||
noReply bool
|
||||
wantErr string
|
||||
}{
|
||||
{name: "terminated", reply: []byte(`{}`)},
|
||||
{name: "already gone", reply: awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound)},
|
||||
{name: "no responder", noReply: true},
|
||||
{name: "refused", reply: awserrors.GenerateErrorPayload(awserrors.ErrorIncorrectInstanceState), wantErr: awserrors.ErrorIncorrectInstanceState},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
var got struct {
|
||||
InstanceID string
|
||||
Account string
|
||||
}
|
||||
if !tc.noReply {
|
||||
_, err := nc.Subscribe("ec2.terminate", func(msg *nats.Msg) {
|
||||
var in ec2instance.TerminateStoppedInstanceInput
|
||||
_ = json.Unmarshal(msg.Data, &in)
|
||||
got.InstanceID, got.Account = in.InstanceID, msg.Header.Get(natsmsg.AccountIDHeader)
|
||||
_ = msg.Respond(tc.reply)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, nc.Flush())
|
||||
}
|
||||
|
||||
err := NewNATSInstanceCommander(nc).TerminateStoppedInstance(t.Context(), testInstance)
|
||||
if tc.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.wantErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
if !tc.noReply {
|
||||
assert.Equal(t, testInstance, got.InstanceID)
|
||||
assert.Equal(t, awsidentifiers.GlobalAccountID, got.Account)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,9 +97,7 @@ func (s *Service) claimModifyLease(
|
||||
) (bool, time.Time, error) {
|
||||
var expiresAt time.Time
|
||||
claimed, err := s.updateInstanceIf(ctx, kv, id, func(rec *DBInstanceRecord) bool {
|
||||
// A delete moves the record to deleting before its teardown checks for a
|
||||
// live lease, so a modify that has not claimed one by then never does.
|
||||
if rec.Status == StatusDeleting || (rec.ModifyLease.live() && rec.ModifyLease.Holder != holder) {
|
||||
if rec.ModifyLease.live() && rec.ModifyLease.Holder != holder {
|
||||
return false
|
||||
}
|
||||
expiresAt = time.Now().UTC().Add(s.modifyLeaseTTL())
|
||||
|
||||
@@ -260,39 +260,3 @@ func TestClaimModifyLease_RefusesAnotherHolderButNotItsOwn(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.True(t, claimed)
|
||||
}
|
||||
|
||||
// A delete that lands while a modify holds the lease, possibly mid-grow, must
|
||||
// not terminate or detach anything until that modify lets go; the reconciler
|
||||
// finishes the delete afterwards.
|
||||
func TestDeleteDBInstance_WaitsForAModifyThatHoldsTheLease(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
rec := availableRecord()
|
||||
rec.ModifyLease = heldLease(modifyLeaseTTL)
|
||||
seedInstance(t, h.svc, rec)
|
||||
|
||||
out, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(StatusDeleting), aws.StringValue(out.DBInstance.DBInstanceStatus))
|
||||
|
||||
assert.Empty(t, h.launcher.terminated, "nothing may be terminated while a modify holds the lease")
|
||||
assert.Empty(t, h.cmdr.calls)
|
||||
assert.Empty(t, h.volumes.deleted)
|
||||
assert.True(t, h.recordExists(t, testDBID), "the record stays for the reconciler to finish")
|
||||
}
|
||||
|
||||
// Once the record is deleting, a modify that had not yet claimed the lease can
|
||||
// never claim it, so the teardown's lease check cannot be overtaken.
|
||||
func TestClaimModifyLease_RefusesADeletingInstance(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newLifecycleHarness(t, false)
|
||||
rec := availableRecord()
|
||||
rec.Status = StatusDeleting
|
||||
seedInstance(t, h.svc, rec)
|
||||
kv, err := h.svc.bucket(t.Context(), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
claimed, _, err := h.svc.claimModifyLease(t.Context(), kv, testDBID, h.svc.newModifyLeaseHolder())
|
||||
require.NoError(t, err)
|
||||
assert.False(t, claimed)
|
||||
}
|
||||
|
||||
@@ -611,7 +611,7 @@ func (r *Reconciler) reconcileStopping(ctx context.Context, kv *kvstore.Bucket,
|
||||
// what lets the customer retry the delete.
|
||||
func (r *Reconciler) reconcileDeleting(ctx context.Context, kv *kvstore.Bucket, rev uint64, accountID string, rec *DBInstanceRecord) error {
|
||||
err := r.svc.teardownDBInstance(ctx, kv, accountID, rec, false)
|
||||
if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errModifyInProgress) {
|
||||
if err == nil || errors.Is(err, errFinalSnapshotInProgress) {
|
||||
return nil
|
||||
}
|
||||
if time.Since(transitionStarted(rec)) > transitionTimeout {
|
||||
|
||||
@@ -89,9 +89,6 @@ func (s *Service) replaceInstanceVM(ctx context.Context, kv *kvstore.Bucket, acc
|
||||
if err := s.terminateInstanceVM(ctx, oldInstanceID); err != nil {
|
||||
return fmt.Errorf("terminate the VM behind %s: %w", rec.DBInstanceIdentifier, err)
|
||||
}
|
||||
if err := s.requireDataVolumeDetached(ctx, rec.DataVolumeID, oldInstanceID); err != nil {
|
||||
return err
|
||||
}
|
||||
// The system NIC is disposable and the fresh launch mints its own; leaving
|
||||
// this one behind would hold an address in the shared RDS system subnet.
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
@@ -105,7 +102,7 @@ func (s *Service) replaceInstanceVM(ctx context.Context, kv *kvstore.Bucket, acc
|
||||
return err
|
||||
}
|
||||
if in.GrowStorageToGiB > 0 {
|
||||
if err := s.growDataVolume(ctx, rec.DataVolumeID, "", in.GrowStorageToGiB); err != nil {
|
||||
if err := s.growDataVolume(ctx, rec.DataVolumeID, in.GrowStorageToGiB); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -201,42 +199,6 @@ func TestReplaceInstanceVM_GrowsTheVolumeWhileNoVMHoldsIt(t *testing.T) {
|
||||
assert.Equal(t, int64(80), h.storage.sizes[testDataVolume])
|
||||
}
|
||||
|
||||
// A guest poweroff now ends as a stop, so the old VM may be held by no node with
|
||||
// the data volume still attached; it is terminated from its record before the grow.
|
||||
func TestReplaceInstanceVM_TerminatesAStoppedOldVMBeforeGrowing(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
rec := seedReplaceable(t, h, modifiableRecord())
|
||||
h.launch.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
|
||||
h.cmdr.onTerminateStopped = func(string) { delete(h.storage.attachedTo, testDataVolume) }
|
||||
|
||||
require.NoError(t, h.svc.replaceInstanceVM(t.Context(), h.kv(t), testAccountID, &rec, replaceInput{
|
||||
InstanceClass: "db.m5.large", InstanceType: "m5.large", GrowStorageToGiB: 80, Reason: "the instance class changed",
|
||||
}))
|
||||
|
||||
assert.Contains(t, h.cmdr.calls, "terminate-stopped:"+testInstance)
|
||||
assert.Equal(t, int64(80), h.storage.sizes[testDataVolume])
|
||||
assert.NotEqual(t, testInstance, h.record(t).InstanceID)
|
||||
}
|
||||
|
||||
// A replace that cannot get the data volume off the old VM goes no further: the
|
||||
// replacement would boot without its datadir.
|
||||
func TestReplaceInstanceVM_StopsWhileTheDataVolumeIsStillOnTheOldVM(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
rec := seedReplaceable(t, h, modifiableRecord())
|
||||
h.launch.launcher.terminateErr = systeminstance.ErrSystemInstanceNotFound
|
||||
h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
|
||||
h.storage.detachErr = errors.New("the volume store is unavailable")
|
||||
|
||||
err := h.svc.replaceInstanceVM(t.Context(), h.kv(t), testAccountID, &rec, replaceInput{Reason: "recovery"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "detach the data volume")
|
||||
assert.Nil(t, h.launch.launcher.input, "no replacement may launch without the data volume")
|
||||
assert.Equal(t, testInstance, h.record(t).InstanceID)
|
||||
}
|
||||
|
||||
// An unmapped class has no instance type behind it, so the replace is refused
|
||||
// before the VM it would land on is torn down.
|
||||
func TestReplaceInstanceVM_RefusesAnUnmappedRecordedClass(t *testing.T) {
|
||||
|
||||
@@ -23,8 +23,6 @@ import (
|
||||
type volumeResizer interface {
|
||||
DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, accountID string) (*ec2.DescribeVolumesOutput, error)
|
||||
ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error)
|
||||
ExpandAttachedToStopped(ctx context.Context, accountID, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error)
|
||||
DetachVolumeOnTerminate(ctx context.Context, volumeID, instanceID, accountID string) error
|
||||
}
|
||||
|
||||
// Everything a resize request can be wrong about, checked before the instance
|
||||
@@ -67,7 +65,7 @@ func (s *Service) growInstanceStorage(ctx context.Context, accountID string, rec
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.growDataVolume(ctx, rec.DataVolumeID, rec.InstanceID, targetGiB); err != nil {
|
||||
if err := s.growDataVolume(ctx, rec.DataVolumeID, targetGiB); err != nil {
|
||||
return s.restartAfterStorageGrowFailure(ctx, accountID, rec, err)
|
||||
}
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
@@ -103,10 +101,10 @@ func (s *Service) restartAfterStorageGrowFailure(
|
||||
return growErr
|
||||
}
|
||||
|
||||
// Grows the data volume itself, idempotently: a resumed grow already at the
|
||||
// target returns without a modify. An attached volume is grown only as
|
||||
// stoppedVM's, the VM the caller stopped; an empty stoppedVM requires it detached.
|
||||
func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string, targetGiB int64) error {
|
||||
// Grows the data volume itself. Idempotent: a resumed grow re-reads the volume
|
||||
// and returns without a second modify once it is already at the target, which
|
||||
// ModifyVolume would otherwise reject as a shrink.
|
||||
func (s *Service) growDataVolume(ctx context.Context, volumeID string, targetGiB int64) error {
|
||||
if volumeID == "" {
|
||||
return errors.New("rds: the DB instance has no data volume to grow")
|
||||
}
|
||||
@@ -114,7 +112,7 @@ func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string
|
||||
return errors.New("rds: no volume resize path configured")
|
||||
}
|
||||
|
||||
current, attachedTo, err := s.dataVolumeState(ctx, volumeID)
|
||||
current, err := s.dataVolumeSize(ctx, volumeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -127,43 +125,31 @@ func (s *Service) growDataVolume(ctx context.Context, volumeID, stoppedVM string
|
||||
return err
|
||||
}
|
||||
|
||||
if attachedTo != "" && stoppedVM != "" {
|
||||
_, err = s.deps.Storage.ExpandAttachedToStopped(ctx, awsidentifiers.GlobalAccountID, volumeID, stoppedVM, targetGiB)
|
||||
} else {
|
||||
_, err = s.deps.Storage.ModifyVolume(ctx, &ec2.ModifyVolumeInput{
|
||||
VolumeId: aws.String(volumeID),
|
||||
Size: aws.Int64(targetGiB),
|
||||
}, awsidentifiers.GlobalAccountID)
|
||||
}
|
||||
if err != nil {
|
||||
if _, err := s.deps.Storage.ModifyVolume(ctx, &ec2.ModifyVolumeInput{
|
||||
VolumeId: aws.String(volumeID),
|
||||
Size: aws.Int64(targetGiB),
|
||||
}, awsidentifiers.GlobalAccountID); err != nil {
|
||||
return fmt.Errorf("grow the data volume %s to %d GiB: %w", volumeID, targetGiB, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "rds: data volume grown",
|
||||
"volumeId", volumeID, "fromGiB", current, "toGiB", targetGiB, "attachedTo", attachedTo)
|
||||
"volumeId", volumeID, "fromGiB", current, "toGiB", targetGiB)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The volume's own reported size, which is what makes the grow idempotent, and
|
||||
// the instance it is attached to. A volume the store cannot describe is an error
|
||||
// rather than a zero, since zero would read as "smaller than the target".
|
||||
func (s *Service) dataVolumeState(ctx context.Context, volumeID string) (int64, string, error) {
|
||||
// The volume's own reported size, which is what makes the grow idempotent. A
|
||||
// volume the store cannot describe is an error rather than a zero, since zero
|
||||
// would read as "smaller than the target" and trigger a modify.
|
||||
func (s *Service) dataVolumeSize(ctx context.Context, volumeID string) (int64, error) {
|
||||
out, err := s.deps.Storage.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{
|
||||
VolumeIds: aws.StringSlice([]string{volumeID}),
|
||||
}, awsidentifiers.GlobalAccountID)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("read the data volume %s: %w", volumeID, err)
|
||||
return 0, fmt.Errorf("read the data volume %s: %w", volumeID, err)
|
||||
}
|
||||
for _, volume := range out.Volumes {
|
||||
if aws.StringValue(volume.VolumeId) != volumeID {
|
||||
continue
|
||||
if aws.StringValue(volume.VolumeId) == volumeID {
|
||||
return aws.Int64Value(volume.Size), nil
|
||||
}
|
||||
attachedTo := ""
|
||||
if aws.StringValue(volume.State) == ec2.VolumeStateInUse {
|
||||
for _, a := range volume.Attachments {
|
||||
attachedTo = aws.StringValue(a.InstanceId)
|
||||
}
|
||||
}
|
||||
return aws.Int64Value(volume.Size), attachedTo, nil
|
||||
}
|
||||
return 0, "", fmt.Errorf("rds: data volume %s no longer exists", volumeID)
|
||||
return 0, fmt.Errorf("rds: data volume %s no longer exists", volumeID)
|
||||
}
|
||||
|
||||
@@ -20,14 +20,6 @@ type fakeVolumeResizer struct {
|
||||
sizes map[string]int64
|
||||
modified []*ec2.ModifyVolumeInput
|
||||
|
||||
// attachedTo is the instance each in-use volume is attached to, as a stop now
|
||||
// leaves it; expanded records the grows taken through that attachment.
|
||||
attachedTo map[string]string
|
||||
expanded []string
|
||||
// detachErr fails the terminate-time detach; detached records the ones made.
|
||||
detachErr error
|
||||
detached []string
|
||||
|
||||
describeErr error
|
||||
modifyErr error
|
||||
// missing makes the volume read as gone, which is the state a grow must not
|
||||
@@ -58,12 +50,7 @@ func (f *fakeVolumeResizer) DescribeVolumes(_ context.Context, in *ec2.DescribeV
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
vol := &ec2.Volume{VolumeId: aws.String(id), Size: aws.Int64(size), State: aws.String(ec2.VolumeStateAvailable)}
|
||||
if inst := f.attachedTo[id]; inst != "" {
|
||||
vol.State = aws.String(ec2.VolumeStateInUse)
|
||||
vol.Attachments = []*ec2.VolumeAttachment{{VolumeId: aws.String(id), InstanceId: aws.String(inst)}}
|
||||
}
|
||||
out.Volumes = append(out.Volumes, vol)
|
||||
out.Volumes = append(out.Volumes, &ec2.Volume{VolumeId: aws.String(id), Size: aws.Int64(size)})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -84,35 +71,6 @@ func (f *fakeVolumeResizer) ModifyVolume(ctx context.Context, in *ec2.ModifyVolu
|
||||
return &ec2.ModifyVolumeOutput{}, nil
|
||||
}
|
||||
|
||||
// Mirrors the store: a grow through an attachment is refused unless the volume
|
||||
// is attached to the instance the caller names.
|
||||
func (f *fakeVolumeResizer) ExpandAttachedToStopped(ctx context.Context, _, volumeID, instanceID string, sizeGiB int64) (*ec2.ModifyVolumeOutput, error) {
|
||||
if f.onModify != nil {
|
||||
f.onModify(ctx)
|
||||
}
|
||||
if f.modifyErr != nil {
|
||||
return nil, f.modifyErr
|
||||
}
|
||||
if instanceID == "" || f.attachedTo[volumeID] != instanceID {
|
||||
return nil, errors.New(awserrors.ErrorIncorrectState)
|
||||
}
|
||||
f.expanded = append(f.expanded, volumeID+":"+instanceID)
|
||||
f.sizes[volumeID] = sizeGiB
|
||||
return &ec2.ModifyVolumeOutput{}, nil
|
||||
}
|
||||
|
||||
// Mirrors the store: only the terminated instance's own attachment is cleared.
|
||||
func (f *fakeVolumeResizer) DetachVolumeOnTerminate(_ context.Context, volumeID, instanceID, _ string) error {
|
||||
if f.detachErr != nil {
|
||||
return f.detachErr
|
||||
}
|
||||
if f.attachedTo[volumeID] == instanceID {
|
||||
delete(f.attachedTo, volumeID)
|
||||
f.detached = append(f.detached, volumeID+":"+instanceID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The whole rule set a resize is checked against, ahead of any state change.
|
||||
func TestValidateStorageGrow(t *testing.T) {
|
||||
cases := []struct {
|
||||
@@ -146,7 +104,7 @@ func TestGrowDataVolume_SkipsAVolumeAlreadyAtTheTarget(t *testing.T) {
|
||||
h := newModifyHarness(t)
|
||||
h.storage.sizes[testDataVolume] = 50
|
||||
|
||||
require.NoError(t, h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50))
|
||||
require.NoError(t, h.svc.growDataVolume(t.Context(), testDataVolume, 50))
|
||||
assert.Empty(t, h.storage.modified, "a volume already at the target is not modified again")
|
||||
}
|
||||
|
||||
@@ -157,7 +115,7 @@ func TestGrowDataVolume_FailsWhenTheVolumeIsGone(t *testing.T) {
|
||||
h := newModifyHarness(t)
|
||||
h.storage.missing = true
|
||||
|
||||
err := h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50)
|
||||
err := h.svc.growDataVolume(t.Context(), testDataVolume, 50)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "no longer exists")
|
||||
assert.Empty(t, h.storage.modified)
|
||||
@@ -167,7 +125,7 @@ func TestGrowDataVolume_FailsWithoutAVolume(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
|
||||
require.Error(t, h.svc.growDataVolume(t.Context(), "", testInstance, 50))
|
||||
require.Error(t, h.svc.growDataVolume(t.Context(), "", 50))
|
||||
}
|
||||
|
||||
// ModifyVolume refuses a volume a running VM holds, so the order is the
|
||||
@@ -193,35 +151,6 @@ func TestGrowInstanceStorage_StopsTheEngineAndTheVMBeforeGrowing(t *testing.T) {
|
||||
assert.Equal(t, CommandStopEngine, issued[0].Type)
|
||||
}
|
||||
|
||||
// A stop leaves the data volume attached to the stopped VM, so the grow goes
|
||||
// through that attachment rather than the customer path, which refuses it.
|
||||
func TestGrowInstanceStorage_GrowsTheVolumeStillAttachedToTheStoppedVM(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
h.storage.attachedTo = map[string]string{testDataVolume: testInstance}
|
||||
rec := modifiableRecord()
|
||||
|
||||
require.NoError(t, h.svc.growInstanceStorage(t.Context(), testAccountID, &rec, 50))
|
||||
|
||||
assert.Equal(t, []string{testDataVolume + ":" + testInstance}, h.storage.expanded)
|
||||
assert.Empty(t, h.storage.modified, "an attached volume must not go through ModifyVolume")
|
||||
assert.Equal(t, int64(50), h.storage.sizes[testDataVolume])
|
||||
assert.Equal(t, []string{"stop:" + testInstance, "start:" + testInstance}, h.cmdr.calls)
|
||||
}
|
||||
|
||||
// A data volume attached to some other instance is not this DB VM's to grow.
|
||||
func TestGrowDataVolume_RefusesAVolumeAttachedToAnotherInstance(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
h.storage.attachedTo = map[string]string{testDataVolume: "i-someone-else"}
|
||||
|
||||
err := h.svc.growDataVolume(t.Context(), testDataVolume, testInstance, 50)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorIncorrectState)
|
||||
assert.Empty(t, h.storage.expanded)
|
||||
assert.NotEqual(t, int64(50), h.storage.sizes[testDataVolume])
|
||||
}
|
||||
|
||||
// The stop command is accepted seconds before the volume detaches, and the
|
||||
// store refuses a resize until it has. The grow holds off until the fleet
|
||||
// reports the VM down rather than resizing on the accepted command.
|
||||
|
||||
@@ -750,7 +750,7 @@ func (a *instanceCleanerAdapter) DeleteVolumes(instance *vm.VM) error {
|
||||
"name", ebsRequest.Name, "id", instance.ID)
|
||||
continue
|
||||
}
|
||||
if err := a.d.volumeService.DetachVolumeOnTerminate(context.Background(), ebsRequest.Name, instance.ID, instance.AccountID); err != nil && !awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
|
||||
if err := a.d.volumeService.DetachVolumeOnTerminate(context.Background(), ebsRequest.Name, instance.AccountID); err != nil && !awserrors.IsNotFound(err) && !objectstore.IsNoSuchKeyError(err) {
|
||||
slog.Error("Failed to detach volume on termination",
|
||||
"name", ebsRequest.Name, "id", instance.ID, "err", err)
|
||||
firstErr = cmp.Or(firstErr, err)
|
||||
|
||||
+6
-12
@@ -2302,8 +2302,8 @@ packages:
|
||||
bowser@2.14.1:
|
||||
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
|
||||
|
||||
brace-expansion@5.0.9:
|
||||
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
|
||||
brace-expansion@5.0.12:
|
||||
resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==}
|
||||
engines: {node: 20 || >=22}
|
||||
|
||||
braces@3.0.3:
|
||||
@@ -3453,10 +3453,6 @@ packages:
|
||||
resolution: {integrity: sha512-SO/3iYL5S3W57LLEniscOGPZgOqZUPCx6d3dB+52B80yJ0XstzsC/eV8gnA4tM3MHDrKz+OCFSLNjswdSC+/bA==}
|
||||
engines: {node: '>=22'}
|
||||
|
||||
source-map-js@1.2.1:
|
||||
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
source-map-js@1.2.2:
|
||||
resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
@@ -4986,7 +4982,7 @@ snapshots:
|
||||
jiti: 2.7.0
|
||||
lightningcss: 1.32.0
|
||||
magic-string: 0.30.21
|
||||
source-map-js: 1.2.1
|
||||
source-map-js: 1.2.2
|
||||
tailwindcss: 4.3.3
|
||||
|
||||
'@tailwindcss/oxide-android-arm64@4.3.3':
|
||||
@@ -5566,7 +5562,7 @@ snapshots:
|
||||
|
||||
bowser@2.14.1: {}
|
||||
|
||||
brace-expansion@5.0.9:
|
||||
brace-expansion@5.0.12:
|
||||
dependencies:
|
||||
balanced-match: 4.0.4
|
||||
|
||||
@@ -6189,7 +6185,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@babel/parser': 7.29.8
|
||||
'@babel/types': 7.29.8
|
||||
source-map-js: 1.2.1
|
||||
source-map-js: 1.2.2
|
||||
|
||||
markdown-table@3.0.4: {}
|
||||
|
||||
@@ -6550,7 +6546,7 @@ snapshots:
|
||||
|
||||
minimatch@10.2.6:
|
||||
dependencies:
|
||||
brace-expansion: 5.0.9
|
||||
brace-expansion: 5.0.12
|
||||
|
||||
mlly@1.8.2:
|
||||
dependencies:
|
||||
@@ -6946,8 +6942,6 @@ snapshots:
|
||||
ansi-styles: 6.2.3
|
||||
is-fullwidth-code-point: 5.1.0
|
||||
|
||||
source-map-js@1.2.1: {}
|
||||
|
||||
source-map-js@1.2.2: {}
|
||||
|
||||
space-separated-tokens@2.0.2: {}
|
||||
|
||||
Reference in New Issue
Block a user