mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 19:56:33 +00:00
fix: persist EKS control-plane VM refs before egress alloc to prevent leak (#338)
CreateCluster set the control-plane VM/ENI/spread-group refs into the cluster meta but first persisted them to KV only after allocateClusterEgressIP. A failure between placement and that persist left the live sys.medium CP VMs unrecorded, so neither DeleteCluster nor the FAILED-cluster reclaim could reach them and they leaked, reserving host capacity. Persist the meta immediately after placeControlPlane returns. The reclaim path keys off the persisted ControlPlaneNodes + ControlPlaneSpreadGroup, so this also frees reserved spread capacity on teardown.
This commit is contained in:
@@ -101,6 +101,46 @@ func TestCreateCluster_FailedCreateThenDeleteReclaimsNLB(t *testing.T) {
|
||||
assert.ErrorIs(t, getErr, ErrClusterNotFound)
|
||||
}
|
||||
|
||||
// A post-placement failure (egress allocation) must leave the just-launched
|
||||
// control-plane VM refs persisted in the FAILED meta. Otherwise the live
|
||||
// sys.medium VMs are unrecorded and neither DeleteCluster nor the FAILED-cluster
|
||||
// reclaim can reach them — they leak.
|
||||
func TestCreateCluster_EgressFailureLeavesControlPlaneRecorded(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
|
||||
// Placement succeeds and launches the CP VM, then egress allocation fails.
|
||||
f.eip.allocateErr = errors.New("no addresses in hidden pool")
|
||||
|
||||
_, err := f.svc.CreateCluster(createInput("alpha"), testAccountID, "")
|
||||
require.Error(t, err)
|
||||
require.NotEmpty(t, f.inst.launchCalls, "control-plane VM was launched before the egress step")
|
||||
|
||||
meta, getErr := GetClusterMeta(f.kv, "alpha")
|
||||
require.NoError(t, getErr, "meta must remain so the leaked CP VM has an owning record")
|
||||
assert.Equal(t, ClusterStatusFailed, meta.Status)
|
||||
assert.NotEmpty(t, meta.ControlPlaneInstanceID, "CP instance ID must be persisted before the egress step")
|
||||
assert.NotEmpty(t, meta.ControlPlaneNodes, "CP node list must be persisted before the egress step")
|
||||
}
|
||||
|
||||
// End-to-end: a create that fails at egress, then delete-cluster, must terminate
|
||||
// the control-plane VM the partial create launched — driven by the CP refs the
|
||||
// early persist recorded.
|
||||
func TestCreateCluster_EgressFailedThenDeleteTerminatesControlPlane(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
f.eip.allocateErr = errors.New("no addresses in hidden pool")
|
||||
|
||||
_, err := f.svc.CreateCluster(createInput("alpha"), testAccountID, "")
|
||||
require.Error(t, err)
|
||||
require.NotEmpty(t, f.inst.launchCalls, "create launched a CP VM")
|
||||
|
||||
_, err = f.svc.DeleteCluster(deleteInput("alpha"), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEmpty(t, f.inst.terminateCalls, "DeleteCluster must terminate the CP VM recorded by the failed create")
|
||||
_, getErr := GetClusterMeta(f.kv, "alpha")
|
||||
assert.ErrorIs(t, getErr, ErrClusterNotFound)
|
||||
}
|
||||
|
||||
// A live (CREATING/ACTIVE) cluster of the same name blocks create with a
|
||||
// cluster-scoped ResourceInUseException, not the ELBv2 target-group message.
|
||||
func TestCreateCluster_ExistingClusterReturnsResourceInUse(t *testing.T) {
|
||||
|
||||
@@ -506,6 +506,16 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
meta.ControlPlaneENIIP = primary.ENIIP
|
||||
meta.ControlPlaneMgmtIP = primary.MgmtIP
|
||||
|
||||
// Persist the CP VM + ENI + spread-group refs now, before any further fallible
|
||||
// step. The VMs are live the moment placeControlPlane returns; without this a
|
||||
// failure between here and the next PutClusterMeta (e.g. egress allocation)
|
||||
// leaves them launched but unrecorded, so neither DeleteCluster nor the
|
||||
// FAILED-cluster reclaim could reach them and the sys.medium VMs leak.
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, logCreateErr(name, accountID, "persist control-plane ids", err)
|
||||
}
|
||||
|
||||
// Egress: the control-plane VM must pull container images. Allocate a
|
||||
// hidden pool address and wire an egress-only SNAT for its /32 (no DNAT —
|
||||
// the VM stays unreachable inbound; the NLB is its only front door).
|
||||
@@ -523,11 +533,11 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
|
||||
ExternalIp: egressIP,
|
||||
})
|
||||
|
||||
// Record the VM + ENI + egress IP before registering the target so a failed
|
||||
// register (or anything after) leaves them recoverable by DeleteCluster.
|
||||
// Record the egress allocation before registering the target so a failed
|
||||
// register (or anything after) leaves it recoverable by DeleteCluster.
|
||||
if err := PutClusterMeta(acctKV, meta); err != nil {
|
||||
s.markFailed(acctKV, name)
|
||||
return nil, logCreateErr(name, accountID, "persist control-plane ids", err)
|
||||
return nil, logCreateErr(name, accountID, "persist egress allocation", err)
|
||||
}
|
||||
|
||||
cpENIIPs := make([]string, 0, len(cpNodes))
|
||||
|
||||
Reference in New Issue
Block a user