diff --git a/spinifex/handlers/ecs/instance_recovery_test.go b/spinifex/handlers/ecs/instance_recovery_test.go index df1157e25..816eb9b0e 100644 --- a/spinifex/handlers/ecs/instance_recovery_test.go +++ b/spinifex/handlers/ecs/instance_recovery_test.go @@ -60,6 +60,56 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) { assert.False(t, recovered.Reaped) } +// Reaping an instance must release the capacity its tasks reserved, so a +// DRAINING->ACTIVE flip on agent re-register (involuntary reap recovery) never +// carries stale ReservedCPU/Memory or PlacedTasks into the re-activated instance. +func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) { + svc, nc := newTestService(t) + _, err := svc.CreateCluster(&ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID) + require.NoError(t, err) + registerTaskDef(t, svc, "app", 128, 256) + registerInstance(t, svc, "web", "i-1", 1024, 2048) + + out, err := svc.RunTask(&ecs.RunTaskInput{ + Cluster: aws.String("web"), TaskDefinition: aws.String("app"), Count: aws.Int64(1), + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.Tasks, 1) + taskID := containerInstanceShortID(aws.StringValue(out.Tasks[0].TaskArn)) + + seeded := instanceStatus(t, svc, "web", "i-1") + require.Equal(t, 128, seeded.ReservedCPU) + require.Equal(t, 256, seeded.ReservedMemoryMiB) + require.Contains(t, seeded.PlacedTasks, taskID) + + kv, err := svc.bucket(testAccountID) + require.NoError(t, err) + seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout) + require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &seeded)) + + NewScheduler(nc, svc, "test-holder").reapBucket(kv, testAccountID, time.Now().UTC()) + + reaped := instanceStatus(t, svc, "web", "i-1") + require.Equal(t, InstanceStatusDraining, reaped.Status) + require.True(t, reaped.Reaped) + assert.Zero(t, reaped.ReservedCPU, "reaped instance must release reserved CPU") + assert.Zero(t, reaped.ReservedMemoryMiB, "reaped instance must release reserved memory") + assert.Empty(t, reaped.PlacedTasks, "reaped instance must drop placed tasks") + + dt, err := svc.DescribeTasks(&ecs.DescribeTasksInput{ + Cluster: aws.String("web"), Tasks: []*string{aws.String(taskID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, dt.Tasks, 1) + assert.Equal(t, TaskStatusStopped, aws.StringValue(dt.Tasks[0].LastStatus)) + + reRegister(t, svc, "web", "i-1") + recovered := instanceStatus(t, svc, "web", "i-1") + assert.Equal(t, InstanceStatusActive, recovered.Status) + assert.Zero(t, recovered.ReservedCPU, "re-activated instance must start with no stale reservation") + assert.Empty(t, recovered.PlacedTasks) +} + // An operator UpdateContainerInstancesState=DRAINING is intentional and must // persist even though the live agent keeps re-registering. func TestRegister_DoesNotUndoOperatorDrain(t *testing.T) {