test: lock reaper capacity release + clean re-activation (#432)

spinifex/handlers/ecs/instance_recovery_test.go

@@ -60,6 +60,56 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	assert.False(t, recovered.Reaped)
 }
 
+// Reaping an instance must release the capacity its tasks reserved, so a
+// DRAINING->ACTIVE flip on agent re-register (involuntary reap recovery) never
+// carries stale ReservedCPU/Memory or PlacedTasks into the re-activated instance.
+func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
+	svc, nc := newTestService(t)
+	_, err := svc.CreateCluster(&ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	registerTaskDef(t, svc, "app", 128, 256)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+
+	out, err := svc.RunTask(&ecs.RunTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("app"), Count: aws.Int64(1),
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.Tasks, 1)
+	taskID := containerInstanceShortID(aws.StringValue(out.Tasks[0].TaskArn))
+
+	seeded := instanceStatus(t, svc, "web", "i-1")
+	require.Equal(t, 128, seeded.ReservedCPU)
+	require.Equal(t, 256, seeded.ReservedMemoryMiB)
+	require.Contains(t, seeded.PlacedTasks, taskID)
+
+	kv, err := svc.bucket(testAccountID)
+	require.NoError(t, err)
+	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
+	require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &seeded))
+
+	NewScheduler(nc, svc, "test-holder").reapBucket(kv, testAccountID, time.Now().UTC())
+
+	reaped := instanceStatus(t, svc, "web", "i-1")
+	require.Equal(t, InstanceStatusDraining, reaped.Status)
+	require.True(t, reaped.Reaped)
+	assert.Zero(t, reaped.ReservedCPU, "reaped instance must release reserved CPU")
+	assert.Zero(t, reaped.ReservedMemoryMiB, "reaped instance must release reserved memory")
+	assert.Empty(t, reaped.PlacedTasks, "reaped instance must drop placed tasks")
+
+	dt, err := svc.DescribeTasks(&ecs.DescribeTasksInput{
+		Cluster: aws.String("web"), Tasks: []*string{aws.String(taskID)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, dt.Tasks, 1)
+	assert.Equal(t, TaskStatusStopped, aws.StringValue(dt.Tasks[0].LastStatus))
+
+	reRegister(t, svc, "web", "i-1")
+	recovered := instanceStatus(t, svc, "web", "i-1")
+	assert.Equal(t, InstanceStatusActive, recovered.Status)
+	assert.Zero(t, recovered.ReservedCPU, "re-activated instance must start with no stale reservation")
+	assert.Empty(t, recovered.PlacedTasks)
+}
+
 // An operator UpdateContainerInstancesState=DRAINING is intentional and must
 // persist even though the live agent keeps re-registering.
 func TestRegister_DoesNotUndoOperatorDrain(t *testing.T) {