diff --git a/tests/e2e/single/spot_test.go b/tests/e2e/single/spot_test.go deleted file mode 100644 index 685e20a78..000000000 --- a/tests/e2e/single/spot_test.go +++ /dev/null @@ -1,192 +0,0 @@ -//go:build e2e - -package single - -import ( - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/mulgadc/spinifex/tests/e2e/harness" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// runSpotInstanceLifecycle exercises the Spot Instance Request mock end to end. -// RequestSpotInstances synchronously launches real VMs via the on-demand path -// and reports the requests active/fulfilled — there is no bidding, interruption, -// or reclamation. One SIR is cancelled (request goes cancelled, instance keeps -// running); the other's instance is terminated (request moves to closed). A final -// oversized request asserts capacity failure persists no SIRs. Maps to the E2E -// row in docs/development/feature/spot-instances-v1.md. -func runSpotInstanceLifecycle(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — Spot Instance Requests (mock fulfilment)") - - amiID := needAMI(t, fix) - instType, _ := needInstanceTypeArch(t, fix) - keyName, _ := needKeyPair(t, fix) - - // InstanceCount=2 -> MinCount=MaxCount=2 (all-or-nothing). Two requests let - // us drive both terminal transitions: cancel one, terminate the other. - const count = 2 - harness.Step(t, "request-spot-instances ami=%s type=%s count=%d", amiID, instType, count) - reqOut, err := fix.AWS.EC2.RequestSpotInstances(&ec2.RequestSpotInstancesInput{ - InstanceCount: aws.Int64(count), - Type: aws.String(ec2.SpotInstanceTypeOneTime), - LaunchSpecification: &ec2.RequestSpotLaunchSpecification{ - ImageId: aws.String(amiID), - InstanceType: aws.String(instType), - KeyName: aws.String(keyName), - }, - }) - require.NoError(t, err, "request-spot-instances --instance-count 2") - require.Lenf(t, reqOut.SpotInstanceRequests, count, - "expected %d spot instance requests, got %d", count, len(reqOut.SpotInstanceRequests)) - - sirIDs := make([]string, 0, count) - for _, sir := range reqOut.SpotInstanceRequests { - id := aws.StringValue(sir.SpotInstanceRequestId) - require.NotEmpty(t, id, "empty SpotInstanceRequestId in response") - require.Truef(t, strings.HasPrefix(id, "sir-"), "SIR id %q lacks sir- prefix", id) - sirIDs = append(sirIDs, id) - assert.Equal(t, ec2.SpotInstanceStateActive, aws.StringValue(sir.State), - "SIR %s should be active at create", id) - } - harness.Detail(t, "sir_cancel", sirIDs[0], "sir_terminate", sirIDs[1]) - - // Pre-register teardown of every launched VM before the first blocking wait, - // so a fatal still tears the real instances down. - var launchedIDs []string - t.Cleanup(func() { - if len(launchedIDs) == 0 { - return - } - _, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: aws.StringSlice(launchedIDs), - }) - }) - - // `aws ec2 wait spot-instance-request-fulfilled` — fulfilment is synchronous, - // so the waiter returns on the first poll. - harness.Step(t, "wait spot-instance-request-fulfilled") - require.NoError(t, fix.AWS.EC2.WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice(sirIDs), - }), "wait spot-instance-request-fulfilled") - - // Describe both SIRs: active + fulfilled + an InstanceId mapped per request. - descOut, err := fix.AWS.EC2.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice(sirIDs), - }) - require.NoError(t, err, "describe-spot-instance-requests") - require.Lenf(t, descOut.SpotInstanceRequests, count, - "expected %d SIRs from describe, got %d", count, len(descOut.SpotInstanceRequests)) - - sirInstance := make(map[string]string, count) - for _, sir := range descOut.SpotInstanceRequests { - id := aws.StringValue(sir.SpotInstanceRequestId) - assert.Equal(t, ec2.SpotInstanceStateActive, aws.StringValue(sir.State), "SIR %s state", id) - require.NotNilf(t, sir.Status, "SIR %s missing status", id) - assert.Equal(t, "fulfilled", aws.StringValue(sir.Status.Code), "SIR %s status code", id) - instID := aws.StringValue(sir.InstanceId) - require.NotEmptyf(t, instID, "SIR %s has no InstanceId", id) - sirInstance[id] = instID - launchedIDs = append(launchedIDs, instID) - } - - // Both backing VMs are real on-demand launches — wait for running, then assert - // full spot lineage: each VM reports InstanceLifecycle=spot and links back to - // its originating SIR via SpotInstanceRequestId (stamped by the write-back). - for sirID, instID := range sirInstance { - inst := harness.WaitForInstanceState(t, fix.AWS, instID, "running") - assert.Equal(t, ec2.InstanceLifecycleTypeSpot, aws.StringValue(inst.InstanceLifecycle), - "instance %s should report InstanceLifecycle=spot", instID) - assert.Equal(t, sirID, aws.StringValue(inst.SpotInstanceRequestId), - "instance %s should link back to SIR %s", instID, sirID) - } - - cancelSIR, termSIR := sirIDs[0], sirIDs[1] - cancelInst, termInst := sirInstance[cancelSIR], sirInstance[termSIR] - require.NotEmpty(t, cancelInst, "cancel SIR resolved no instance") - require.NotEmpty(t, termInst, "terminate SIR resolved no instance") - - // Cancel one request: it flips to cancelled but the instance keeps running - // (cancel != terminate). The gateway->daemon move is synchronous. - harness.Step(t, "cancel-spot-instance-requests %s", cancelSIR) - cancelOut, err := fix.AWS.EC2.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice([]string{cancelSIR}), - }) - require.NoError(t, err, "cancel-spot-instance-requests %s", cancelSIR) - require.Lenf(t, cancelOut.CancelledSpotInstanceRequests, 1, - "expected 1 cancelled request, got %d", len(cancelOut.CancelledSpotInstanceRequests)) - assert.Equal(t, ec2.CancelSpotInstanceRequestStateCancelled, - aws.StringValue(cancelOut.CancelledSpotInstanceRequests[0].State)) - - cancelled := describeSpotRequest(t, fix, cancelSIR) - assert.Equal(t, ec2.SpotInstanceStateCancelled, aws.StringValue(cancelled.State), - "SIR %s should be cancelled", cancelSIR) - require.NotNil(t, cancelled.Status) - assert.Equal(t, "request-canceled-and-instance-running", aws.StringValue(cancelled.Status.Code)) - - running := harness.WaitForInstanceState(t, fix.AWS, cancelInst, "running") - assert.Equal(t, "running", aws.StringValue(running.State.Name), - "cancelled SIR's instance %s must keep running", cancelInst) - - // Terminate the other request's instance: the daemon teardown cleaner scans - // the active bucket by InstanceId and moves the SIR to closed. - harness.Step(t, "terminate-instances %s (closes %s)", termInst, termSIR) - _, err = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: aws.StringSlice([]string{termInst}), - }) - require.NoError(t, err, "terminate-instances %s", termInst) - harness.WaitForInstanceState(t, fix.AWS, termInst, "terminated") - - // The close runs through the async teardown chain, so poll for it. - var closed *ec2.SpotInstanceRequest - harness.Eventually(t, func() bool { - out, derr := fix.AWS.EC2.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice([]string{termSIR}), - }) - if derr != nil || len(out.SpotInstanceRequests) != 1 { - return false - } - closed = out.SpotInstanceRequests[0] - return aws.StringValue(closed.State) == ec2.SpotInstanceStateClosed - }, 60*time.Second, 2*time.Second, "SIR %s should move to closed after terminate", termSIR) - require.NotNil(t, closed, "describe never returned SIR %s", termSIR) - require.NotNil(t, closed.Status) - assert.Equal(t, "instance-terminated-by-user", aws.StringValue(closed.Status.Code)) - - // Insufficient capacity: an oversized request short-circuits before any - // launch (totalCapacity < MinCount) and persists no SIRs. - t.Run("InsufficientCapacityNoSIRs", func(t *testing.T) { - harness.Step(t, "request-spot-instances count=1000000 (capacity short-circuit)") - out, rerr := fix.AWS.EC2.RequestSpotInstances(&ec2.RequestSpotInstancesInput{ - InstanceCount: aws.Int64(1_000_000), - LaunchSpecification: &ec2.RequestSpotLaunchSpecification{ - ImageId: aws.String(amiID), - InstanceType: aws.String(instType), - KeyName: aws.String(keyName), - }, - }) - harness.AssertAWSError(t, rerr, "InsufficientInstanceCapacity") - if out != nil { - assert.Empty(t, out.SpotInstanceRequests, - "capacity failure must persist/return no SIRs") - } - }) -} - -// describeSpotRequest returns the single SIR matching sirID, failing the test if -// the gateway does not return exactly one. -func describeSpotRequest(t *testing.T, fix *Fixture, sirID string) *ec2.SpotInstanceRequest { - t.Helper() - out, err := fix.AWS.EC2.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice([]string{sirID}), - }) - require.NoError(t, err, "describe-spot-instance-requests %s", sirID) - require.Lenf(t, out.SpotInstanceRequests, 1, - "expected exactly 1 SIR for %s, got %d", sirID, len(out.SpotInstanceRequests)) - return out.SpotInstanceRequests[0] -} diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index dd68895d6..c88f42800 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -154,13 +154,6 @@ func TestRebootInstance(t *testing.T) { runRebootInstance(t, requireSingleNodeFixture(t)) } -// TestSpotInstanceLifecycle drives the Spot Instance Request mock: request -> -// fulfilled -> running -> cancel/terminate terminal transitions. Sequential: it -// consumes node capacity for its own short-lived VMs like the multi-count test. -func TestSpotInstanceLifecycle(t *testing.T) { - runSpotInstanceLifecycle(t, requireSingleNodeFixture(t)) -} - // --- Sequential: shared-state / sub-test-parallel Tests --- func TestNegativeErrorPaths(t *testing.T) { diff --git a/tests/integration/spot_daemon_lite.go b/tests/integration/spot_daemon_lite.go new file mode 100644 index 000000000..dad065678 --- /dev/null +++ b/tests/integration/spot_daemon_lite.go @@ -0,0 +1,38 @@ +//go:build integration + +package integration + +import ( + "testing" + + handlers_ec2_spotinstance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/spotinstance" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/require" +) + +// StartSpotDaemonLite subscribes a real handlers_ec2_spotinstance.SpotInstanceServiceImpl — +// the same production code a live daemon runs (daemon/daemon_handlers_spotinstance.go) — to +// the ec2.PutSpotInstanceRequests/DescribeSpotInstanceRequests/CancelSpotInstanceRequests +// subjects the gateway's NATSSpotInstanceService client calls, so RequestSpotInstances / +// DescribeSpotInstanceRequests / CancelSpotInstanceRequests exercise genuine daemon-side +// persistence instead of a StubSubject canned reply. +// +// CloseForInstance is never NATS-routed: a live daemon calls it in-process from its teardown +// cleaner (spinifex/daemon/vm_adapters.go instanceCleanerAdapter.RemoveFromSpotRequest), which +// needs a real vm.Manager and is out of scope for this tier, matching DaemonLite's own +// carve-out for instance lifecycle. The returned service exposes CloseForInstance directly so +// a test can invoke it the same way the teardown cleaner would, without standing up VM +// lifecycle machinery. +func StartSpotDaemonLite(t *testing.T, gw *Gateway) *handlers_ec2_spotinstance.SpotInstanceServiceImpl { + t.Helper() + + svc, err := handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(nil, gw.NATSConn) + require.NoError(t, err, "construct spot instance service") + + nc := gw.NATSConn + sub(t, nc, "ec2.PutSpotInstanceRequests", func(m *nats.Msg) { dispatch(m, svc.PutSpotInstanceRequests) }) + sub(t, nc, "ec2.DescribeSpotInstanceRequests", func(m *nats.Msg) { dispatch(m, svc.DescribeSpotInstanceRequests) }) + sub(t, nc, "ec2.CancelSpotInstanceRequests", func(m *nats.Msg) { dispatch(m, svc.CancelSpotInstanceRequests) }) + + return svc +} diff --git a/tests/integration/spot_instance_test.go b/tests/integration/spot_instance_test.go new file mode 100644 index 000000000..9ec3d3ee1 --- /dev/null +++ b/tests/integration/spot_instance_test.go @@ -0,0 +1,260 @@ +//go:build integration + +package integration + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/types" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRequestSpotInstances_Lifecycle exercises the gateway's Spot Instance Request +// orchestration end to end: RequestSpotInstances validates the input, dispatches the shared +// on-demand RunInstances placement/quota path (spinifex/gateway/ec2/spotinstance/ +// RequestSpotInstances.go), persists one active/fulfilled SpotInstanceRequest per launched +// instance via the daemon's spot service, and stamps spot lineage back onto each instance. +// Fulfilment itself is a documented mock (RequestSpotInstances.go:2 — no bidding, +// interruption, or reclamation), so this drives the request/describe/cancel/close state +// machine and the real placement/capacity logic, not a real guest boot: that on-demand +// lifecycle (RunInstances expansion, real VM running/terminated transitions) is already +// covered by TestRunInstances_MultiCountExpansion (this package) and the single-tier e2e +// baseline helpers (tests/e2e/single/baseline_test.go launchBaselineInstance, +// tests/e2e/single/helpers_test.go needInstance). +// +// The stubbed node offers capacity for exactly 2 of instanceType, so both the 2-instance +// request and the later 1,000,000-instance request resolve through distributeInstances' real +// totalCapacity accounting (placement.go) rather than the degenerate zero-node path +// documented on TestRunInstances_MultiCountExpansion. +func TestRequestSpotInstances_Lifecycle(t *testing.T) { + gw := StartGateway(t) + spotSvc := StartSpotDaemonLite(t, gw) + + const ( + instanceType = "t3.micro" + nodeID = "node-1" + count = 2 + amiID = "ami-0123456789abcdef0" + ) + + statusResp := mustMarshal(t, &types.NodeStatusResponse{ + Node: nodeID, + InstanceTypes: []types.InstanceTypeCap{ + {Name: instanceType, Available: count}, + }, + }) + gw.StubSubject(t, "spinifex.node.status", statusResp) + + // The oversized request at the end of this test drives distributeInstances into + // InsufficientInstanceCapacity; RunInstances then disambiguates "no capacity" from + // "unknown type" via a DescribeInstanceTypes round trip (RunInstances.go + // isKnownInstanceType) before surfacing the error. Stubbing instanceType as known keeps + // that disambiguation truthful instead of masking the intended error as InvalidInstanceType. + gw.StubSubject(t, "ec2.DescribeInstanceTypes", mustMarshal(t, &ec2.DescribeInstanceTypesOutput{ + InstanceTypes: []*ec2.InstanceTypeInfo{{InstanceType: aws.String(instanceType)}}, + })) + + launchedInstanceIDs := []string{"i-spot-1", "i-spot-2"} + + // Subscribing directly to the per-node launch subject (rather than a StubSubject + // canned reply) lets the test assert the gateway genuinely dispatched + // MinCount=MaxCount=2, mirroring TestRunInstances_MultiCountExpansion's anti-tautology + // check for the on-demand path. + var gotMinCount, gotMaxCount int64 + launchSubject := fmt.Sprintf("ec2.RunInstances.%s.%s", instanceType, nodeID) + launchSub, err := gw.NATSConn.Subscribe(launchSubject, func(msg *nats.Msg) { + var nodeInput ec2.RunInstancesInput + if jsonErr := json.Unmarshal(msg.Data, &nodeInput); jsonErr != nil { + t.Errorf("unmarshal per-node RunInstances request: %v", jsonErr) + return + } + gotMinCount = aws.Int64Value(nodeInput.MinCount) + gotMaxCount = aws.Int64Value(nodeInput.MaxCount) + _ = msg.Respond(mustMarshal(t, &ec2.Reservation{ + ReservationId: aws.String("r-spot"), + Instances: []*ec2.Instance{ + {InstanceId: aws.String(launchedInstanceIDs[0])}, + {InstanceId: aws.String(launchedInstanceIDs[1])}, + }, + })) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = launchSub.Unsubscribe() }) + + // RequestSpotInstances stamps spot lineage back onto each launched instance's owner + // subject (ec2.cmd.{id}) asynchronously after the response. Capturing every command sent + // lets the write-back be asserted against the real SIR ids RequestSpotInstances just + // minted, rather than the hand-built fixture lineage_test.go uses in isolation. + lineageCmds := make(chan types.EC2InstanceCommand, count) + cmdSub, err := gw.NATSConn.Subscribe("ec2.cmd.>", func(msg *nats.Msg) { + var cmd types.EC2InstanceCommand + if jsonErr := json.Unmarshal(msg.Data, &cmd); jsonErr != nil { + t.Errorf("unmarshal spot lineage command: %v", jsonErr) + return + } + lineageCmds <- cmd + _ = msg.Respond([]byte(`{}`)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cmdSub.Unsubscribe() }) + + ec2c := gw.EC2Client(t) + + reqOut, err := ec2c.RequestSpotInstances(&ec2.RequestSpotInstancesInput{ + InstanceCount: aws.Int64(count), + Type: aws.String(ec2.SpotInstanceTypeOneTime), + LaunchSpecification: &ec2.RequestSpotLaunchSpecification{ + ImageId: aws.String(amiID), + InstanceType: aws.String(instanceType), + }, + }) + require.NoError(t, err, "request-spot-instances --instance-count 2") + require.Lenf(t, reqOut.SpotInstanceRequests, count, + "expected %d spot instance requests, got %d", count, len(reqOut.SpotInstanceRequests)) + + // Proves distributeInstances genuinely expanded the spot count onto the per-node launch + // (rather than the zero-daemon path, which never reaches this subject at all) exactly like + // TestRunInstances_MultiCountExpansion proves for the on-demand call. + assert.Equal(t, int64(count), gotMinCount, "gateway must dispatch MinCount=2 on the per-node launch") + assert.Equal(t, int64(count), gotMaxCount, "gateway must dispatch MaxCount=2 on the per-node launch") + + sirIDs := make([]string, 0, count) + sirForInstance := make(map[string]string, count) + for _, sir := range reqOut.SpotInstanceRequests { + id := aws.StringValue(sir.SpotInstanceRequestId) + require.NotEmpty(t, id, "empty SpotInstanceRequestId in response") + require.Truef(t, strings.HasPrefix(id, "sir-"), "SIR id %q lacks sir- prefix", id) + sirIDs = append(sirIDs, id) + assert.Equal(t, ec2.SpotInstanceStateActive, aws.StringValue(sir.State), + "SIR %s should be active at create", id) + instID := aws.StringValue(sir.InstanceId) + require.NotEmptyf(t, instID, "SIR %s has no InstanceId", id) + sirForInstance[instID] = id + } + + // Describe both SIRs: active + fulfilled + an InstanceId mapped per request. + descOut, err := ec2c.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{ + SpotInstanceRequestIds: aws.StringSlice(sirIDs), + }) + require.NoError(t, err, "describe-spot-instance-requests") + require.Lenf(t, descOut.SpotInstanceRequests, count, + "expected %d SIRs from describe, got %d", count, len(descOut.SpotInstanceRequests)) + for _, sir := range descOut.SpotInstanceRequests { + id := aws.StringValue(sir.SpotInstanceRequestId) + assert.Equal(t, ec2.SpotInstanceStateActive, aws.StringValue(sir.State), "SIR %s state", id) + require.NotNilf(t, sir.Status, "SIR %s missing status", id) + assert.Equal(t, "fulfilled", aws.StringValue(sir.Status.Code), "SIR %s status code", id) + assert.NotEmptyf(t, aws.StringValue(sir.InstanceId), "SIR %s has no InstanceId", id) + } + + // The lineage write-back is best-effort and asynchronous; wait for both commands and + // assert each carries the correct SIR id for its instance — the same link the live test + // proved by reading InstanceLifecycle/SpotInstanceRequestId back off a real running VM. + seen := 0 + deadline := time.After(5 * time.Second) + for seen < count { + select { + case cmd := <-lineageCmds: + wantSIR, ok := sirForInstance[cmd.ID] + require.Truef(t, ok, "lineage command for unexpected instance %s", cmd.ID) + assert.True(t, cmd.Attributes.SetSpotLineage, "instance %s command must set spot lineage", cmd.ID) + require.NotNilf(t, cmd.SpotLineageData, "instance %s command missing spot lineage data", cmd.ID) + assert.Equal(t, wantSIR, cmd.SpotLineageData.SpotInstanceRequestId, + "instance %s should link back to its SIR", cmd.ID) + seen++ + case <-deadline: + t.Fatalf("timed out waiting for spot lineage write-back; got %d/%d", seen, count) + } + } + + cancelSIR, termSIR := sirIDs[0], sirIDs[1] + + // Cancel one request: it flips to cancelled but the instance keeps running (cancel != + // terminate) — CancelSpotInstanceRequests only moves the SIR bucket entry and never calls + // into instance/VM state, so the InstanceId staying attached to the cancelled record is + // the externally-visible proof that cancel leaves the instance alone. + cancelOut, err := ec2c.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{ + SpotInstanceRequestIds: aws.StringSlice([]string{cancelSIR}), + }) + require.NoError(t, err, "cancel-spot-instance-requests %s", cancelSIR) + require.Lenf(t, cancelOut.CancelledSpotInstanceRequests, 1, + "expected 1 cancelled request, got %d", len(cancelOut.CancelledSpotInstanceRequests)) + assert.Equal(t, ec2.CancelSpotInstanceRequestStateCancelled, + aws.StringValue(cancelOut.CancelledSpotInstanceRequests[0].State)) + + cancelled := describeOneSIR(t, ec2c, cancelSIR) + assert.Equal(t, ec2.SpotInstanceStateCancelled, aws.StringValue(cancelled.State), + "SIR %s should be cancelled", cancelSIR) + require.NotNil(t, cancelled.Status) + assert.Equal(t, "request-canceled-and-instance-running", aws.StringValue(cancelled.Status.Code)) + assert.NotEmpty(t, aws.StringValue(cancelled.InstanceId), + "cancelled SIR must keep its InstanceId — the instance itself is untouched") + + // Terminate-triggered close: a live daemon's teardown cleaner calls CloseForInstance + // in-process when a spot-backed instance is terminated (spinifex/daemon/vm_adapters.go + // RemoveFromSpotRequest) — that one-line delegation is already covered by + // TestRemoveFromSpotRequest_NoService_NoOp plus CloseForInstance's own unit tests + // (spinifex/handlers/ec2/spotinstance/service_impl_test.go). What those unit tests can't + // prove is that a SIR minted by the real gateway orchestration — a generated id, keyed by + // the account resolved from a genuine SigV4-authenticated request — closes correctly; this + // does, by calling CloseForInstance directly the same way the teardown cleaner would. + termInstanceID, ok := instanceForSIR(sirForInstance, termSIR) + require.Truef(t, ok, "could not resolve instance for SIR %s", termSIR) + require.NoError(t, spotSvc.CloseForInstance(context.Background(), termInstanceID, gw.AccountID)) + + closed := describeOneSIR(t, ec2c, termSIR) + assert.Equal(t, ec2.SpotInstanceStateClosed, aws.StringValue(closed.State), "SIR %s should be closed", termSIR) + require.NotNil(t, closed.Status) + assert.Equal(t, "instance-terminated-by-user", aws.StringValue(closed.Status.Code)) + + // Insufficient capacity: an oversized request short-circuits before any launch + // (totalCapacity(2) < MinCount(1,000,000), the real placement.go accounting — not the + // zero-node degenerate path, since a node genuinely answered spinifex.node.status) and + // persists no SIRs. + t.Run("InsufficientCapacityNoSIRs", func(t *testing.T) { + out, err := ec2c.RequestSpotInstances(&ec2.RequestSpotInstancesInput{ + InstanceCount: aws.Int64(1_000_000), + LaunchSpecification: &ec2.RequestSpotLaunchSpecification{ + ImageId: aws.String(amiID), + InstanceType: aws.String(instanceType), + }, + }) + requireAWSErrorCode(t, err, "InsufficientInstanceCapacity") + if out != nil { + assert.Empty(t, out.SpotInstanceRequests, "capacity failure must persist/return no SIRs") + } + }) +} + +// instanceForSIR reverse-looks-up the instance ID mapped to sirID in a +// instanceID->SIRID map built from a RequestSpotInstances response. +func instanceForSIR(sirForInstance map[string]string, sirID string) (instanceID string, ok bool) { + for instID, sir := range sirForInstance { + if sir == sirID { + return instID, true + } + } + return "", false +} + +// describeOneSIR returns the single SIR matching sirID, failing the test if +// the gateway does not return exactly one. +func describeOneSIR(t *testing.T, ec2c *ec2.EC2, sirID string) *ec2.SpotInstanceRequest { + t.Helper() + out, err := ec2c.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{ + SpotInstanceRequestIds: aws.StringSlice([]string{sirID}), + }) + require.NoError(t, err, "describe-spot-instance-requests %s", sirID) + require.Lenf(t, out.SpotInstanceRequests, 1, + "expected exactly 1 SIR for %s, got %d", sirID, len(out.SpotInstanceRequests)) + return out.SpotInstanceRequests[0] +}