test: cover DescribeInstanceStatus and the SDK instance-status-ok waiter (#588)

This commit is contained in:
Julian Sommer
2026-07-27 13:46:16 +10:00
committed by GitHub
parent 331e46951f
commit 7e356b5b9b
4 changed files with 424 additions and 0 deletions
+122
View File
@@ -0,0 +1,122 @@
//go:build e2e
package multinode
import (
"fmt"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/require"
)
// runInstanceStatusFanout asserts DescribeInstanceStatus aggregates every
// node's instances rather than only the gateway-local ones. The gateway gathers
// the per-node responses under a 3-second budget and silently returns whatever
// arrived in time, so a truncated fan-out is invisible at the API surface —
// callers see a well-formed response that is simply missing instances. This is
// the only place that failure mode is observable.
func runInstanceStatusFanout(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — DescribeInstanceStatus cross-node fan-out")
trio := needInstanceTrio(t, fix)
// Bracket the describe with two running-instance snapshots: an id in both
// was running for the whole window, so its absence from the status response
// is a truncated fan-out and not a launch or terminate racing the calls.
before := runningInstanceIDs(t, fix)
start := time.Now()
out, err := fix.AWS.EC2.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{})
elapsed := time.Since(start)
require.NoError(t, err, "describe-instance-status")
after := runningInstanceIDs(t, fix)
// Logged on every run, not just on failure: a latency drifting toward the
// gateway's 3-second Gather budget is the leading indicator of the
// truncation this test exists to catch.
harness.Detail(t, "describe_latency", elapsed.Round(time.Millisecond),
"statuses", len(out.InstanceStatuses), "gather_budget", "3s")
reported := make(map[string]bool, len(out.InstanceStatuses))
for _, s := range out.InstanceStatuses {
id := aws.StringValue(s.InstanceId)
require.NotEmpty(t, id, "describe-instance-status returned an entry with no InstanceId")
require.Falsef(t, reported[id], "describe-instance-status returned %s twice", id)
reported[id] = true
}
t.Run("EveryRunningInstanceReported", func(t *testing.T) {
for id := range before {
if !after[id] {
continue
}
require.Truef(t, reported[id],
"instance %s was running before and after the describe but has no status entry "+
"(fan-out truncated by the 3s Gather budget?)", id)
}
})
t.Run("SpansMultipleNodes", func(t *testing.T) {
// Placement is resolved from the qemu process on each node, not from the
// response under test — a gateway answering only locally would otherwise
// look self-consistent. Only the trio is resolved: each lookup costs an
// SSH ps per node, and the trio is the set the fixture staggers across
// the cluster.
placement := make(map[string]string, len(trio))
for _, id := range trio {
if node := harness.InstanceHostingNode(t, fix.Cluster, id); node != nil {
placement[id] = node.Name
}
}
harness.Detail(t, "trio_placement", fmt.Sprintf("%v", placement))
if distinctNodes(placement) < 2 {
t.Skipf("trio placed on %d node(s) (%v); scheduler colocation, not a fan-out signal",
distinctNodes(placement), placement)
}
var missing []string
nodes := make(map[string]struct{}, len(placement))
for id, node := range placement {
if !reported[id] {
missing = append(missing, fmt.Sprintf("%s on %s", id, node))
continue
}
nodes[node] = struct{}{}
}
require.GreaterOrEqualf(t, len(nodes), 2,
"describe-instance-status returned instances from %d node(s), want at least 2 "+
"(placement %v, missing from the response: %v)", len(nodes), placement, missing)
})
}
// runningInstanceIDs returns the set of instance IDs DescribeInstances reports
// as running.
func runningInstanceIDs(t *testing.T, fix *Fixture) map[string]bool {
t.Helper()
out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{{
Name: aws.String("instance-state-name"),
Values: []*string{aws.String("running")},
}},
})
require.NoError(t, err, "describe-instances --filter instance-state-name=running")
ids := make(map[string]bool)
for _, r := range out.Reservations {
for _, inst := range r.Instances {
ids[aws.StringValue(inst.InstanceId)] = true
}
}
return ids
}
// distinctNodes counts the unique node names in an instance-to-node placement map.
func distinctNodes(placement map[string]string) int {
nodes := make(map[string]struct{}, len(placement))
for _, node := range placement {
nodes[node] = struct{}{}
}
return len(nodes)
}
+7
View File
@@ -53,6 +53,13 @@ func TestMultinodeCrossNodeGateway(t *testing.T) {
runCrossNodeGateway(t, requireMultiNodeFixture(t))
}
// TestMultinodeInstanceStatusFanout is sequential and declared before
// CrossNodeOps (which stops trio[0]) so the trio is still spread across nodes
// when the fan-out spread is measured.
func TestMultinodeInstanceStatusFanout(t *testing.T) {
runInstanceStatusFanout(t, requireMultiNodeFixture(t))
}
// TestMultinodeCrossNodeOps is sequential: stops/starts trio[0].
func TestMultinodeCrossNodeOps(t *testing.T) {
runCrossNodeOps(t, requireMultiNodeFixture(t))
+288
View File
@@ -0,0 +1,288 @@
//go:build e2e
package single
import (
"fmt"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// DescribeInstanceStatus vocabulary, duplicated from the handler rather than
// imported: the test asserts the wire contract an AWS SDK client sees, so it
// must fail if the handler's constants change.
const (
instStatusOK = "ok"
instStatusPassed = "passed"
instStatusInitializing = "initializing"
instStatusNotApplicable = "not-applicable"
instStatusReachability = "reachability"
)
// instStatusGraceProbe bounds how long after RunInstances the initializing
// assertion stays a hard requirement. The handler reports initializing for two
// minutes from LaunchTime, so a probe inside this window is inside the grace by
// construction; past it a loaded runner may legitimately have crossed into ok
// and a hard failure would be flaky for a reason unrelated to the contract.
const instStatusGraceProbe = 90 * time.Second
// runInstanceStatus drives DescribeInstanceStatus end to end against a guest
// this test launches and owns. It is sequential: it stops and terminates that
// guest, and the initializing -> ok transition is only observable on a freshly
// launched VM, so the long-booted singleton cannot stand in for it.
//
// The valuable assertion is stage 2 — the real SDK waiter is exactly what
// `aws ec2 wait instance-status-ok` and Terraform's aws_instance creation poll
// run, so a malformed status envelope surfaces here as a hang, not a failed
// call. The remaining stages cover the paths a mock cannot express: the
// running-only state gate, the stopped-instance KV augmentation behind
// IncludeAllInstances, gateway-side dedup, and the status-specific filter map.
func runInstanceStatus(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — DescribeInstanceStatus + SDK status waiter")
instType, _ := needInstanceTypeArch(t, fix)
keyName, _ := needKeyPair(t, fix)
amiID := needAMI(t, fix)
az := needAZ(t, fix)
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
require.NotEmpty(t, vpc.SGID, "default SG ID required")
harness.Step(t, "run-instances ami=%s type=%s (test-owned guest)", amiID, instType)
launchedAt := time.Now()
runOut, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{ // e2e:allow-create — the initializing→ok transition needs a guest launched by this test, not the long-booted singleton.
ImageId: aws.String(amiID),
InstanceType: aws.String(instType),
KeyName: aws.String(keyName),
SubnetId: aws.String(vpc.SubnetID),
SecurityGroupIds: []*string{aws.String(vpc.SGID)},
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
})
require.NoError(t, err, "run-instances")
require.NotEmpty(t, runOut.Instances, "run-instances returned no Instances")
instanceID := aws.StringValue(runOut.Instances[0].InstanceId)
require.NotEmpty(t, instanceID, "run-instances returned empty InstanceId")
harness.Detail(t, "instance", instanceID)
// Registered before the first blocking wait so a fatal anywhere below still
// tears the guest down; the stage-8 terminate makes this a no-op on success.
t.Cleanup(func() {
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(instanceID)},
})
})
harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
// --- Stage 1: a fresh launch reports initializing ---
harness.Step(t, "describe-instance-status %s (fresh launch)", instanceID)
fresh := requireInstanceStatus(t, fix, instanceID,
&ec2.DescribeInstanceStatusInput{InstanceIds: []*string{aws.String(instanceID)}})
freshStatus := aws.StringValue(fresh.InstanceStatus.Status)
sinceLaunch := time.Since(launchedAt)
harness.Detail(t, "fresh_status", freshStatus, "since_launch", sinceLaunch.Round(time.Second))
if sinceLaunch < instStatusGraceProbe {
require.Equalf(t, instStatusInitializing, freshStatus,
"guest %s described %s after launch must still be inside the initializing grace", instanceID, sinceLaunch)
requireReachability(t, fresh.InstanceStatus, instStatusInitializing)
} else {
// Boot outran the probe window on a loaded runner: ok is then a correct
// answer too, so record which was seen rather than failing on timing.
require.Containsf(t, []string{instStatusInitializing, instStatusOK}, freshStatus,
"fresh guest %s reported unexpected InstanceStatus %q", instanceID, freshStatus)
t.Logf("fresh-launch probe ran %s after RunInstances (grace is 2m); observed InstanceStatus=%q",
sinceLaunch.Round(time.Second), freshStatus)
}
// --- Stage 2: the real SDK waiter terminates ---
harness.Step(t, "wait instance-status-ok %s (SDK waiter)", instanceID)
waiterStart := time.Now()
require.NoErrorf(t, fix.AWS.EC2.WaitUntilInstanceStatusOk(&ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
}), "wait instance-status-ok %s", instanceID)
harness.Detail(t, "waiter_elapsed", time.Since(waiterStart).Round(time.Second))
// --- Stage 3: steady-state envelope ---
harness.Step(t, "describe-instance-status %s (steady state)", instanceID)
steady := requireInstanceStatus(t, fix, instanceID,
&ec2.DescribeInstanceStatusInput{InstanceIds: []*string{aws.String(instanceID)}})
assert.Equal(t, instStatusOK, aws.StringValue(steady.InstanceStatus.Status), "InstanceStatus.Status")
instDetail := requireReachability(t, steady.InstanceStatus, instStatusPassed)
// ImpairedSince is only stamped once QMP health crosses the failure gate.
assert.Nil(t, instDetail.ImpairedSince, "healthy guest must not carry ImpairedSince")
assert.Equal(t, instStatusOK, aws.StringValue(steady.SystemStatus.Status), "SystemStatus.Status")
requireReachability(t, steady.SystemStatus, instStatusPassed)
require.NotNil(t, steady.InstanceState, "InstanceState missing")
assert.Equal(t, "running", aws.StringValue(steady.InstanceState.Name), "InstanceState.Name")
assert.Equal(t, int64(16), aws.Int64Value(steady.InstanceState.Code), "InstanceState.Code")
assert.Equal(t, az, aws.StringValue(steady.AvailabilityZone), "AvailabilityZone")
// --- Stage 4: the default describe excludes non-running instances ---
harness.Step(t, "stop-instances %s", instanceID)
_, err = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{
InstanceIds: []*string{aws.String(instanceID)},
})
require.NoError(t, err, "stop-instances %s", instanceID)
harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped")
// The node handler drops the VM from its running-only view asynchronously
// with the KV write that DescribeInstances reads, so poll for absence
// rather than assuming the two flip together.
harness.Step(t, "describe-instance-status %s (default: running only)", instanceID)
harness.EventuallyErr(t, func() error {
statuses := describeInstanceStatuses(t, fix,
&ec2.DescribeInstanceStatusInput{InstanceIds: []*string{aws.String(instanceID)}})
if s := findInstanceStatus(statuses, instanceID); s != nil {
return fmt.Errorf("stopped guest %s still returned by the default describe (state=%s)",
instanceID, aws.StringValue(s.InstanceState.Name))
}
return nil
}, 60*time.Second, 2*time.Second)
// --- Stage 5: IncludeAllInstances surfaces the stopped guest from the KV path ---
harness.Step(t, "describe-instance-status %s --include-all-instances", instanceID)
stopped := requireInstanceStatus(t, fix, instanceID, &ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
IncludeAllInstances: aws.Bool(true),
})
require.NotNil(t, stopped.InstanceState, "InstanceState missing")
assert.Equal(t, "stopped", aws.StringValue(stopped.InstanceState.Name), "InstanceState.Name")
assert.Equal(t, instStatusNotApplicable, aws.StringValue(stopped.InstanceStatus.Status), "InstanceStatus.Status")
requireReachability(t, stopped.InstanceStatus, instStatusNotApplicable)
assert.Equal(t, instStatusNotApplicable, aws.StringValue(stopped.SystemStatus.Status), "SystemStatus.Status")
requireReachability(t, stopped.SystemStatus, instStatusNotApplicable)
// --- Stage 6: no duplicates across the unscoped fan-out + KV union ---
// describeInstanceStatuses fails on any repeated id, so an unscoped
// IncludeAllInstances describe — where the node fan-out and the stopped-KV
// augmentation both contribute — is the widest check on gateway dedup.
harness.Step(t, "describe-instance-status --include-all-instances (unscoped, dedup)")
unscoped := describeInstanceStatuses(t, fix,
&ec2.DescribeInstanceStatusInput{IncludeAllInstances: aws.Bool(true)})
require.NotNilf(t, findInstanceStatus(unscoped, instanceID),
"unscoped IncludeAllInstances describe omitted stopped guest %s", instanceID)
harness.Detail(t, "unscoped_statuses", len(unscoped))
// --- Stage 7: status-specific filters narrow ---
harness.Step(t, "describe-instance-status --filter instance-state-name=stopped")
matched := describeInstanceStatuses(t, fix, &ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
IncludeAllInstances: aws.Bool(true),
Filters: []*ec2.Filter{instStatusFilter("instance-state-name", "stopped")},
})
require.NotNilf(t, findInstanceStatus(matched, instanceID),
"instance-state-name=stopped must return the stopped guest %s", instanceID)
harness.Step(t, "describe-instance-status --filter instance-state-name=running")
unmatched := describeInstanceStatuses(t, fix, &ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
IncludeAllInstances: aws.Bool(true),
Filters: []*ec2.Filter{instStatusFilter("instance-state-name", "running")},
})
require.Nilf(t, findInstanceStatus(unmatched, instanceID),
"instance-state-name=running must not return the stopped guest %s", instanceID)
// Mulga's health model has one static value per status field, so the AWS
// status.* filters are deliberately absent from the valid-filter map.
harness.Step(t, "describe-instance-status --filter instance-status.status (unsupported)")
harness.ExpectError(t, "InvalidParameterValue", func() error {
_, derr := fix.AWS.EC2.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
Filters: []*ec2.Filter{instStatusFilter("instance-status.status", instStatusOK)},
})
return derr
})
// --- Stage 8: terminated instances never appear ---
harness.Step(t, "terminate-instances %s", instanceID)
_, err = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(instanceID)},
})
require.NoError(t, err, "terminate-instances %s", instanceID)
harness.WaitForInstanceState(t, fix.AWS, instanceID, "terminated")
// Termination moves the guest from the stopped bucket to the terminated
// bucket, which DescribeInstanceStatus never reads from — poll so the
// assertion bites on the settled state, not on the move being in flight.
harness.Step(t, "describe-instance-status %s --include-all-instances (terminated)", instanceID)
harness.EventuallyErr(t, func() error {
statuses := describeInstanceStatuses(t, fix, &ec2.DescribeInstanceStatusInput{
InstanceIds: []*string{aws.String(instanceID)},
IncludeAllInstances: aws.Bool(true),
})
if s := findInstanceStatus(statuses, instanceID); s != nil {
return fmt.Errorf("terminated guest %s still returned with IncludeAllInstances (state=%s)",
instanceID, aws.StringValue(s.InstanceState.Name))
}
return nil
}, 60*time.Second, 2*time.Second)
}
// describeInstanceStatuses issues one DescribeInstanceStatus and returns the
// statuses, failing if any InstanceId repeats. Gateway-side dedup is the only
// thing keeping a stop/start race from returning an id twice, and SDK waiters
// assume one entry per id, so every describe in this suite pays for the check.
func describeInstanceStatuses(t *testing.T, fix *Fixture, in *ec2.DescribeInstanceStatusInput) []*ec2.InstanceStatus {
t.Helper()
out, err := fix.AWS.EC2.DescribeInstanceStatus(in)
require.NoError(t, err, "describe-instance-status")
seen := make(map[string]bool, len(out.InstanceStatuses))
for _, s := range out.InstanceStatuses {
id := aws.StringValue(s.InstanceId)
require.NotEmpty(t, id, "describe-instance-status returned an entry with no InstanceId")
require.Falsef(t, seen[id], "describe-instance-status returned %s twice", id)
seen[id] = true
}
return out.InstanceStatuses
}
// requireInstanceStatus runs the describe and returns the entry for instanceID,
// failing if it is absent.
func requireInstanceStatus(t *testing.T, fix *Fixture, instanceID string, in *ec2.DescribeInstanceStatusInput) *ec2.InstanceStatus {
t.Helper()
statuses := describeInstanceStatuses(t, fix, in)
s := findInstanceStatus(statuses, instanceID)
require.NotNilf(t, s, "describe-instance-status returned no entry for %s (%d entries)", instanceID, len(statuses))
return s
}
// findInstanceStatus returns the entry for instanceID, or nil when absent.
func findInstanceStatus(statuses []*ec2.InstanceStatus, instanceID string) *ec2.InstanceStatus {
for _, s := range statuses {
if aws.StringValue(s.InstanceId) == instanceID {
return s
}
}
return nil
}
// requireReachability asserts the summary carries exactly the reachability
// detail at the wanted status, returning it so callers can assert ImpairedSince.
func requireReachability(t *testing.T, summary *ec2.InstanceStatusSummary, want string) *ec2.InstanceStatusDetails {
t.Helper()
require.NotNil(t, summary, "status summary missing")
require.Lenf(t, summary.Details, 1, "expected exactly 1 status detail, got %d", len(summary.Details))
detail := summary.Details[0]
require.Equal(t, instStatusReachability, aws.StringValue(detail.Name), "status detail name")
require.Equal(t, want, aws.StringValue(detail.Status), "reachability detail status")
return detail
}
func instStatusFilter(name string, values ...string) *ec2.Filter {
return &ec2.Filter{Name: aws.String(name), Values: aws.StringSlice(values)}
}
+7
View File
@@ -114,6 +114,13 @@ func TestSpotInstanceLifecycle(t *testing.T) {
runSpotInstanceLifecycle(t, requireSingleNodeFixture(t))
}
// TestInstanceStatus drives DescribeInstanceStatus and the real SDK
// instance-status-ok waiter. Sequential: it launches, stops and terminates its
// own short-lived guest, which the initializing → ok transition requires.
func TestInstanceStatus(t *testing.T) {
runInstanceStatus(t, requireSingleNodeFixture(t))
}
// --- Sequential: shared-state / sub-test-parallel Tests ---
func TestNegativeErrorPaths(t *testing.T) {