spinifex/tests/e2e/harness/lb.go

//go:build e2e

package harness

import (
	"errors"
	"fmt"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/elbv2"
)

// ErrLBTerminalFailed marks a load balancer that entered a terminal failure
// state during provisioning — e.g. the host had no free sys.micro slot for
// the LB VM. Callers can errors.Is on it to retry after capacity reclaim.
var ErrLBTerminalFailed = errors.New("LB entered terminal failure state")

// ErrLBProvisioningTimeout marks a load balancer that never left provisioning
// within the wait window (no terminal failure, just stuck). Callers can
// errors.Is on it to tear down and retry, same as a terminal failure.
var ErrLBProvisioningTimeout = errors.New("LB stuck in provisioning past timeout")

// WaitForLBActive polls describe-load-balancers until state=active. Bails
// immediately if the LB enters a terminal failure state — no point waiting
// the full timeout when provisioning has already given up. Progress emits
// via Step so long polls don't go silent in CI between subtest boundaries.
func WaitForLBActive(t *testing.T, c *AWSClient, lbArn, label string, timeout time.Duration) {
	t.Helper()
	if err := WaitForLBActiveErr(t, c, lbArn, label, timeout); err != nil {
		t.Fatal(err.Error())
	}
}

// WaitForLBActiveErr is WaitForLBActive returning an error instead of failing
// the test, so capacity-race-aware callers can tear down and retry. A terminal
// failure state wraps ErrLBTerminalFailed.
func WaitForLBActiveErr(t *testing.T, c *AWSClient, lbArn, label string, timeout time.Duration) error {
	t.Helper()
	Step(t, "%s: waiting for state=active (timeout %s)", label, timeout)
	var lastState, lastReason string
	deadline := time.Now().Add(timeout)
	nextLog := time.Now().Add(15 * time.Second)
	for {
		out, err := c.ELBv2.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{
			LoadBalancerArns: []*string{aws.String(lbArn)},
		})
		if err == nil && len(out.LoadBalancers) > 0 {
			lastState = aws.StringValue(out.LoadBalancers[0].State.Code)
			lastReason = aws.StringValue(out.LoadBalancers[0].State.Reason)
			if lastState == "active" {
				Step(t, "%s active", label)
				return nil
			}
			if lastState == "failed" || lastState == "provisioning_failed" {
				return fmt.Errorf("%s entered terminal state %s: %s: %w", label, lastState, lastReason, ErrLBTerminalFailed)
			}
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("%s did not become active within %s (last state=%s reason=%s): %w", label, timeout, lastState, lastReason, ErrLBProvisioningTimeout)
		}
		if time.Now().After(nextLog) {
			Step(t, "%s: state=%s, still waiting...", label, lastState)
			nextLog = time.Now().Add(30 * time.Second)
		}
		time.Sleep(3 * time.Second)
	}
}

// WaitForTargetsHealthy polls describe-target-health until expected targets
// report state=healthy or timeout. Returns the final health output for logging.
func WaitForTargetsHealthy(t *testing.T, c *AWSClient, tgArn string, expected int, label string, timeout time.Duration) {
	t.Helper()
	Step(t, "%s: waiting for %d targets healthy (timeout %s)", label, expected, timeout)
	var lastHealthy int
	EventuallyErr(t, func() error {
		out, err := c.ELBv2.DescribeTargetHealth(&elbv2.DescribeTargetHealthInput{
			TargetGroupArn: aws.String(tgArn),
		})
		if err != nil {
			return fmt.Errorf("describe target health %s: %w", label, err)
		}
		lastHealthy = 0
		for _, th := range out.TargetHealthDescriptions {
			if aws.StringValue(th.TargetHealth.State) == "healthy" {
				lastHealthy++
			}
		}
		if lastHealthy >= expected {
			return nil
		}
		return fmt.Errorf("%s: %d/%d healthy", label, lastHealthy, expected)
	}, timeout, 5*time.Second)
	Step(t, "%s: %d targets healthy", label, lastHealthy)
}

// WaitForENICleanup polls describe-network-interfaces until no ENIs match the
// description filter, confirming the LB VM tore down cleanly.
func WaitForENICleanup(t *testing.T, c *AWSClient, descFilter, label string, timeout time.Duration) {
	t.Helper()
	Step(t, "%s: waiting for ENIs to drain (timeout %s)", label, timeout)
	EventuallyErr(t, func() error {
		out, err := c.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
			Filters: []*ec2.Filter{{
				Name:   aws.String("description"),
				Values: []*string{aws.String(descFilter)},
			}},
		})
		if err != nil {
			return fmt.Errorf("describe ENIs for %s: %w", label, err)
		}
		if len(out.NetworkInterfaces) == 0 {
			return nil
		}
		return fmt.Errorf("%s: %d ENIs still present", label, len(out.NetworkInterfaces))
	}, timeout, 3*time.Second)
	Step(t, "%s ENIs cleaned up", label)
}

// WaitForInstanceRunning polls describe-instances until state=running.
// Logs the StateReason on failure to aid debugging stuck launches.
func WaitForInstanceRunning(t *testing.T, c *AWSClient, instanceID string, timeout time.Duration) {
	t.Helper()
	Step(t, "%s: waiting for state=running (timeout %s)", instanceID, timeout)
	var lastState string
	EventuallyErr(t, func() error {
		out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
			InstanceIds: []*string{aws.String(instanceID)},
		})
		if err != nil {
			return fmt.Errorf("describe %s: %w", instanceID, err)
		}
		if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
			return fmt.Errorf("%s not found", instanceID)
		}
		inst := out.Reservations[0].Instances[0]
		lastState = aws.StringValue(inst.State.Name)
		if lastState == "running" {
			return nil
		}
		if lastState == "terminated" {
			reason := aws.StringValue(inst.StateReason.Message)
			return fmt.Errorf("%s terminated: %s", instanceID, reason)
		}
		return fmt.Errorf("%s state=%s", instanceID, lastState)
	}, timeout, 2*time.Second)
	Step(t, "%s running", instanceID)
}

// WaitForInstanceTerminated polls describe-instances until state=terminated.
// Tolerates InvalidInstanceID.NotFound (terminated and reaped).
func WaitForInstanceTerminated(t *testing.T, c *AWSClient, instanceIDs []string, timeout time.Duration) {
	t.Helper()
	if len(instanceIDs) == 0 {
		return
	}
	ids := make([]*string, len(instanceIDs))
	for i, id := range instanceIDs {
		ids[i] = aws.String(id)
	}
	EventuallyErr(t, func() error {
		out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{InstanceIds: ids})
		if err != nil {
			return nil // most likely InvalidInstanceID.NotFound — accept as terminated
		}
		remaining := 0
		for _, r := range out.Reservations {
			for _, inst := range r.Instances {
				if aws.StringValue(inst.State.Name) != "terminated" {
					remaining++
				}
			}
		}
		if remaining == 0 {
			return nil
		}
		return fmt.Errorf("%d instances still terminating", remaining)
	}, timeout, 2*time.Second)
}

// InstanceENI returns the primary ENI for an instance — used to discover the
// public IP assigned to a LB client VM.
func InstanceENI(t *testing.T, c *AWSClient, instanceID string) *ec2.NetworkInterface {
	t.Helper()
	out, err := c.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
		Filters: []*ec2.Filter{{
			Name:   aws.String("attachment.instance-id"),
			Values: []*string{aws.String(instanceID)},
		}},
	})
	if err != nil {
		t.Fatalf("describe ENI for %s: %v", instanceID, err)
	}
	if len(out.NetworkInterfaces) == 0 {
		t.Fatalf("%s has no ENI", instanceID)
	}
	return out.NetworkInterfaces[0]
}