feat: page and validate MaxResults and NextToken on network Describe calls

spinifex/awserrors/awserrors.go

@@ -1012,7 +1012,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorInvalidNetworkLoadBalancerArnNotFound:                 {HTTPCode: 404, Message: "The specified Network Load Balancer ARN does not exist."},
 	ErrorInvalidNextToken:                                      {HTTPCode: 400, Message: "The specified NextToken is not valid."},
 	ErrorInvalidOptionConflict:                                 {HTTPCode: 409, Message: "A VPN connection between the virtual private gateway and the customer gateway already exists."},
-	ErrorInvalidPaginationToken:                                {HTTPCode: 403, Message: "The specified pagination token is not valid or is expired."},
+	ErrorInvalidPaginationToken:                                {HTTPCode: 400, Message: "The specified pagination token is not valid or is expired."},
 	ErrorInvalidParameter:                                      {HTTPCode: 400, Message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
 	ErrorInvalidParameterCombination:                           {HTTPCode: 400, Message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
 	ErrorInvalidParameterDependency:                            {HTTPCode: 400, Message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},

spinifex/handlers/ec2/igw/service_impl.go

@@ -17,6 +17,7 @@ import (
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/mulgadc/spinifex/spinifex/paging"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
@@ -182,6 +183,13 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
 	return &ec2.DeleteInternetGatewayOutput{}, nil
 }
 
+var describeIGWPaging = paging.EC2{
+	MaxResults: 1000,
+	TooLarge:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+	TooSmall:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:    "The parameter InternetGatewayIds cannot be used with the parameter MaxResults",
+}
+
 // describeIGWValidFilters defines the set of filter names accepted by DescribeInternetGateways.
 var describeIGWValidFilters = map[string]bool{
 	"internet-gateway-id": true,
@@ -195,6 +203,11 @@ var describeIGWValidFilters = map[string]bool{
 func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, accountID string) (*ec2.DescribeInternetGatewaysOutput, error) {
 	var igws []*ec2.InternetGateway
 
+	// Validated but not paged: AWS's paging of DescribeInternetGateways is unobserved.
+	if _, err := describeIGWPaging.Parse(input.MaxResults, input.NextToken, len(input.InternetGatewayIds)); err != nil {
+		return nil, err
+	}
+
 	igwIDs := make(map[string]bool)
 	for _, id := range input.InternetGatewayIds {
 		if id != nil {

spinifex/handlers/ec2/igw/service_impl_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/mulgadc/spinifex/spinifex/utils"
@@ -1058,3 +1059,22 @@ func TestAttachInternetGateway_NoGatePublisher_NoOp(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 }
+
+func TestDescribeInternetGateways_PagingValidation(t *testing.T) {
+	svc, _ := setupTestIGWService(t)
+
+	_, err := svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+		MaxResults: aws.Int64(5), InternetGatewayIds: []*string{aws.String("igw-0123456789abcdef0")},
+	}, testAccountID)
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+	assert.Equal(t, "The parameter InternetGatewayIds cannot be used with the parameter MaxResults", message)
+
+	_, err = svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+		NextToken: aws.String("garbage"),
+	}, testAccountID)
+	code, _, ok = awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+}

spinifex/handlers/ec2/routetable/service_impl.go

@@ -19,6 +19,7 @@ import (
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/mulgadc/spinifex/spinifex/paging"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -587,6 +588,13 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
 	return &ec2.DeleteRouteTableOutput{}, nil
 }
 
+var describeRouteTablesPaging = paging.EC2{
+	MaxResults: 100,
+	TooLarge:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 100.",
+	TooSmall:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:    "The parameter RouteTableIds cannot be used with the parameter MaxResults",
+}
+
 var describeRouteTablesValidFilters = map[string]bool{
 	"vpc-id":                                 true,
 	"route-table-id":                         true,
@@ -605,6 +613,11 @@ var describeRouteTablesValidFilters = map[string]bool{
 
 // DescribeRouteTables lists route tables, optionally filtered.
 func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *ec2.DescribeRouteTablesInput, accountID string) (*ec2.DescribeRouteTablesOutput, error) {
+	// Validated but not paged: AWS's paging of DescribeRouteTables is unobserved.
+	if _, err := describeRouteTablesPaging.Parse(input.MaxResults, input.NextToken, len(input.RouteTableIds)); err != nil {
+		return nil, err
+	}
+
 	rtbIDs := make(map[string]bool)
 	for _, id := range input.RouteTableIds {
 		if id != nil {

spinifex/handlers/ec2/routetable/service_impl_test.go

@@ -1495,3 +1495,22 @@ func TestPublishGateDecisionsForVPC_EmptyVPCNoOp(t *testing.T) {
 	assertNoEvents(t, svc, gate, "expected no gate event")
 	assertNoEvents(t, svc, ungate, "expected no ungate event")
 }
+
+func TestDescribeRouteTables_PagingValidation(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	_, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{MaxResults: aws.Int64(101)}, testAccountID)
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+	assert.Equal(t, "Value ( 101 ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 100.", message)
+
+	_, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{
+		MaxResults: aws.Int64(5), RouteTableIds: []*string{aws.String("rtb-0123456789abcdef0")},
+	}, testAccountID)
+	code, message, ok = awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+	assert.Equal(t, "The parameter RouteTableIds cannot be used with the parameter MaxResults", message)
+}

spinifex/handlers/ec2/vpc/eni.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/filterutil"
 	"github.com/mulgadc/spinifex/spinifex/network/topology"
+	"github.com/mulgadc/spinifex/spinifex/paging"
 	"github.com/mulgadc/spinifex/spinifex/tags"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go/jetstream"
@@ -473,6 +474,13 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
 	return &ec2.ModifyNetworkInterfaceAttributeOutput{}, nil
 }
 
+var describeNetworkInterfacesPaging = paging.EC2{
+	MaxResults: 1000,
+	TooLarge:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+	TooSmall:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:    "The parameter NetworkInterfaceIds cannot be used with the parameter MaxResults",
+}
+
 var describeNetworkInterfacesValidFilters = map[string]bool{
 	"network-interface-id":     true,
 	"subnet-id":                true,
@@ -492,6 +500,11 @@ var describeNetworkInterfacesValidFilters = map[string]bool{
 
 // DescribeNetworkInterfaces lists ENIs with optional filters.
 func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *ec2.DescribeNetworkInterfacesInput, accountID string) (*ec2.DescribeNetworkInterfacesOutput, error) {
+	pageReq, err := describeNetworkInterfacesPaging.Parse(input.MaxResults, input.NextToken, len(input.NetworkInterfaceIds))
+	if err != nil {
+		return nil, err
+	}
+
 	parsedFilters, err := filterutil.ParseFilters(input.Filters, describeNetworkInterfacesValidFilters)
 	if err != nil {
 		slog.WarnContext(ctx, "DescribeNetworkInterfaces: invalid filter", "err", err)
@@ -558,10 +571,13 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 		}
 	}
 
+	enis, nextToken := paging.EC2Page(enis, func(e *ec2.NetworkInterface) string { return *e.NetworkInterfaceId }, pageReq)
+
 	slog.InfoContext(ctx, "DescribeNetworkInterfaces completed", "count", len(enis), "accountID", accountID)
 
 	return &ec2.DescribeNetworkInterfacesOutput{
 		NetworkInterfaces: enis,
+		NextToken:         nextToken,
 	}, nil
 }
 

spinifex/handlers/ec2/vpc/paging_test.go

@@ -0,0 +1,146 @@
+package handlers_ec2_vpc
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeSubnets_PagesThroughEverySubnetOnce(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+	want := map[string]bool{}
+	for i := range 7 {
+		want[createTestSubnet(t, svc, vpcID, fmt.Sprintf("10.0.%d.0/24", i))] = true
+	}
+
+	got := map[string]bool{}
+	var token *string
+	pages := 0
+	for {
+		out, err := svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
+			MaxResults: aws.Int64(5),
+			NextToken:  token,
+		}, testAccountID)
+		require.NoError(t, err)
+		require.LessOrEqual(t, len(out.Subnets), 5)
+		for _, s := range out.Subnets {
+			assert.False(t, got[*s.SubnetId], "subnet %s returned twice", *s.SubnetId)
+			got[*s.SubnetId] = true
+		}
+		pages++
+		if out.NextToken == nil {
+			break
+		}
+		token = out.NextToken
+	}
+	assert.Equal(t, want, got)
+	assert.Equal(t, 2, pages)
+}
+
+func TestDescribeSecurityGroups_PagesAfterFiltering(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+	for i := range 6 {
+		createTestSG(t, svc, vpcID, fmt.Sprintf("paged-%d", i))
+	}
+	createTestSG(t, svc, createTestVPC(t, svc, "10.1.0.0/16"), "other-vpc")
+
+	filter := []*ec2.Filter{{Name: aws.String("group-name"), Values: []*string{aws.String("paged-*")}}}
+	first, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+		Filters: filter, MaxResults: aws.Int64(5),
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, first.SecurityGroups, 5)
+	require.NotNil(t, first.NextToken)
+
+	second, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+		Filters: filter, MaxResults: aws.Int64(5), NextToken: first.NextToken,
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, second.SecurityGroups, 1)
+	assert.Nil(t, second.NextToken)
+}
+
+func TestDescribeVpcs_ValidatesPagingButReturnsEverything(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	for i := range 6 {
+		createTestVPC(t, svc, fmt.Sprintf("10.%d.0.0/16", i))
+	}
+
+	out, err := svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{MaxResults: aws.Int64(5)}, testAccountID)
+	require.NoError(t, err)
+	assert.GreaterOrEqual(t, len(out.Vpcs), 6)
+	assert.Nil(t, out.NextToken)
+}
+
+// TestNetworkDescribes_PagingErrors pins each operation to the code and message
+// AWS returns, which differ in wording and casing between operations.
+func TestNetworkDescribes_PagingErrors(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	ctx := context.Background()
+	ids := []*string{aws.String("x-0123456789abcdef0")}
+
+	cases := []struct {
+		name    string
+		call    func() error
+		code    string
+		message string
+	}{
+		{"DescribeVpcs too large", func() error {
+			_, err := svc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{MaxResults: aws.Int64(2000)}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterValue, "Value ( 2000 ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000."},
+		{"DescribeVpcs ids", func() error {
+			_, err := svc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{MaxResults: aws.Int64(5), VpcIds: ids}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterCombination, "The parameter VpcIds cannot be used with the parameter MaxResults"},
+		{"DescribeSubnets too small", func() error {
+			_, err := svc.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{MaxResults: aws.Int64(4)}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterValue, "Value ( 4 ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5."},
+		{"DescribeSubnets bad token", func() error {
+			_, err := svc.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{NextToken: aws.String("garbage")}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterValue, "Value ( garbage ) for parameter NextToken is invalid. The token is invalid."},
+		{"DescribeSecurityGroups too large", func() error {
+			_, err := svc.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{MaxResults: aws.Int64(1001)}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterValue, "Value ( 1001 ) for parameter maxResults is invalid. Expecting a value smaller than 1000."},
+		{"DescribeSecurityGroups ids", func() error {
+			_, err := svc.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{MaxResults: aws.Int64(5), GroupIds: ids}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterCombination, "The parameter securityGroupIdSet cannot be used with the parameter maxResults"},
+		{"DescribeSecurityGroupRules bad token", func() error {
+			_, err := svc.DescribeSecurityGroupRules(ctx, &ec2.DescribeSecurityGroupRulesInput{NextToken: aws.String("garbage")}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidPaginationToken, "Next token 'garbage' is invalid"},
+		{"DescribeSecurityGroupRules ids", func() error {
+			_, err := svc.DescribeSecurityGroupRules(ctx, &ec2.DescribeSecurityGroupRulesInput{MaxResults: aws.Int64(5), SecurityGroupRuleIds: ids}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterCombination, "The parameter 'securityGroupRuleIds' may not be used in combination with 'maxResults'."},
+		{"DescribeNetworkInterfaces ids", func() error {
+			_, err := svc.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{MaxResults: aws.Int64(5), NetworkInterfaceIds: ids}, testAccountID)
+			return err
+		}, awserrors.ErrorInvalidParameterCombination, "The parameter NetworkInterfaceIds cannot be used with the parameter MaxResults"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			code, message, ok := awserrors.ResolveErrorDetail(tc.call())
+			require.True(t, ok)
+			assert.Equal(t, tc.code, code)
+			assert.Equal(t, tc.message, message)
+		})
+	}
+}

spinifex/handlers/ec2/vpc/security_group.go

@@ -18,6 +18,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/filterutil"
+	"github.com/mulgadc/spinifex/spinifex/paging"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -453,10 +454,23 @@ var describeSecurityGroupsValidFilters = map[string]bool{
 	"tag-value":          true,
 }
 
+// AWS accepts 5 and 1000 despite the "smaller than" and "greater than" wording.
+var describeSecurityGroupsPaging = paging.EC2{
+	MaxResults: 1000,
+	TooLarge:   "Value ( %d ) for parameter maxResults is invalid. Expecting a value smaller than 1000.",
+	TooSmall:   "Value ( %d ) for parameter maxResults is invalid. Expecting a value greater than 5.",
+	WithIDs:    "The parameter securityGroupIdSet cannot be used with the parameter maxResults",
+}
+
 // DescribeSecurityGroups lists security groups with optional filters.
 func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error) {
 	groups := []*ec2.SecurityGroup{}
 
+	pageReq, err := describeSecurityGroupsPaging.Parse(input.MaxResults, input.NextToken, len(input.GroupIds))
+	if err != nil {
+		return nil, err
+	}
+
 	groupIDs := make(map[string]bool)
 	for _, id := range input.GroupIds {
 		if id != nil {
@@ -522,10 +536,13 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
 		}
 	}
 
+	groups, nextToken := paging.EC2Page(groups, func(g *ec2.SecurityGroup) string { return *g.GroupId }, pageReq)
+
 	slog.InfoContext(ctx, "DescribeSecurityGroups completed", "count", len(groups), "accountID", accountID)
 
 	return &ec2.DescribeSecurityGroupsOutput{
 		SecurityGroups: groups,
+		NextToken:      nextToken,
 	}, nil
 }
 
@@ -747,25 +764,35 @@ var describeSecurityGroupRulesValidFilters = map[string]bool{
 	"tag-value":              true,
 }
 
+var describeSecurityGroupRulesPaging = paging.EC2{
+	MaxResults:           1000,
+	TooLarge:             "Value ( %d ) for parameter maxResults is invalid. Expecting a value less than or equal to 1000.",
+	TooSmall:             "Value ( %d ) for parameter maxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:              "The parameter 'securityGroupRuleIds' may not be used in combination with 'maxResults'.",
+	PaginationTokenError: true,
+}
+
 // DescribeSecurityGroupRules returns a flat list of SecurityGroupRule objects
 // for the caller's account, optionally narrowed by SecurityGroupRuleIds or
-// filters. MaxResults and NextToken are accepted but ignored.
+// filters, one page at a time.
 func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *ec2.DescribeSecurityGroupRulesInput, accountID string) (*ec2.DescribeSecurityGroupRulesOutput, error) {
-	requested := make(map[string]bool)
-	if input != nil {
-		for _, id := range input.SecurityGroupRuleIds {
-			if id == nil || *id == "" {
-				return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
-			}
-			requested[*id] = true
-		}
+	if input == nil {
+		input = &ec2.DescribeSecurityGroupRulesInput{}
+	}
+	pageReq, err := describeSecurityGroupRulesPaging.Parse(input.MaxResults, input.NextToken, len(input.SecurityGroupRuleIds))
+	if err != nil {
+		return nil, err
 	}
 
-	var filters []*ec2.Filter
-	if input != nil {
-		filters = input.Filters
+	requested := make(map[string]bool)
+	for _, id := range input.SecurityGroupRuleIds {
+		if id == nil || *id == "" {
+			return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
+		}
+		requested[*id] = true
 	}
-	parsedFilters, err := filterutil.ParseFilters(filters, describeSecurityGroupRulesValidFilters)
+
+	parsedFilters, err := filterutil.ParseFilters(input.Filters, describeSecurityGroupRulesValidFilters)
 	if err != nil {
 		slog.WarnContext(ctx, "DescribeSecurityGroupRules: invalid filter", "err", err)
 		return nil, err
@@ -825,10 +852,13 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
 		}
 	}
 
+	rules, nextToken := paging.EC2Page(rules, func(r *ec2.SecurityGroupRule) string { return *r.SecurityGroupRuleId }, pageReq)
+
 	slog.InfoContext(ctx, "DescribeSecurityGroupRules completed", "count", len(rules), "accountID", accountID)
 
 	return &ec2.DescribeSecurityGroupRulesOutput{
 		SecurityGroupRules: rules,
+		NextToken:          nextToken,
 	}, nil
 }
 

spinifex/handlers/ec2/vpc/service_impl.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/filterutil"
 	"github.com/mulgadc/spinifex/spinifex/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/mulgadc/spinifex/spinifex/paging"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -595,6 +596,11 @@ func SupportsDescribeVpcsFilter(name string) bool {
 func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVpcsInput, accountID string) (*ec2.DescribeVpcsOutput, error) {
 	var vpcs []*ec2.Vpc
 
+	// Validated but not paged: AWS's paging of DescribeVpcs is unobserved.
+	if _, err := describeVpcsPaging.Parse(input.MaxResults, input.NextToken, len(input.VpcIds)); err != nil {
+		return nil, err
+	}
+
 	vpcIDs := make(map[string]bool)
 	for _, id := range input.VpcIds {
 		if id != nil {
@@ -967,6 +973,11 @@ func (s *VPCServiceImpl) clearRouteTableAssociationsForSubnet(ctx context.Contex
 func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.DescribeSubnetsInput, accountID string) (*ec2.DescribeSubnetsOutput, error) {
 	var subnets []*ec2.Subnet
 
+	pageReq, err := describeSubnetsPaging.Parse(input.MaxResults, input.NextToken, len(input.SubnetIds))
+	if err != nil {
+		return nil, err
+	}
+
 	subnetIDs := make(map[string]bool)
 	for _, id := range input.SubnetIds {
 		if id != nil {
@@ -1040,10 +1051,13 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 		}
 	}
 
+	subnets, nextToken := paging.EC2Page(subnets, func(s *ec2.Subnet) string { return *s.SubnetId }, pageReq)
+
 	slog.InfoContext(ctx, "DescribeSubnets completed", "count", len(subnets), "accountID", accountID)
 
 	return &ec2.DescribeSubnetsOutput{
-		Subnets: subnets,
+		Subnets:   subnets,
+		NextToken: nextToken,
 	}, nil
 }
 
@@ -1193,6 +1207,20 @@ func vpcMatchesFilters(record *VPCRecord, accountID string, filters map[string][
 	return filterutil.MatchesTags(filters, record.Tags)
 }
 
+var describeVpcsPaging = paging.EC2{
+	MaxResults: 1000,
+	TooLarge:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+	TooSmall:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:    "The parameter VpcIds cannot be used with the parameter MaxResults",
+}
+
+var describeSubnetsPaging = paging.EC2{
+	MaxResults: 1000,
+	TooLarge:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+	TooSmall:   "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+	WithIDs:    "The parameter SubnetIds cannot be used with the parameter MaxResults",
+}
+
 // describeSubnetsValidFilters defines the set of filter names accepted by DescribeSubnets.
 var describeSubnetsValidFilters = map[string]bool{
 	"subnet-id":         true,

spinifex/paging/ec2.go

@@ -0,0 +1,64 @@
+package paging
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// ec2MinResults is the smallest MaxResults the network Describe calls accept.
+const ec2MinResults = 5
+
+// EC2 is one EC2 Describe operation's paging contract. The messages are AWS's
+// own, which differ in wording and casing between operations.
+type EC2 struct {
+	// MaxResults is the largest MaxResults the operation accepts.
+	MaxResults int64
+	// TooLarge and TooSmall answer a MaxResults outside the accepted range;
+	// %d is the requested value.
+	TooLarge string
+	TooSmall string
+	// WithIDs answers MaxResults sent together with explicit resource IDs.
+	WithIDs string
+	// PaginationTokenError selects the InvalidPaginationToken answer to a bad
+	// token that some operations give instead of InvalidParameterValue.
+	PaginationTokenError bool
+}
+
+// Parse validates MaxResults, its combination with ids explicit resource IDs,
+// and NextToken, and returns the page they request.
+func (p EC2) Parse(maxResults *int64, nextToken *string, ids int) (Request, error) {
+	if maxResults != nil {
+		n := *maxResults
+		if n > p.MaxResults {
+			return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, p.TooLarge, n)
+		}
+		if n < ec2MinResults {
+			return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, p.TooSmall, n)
+		}
+		if ids > 0 {
+			return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterCombination, "%s", p.WithIDs)
+		}
+	}
+
+	token := aws.StringValue(nextToken)
+	from, err := DecodeToken(token)
+	if errors.Is(err, ErrInvalidToken) {
+		if p.PaginationTokenError {
+			return Request{}, awserrors.Errorf(awserrors.ErrorInvalidPaginationToken, "Next token '%s' is invalid", token)
+		}
+		return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"Value ( %s ) for parameter NextToken is invalid. The token is invalid.", token)
+	}
+	return Request{Limit: int(aws.Int64Value(maxResults)), From: from}, nil
+}
+
+// EC2Page is Page with the next token in the SDK's form: nil on the last page.
+func EC2Page[T any](items []T, key func(T) string, req Request) ([]T, *string) {
+	page, next := Page(items, key, req)
+	if next == "" {
+		return page, nil
+	}
+	return page, aws.String(next)
+}

spinifex/paging/paging.go

@@ -0,0 +1,64 @@
+// Package paging pages a listing with opaque resume tokens. Each API keeps its
+// own limits and error text; this package only owns the token and the slicing.
+package paging
+
+import (
+	"encoding/base64"
+	"errors"
+	"slices"
+	"strings"
+)
+
+// ErrInvalidToken reports a token this package did not issue.
+var ErrInvalidToken = errors.New("paging: token was not issued by this service")
+
+// Marks a token as ours, so arbitrary base64 is rejected rather than resumed.
+const tokenPrefix = "spx-page-v1:" //nolint:gosec // a format marker, not a credential
+
+// Request is a validated page request. The zero value asks for everything.
+type Request struct {
+	// Limit caps the page; zero or less means no cap.
+	Limit int
+	// From is the key to resume at, decoded from the caller's token.
+	From string
+}
+
+// EncodeToken returns the opaque token that resumes a listing at key.
+func EncodeToken(key string) string {
+	return base64.RawURLEncoding.EncodeToString([]byte(tokenPrefix + key))
+}
+
+// DecodeToken returns the key a token resumes at. An empty token is the start.
+func DecodeToken(token string) (string, error) {
+	if token == "" {
+		return "", nil
+	}
+	decoded, err := base64.RawURLEncoding.DecodeString(token)
+	if err != nil {
+		return "", ErrInvalidToken
+	}
+	key, ok := strings.CutPrefix(string(decoded), tokenPrefix)
+	if !ok || key == "" {
+		return "", ErrInvalidToken
+	}
+	return key, nil
+}
+
+// Page sorts items by key in place and returns the page req selects, plus the
+// token for the next page, or "" on the last page. A resume starts at the first
+// key not below From, so deleting the item a token names skips nothing.
+func Page[T any](items []T, key func(T) string, req Request) ([]T, string) {
+	slices.SortStableFunc(items, func(a, b T) int { return strings.Compare(key(a), key(b)) })
+
+	start := 0
+	if req.From != "" {
+		start, _ = slices.BinarySearchFunc(items, req.From, func(item T, target string) int {
+			return strings.Compare(key(item), target)
+		})
+	}
+	rest := items[start:]
+	if req.Limit <= 0 || len(rest) <= req.Limit {
+		return rest, ""
+	}
+	return rest[:req.Limit], EncodeToken(key(rest[req.Limit]))
+}

spinifex/paging/paging_test.go

@@ -0,0 +1,110 @@
+package paging
+
+import (
+	"encoding/base64"
+	"fmt"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func identity(s string) string { return s }
+
+func TestPage_WalksEveryItemOnceInKeyOrder(t *testing.T) {
+	items := []string{"d", "a", "e", "c", "b"}
+	var seen []string
+	req := Request{Limit: 2}
+	for range len(items) {
+		page, next := Page(items, identity, req)
+		seen = append(seen, page...)
+		if next == "" {
+			break
+		}
+		from, err := DecodeToken(next)
+		require.NoError(t, err)
+		req.From = from
+	}
+	assert.Equal(t, []string{"a", "b", "c", "d", "e"}, seen)
+}
+
+func TestPage_ResumesPastADeletedItem(t *testing.T) {
+	_, next := Page([]string{"a", "b", "c", "d"}, identity, Request{Limit: 2})
+	from, err := DecodeToken(next)
+	require.NoError(t, err)
+	require.Equal(t, "c", from)
+
+	page, _ := Page([]string{"a", "b", "d"}, identity, Request{Limit: 2, From: from})
+	assert.Equal(t, []string{"d"}, page)
+}
+
+func TestPage_NoLimitReturnsEverythingWithoutToken(t *testing.T) {
+	page, next := Page([]string{"b", "a"}, identity, Request{})
+	assert.Equal(t, []string{"a", "b"}, page)
+	assert.Empty(t, next)
+
+	page, next = Page([]string{"a", "b"}, identity, Request{Limit: 2})
+	assert.Len(t, page, 2)
+	assert.Empty(t, next, "an exactly full last page carries no token")
+}
+
+func TestDecodeToken_RejectsTokensItDidNotIssue(t *testing.T) {
+	for _, token := range []string{
+		"garbage!",
+		base64.RawURLEncoding.EncodeToString([]byte("subnet-123")),
+		base64.RawURLEncoding.EncodeToString([]byte(tokenPrefix)),
+	} {
+		_, err := DecodeToken(token)
+		assert.ErrorIs(t, err, ErrInvalidToken, token)
+	}
+}
+
+func requireAWSError(t *testing.T, err error, code, message string) {
+	t.Helper()
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, code, got)
+	assert.Equal(t, message, msg)
+}
+
+func TestEC2Parse(t *testing.T) {
+	p := EC2{
+		MaxResults: 1000,
+		TooLarge:   "too large %d",
+		TooSmall:   "too small %d",
+		WithIDs:    "ids and max",
+	}
+
+	req, err := p.Parse(aws.Int64(1000), aws.String(EncodeToken("subnet-b")), 0)
+	require.NoError(t, err)
+	assert.Equal(t, Request{Limit: 1000, From: "subnet-b"}, req)
+
+	req, err = p.Parse(nil, nil, 2)
+	require.NoError(t, err, "IDs without MaxResults are fine")
+	assert.Equal(t, Request{}, req)
+
+	_, err = p.Parse(aws.Int64(5), nil, 0)
+	require.NoError(t, err)
+
+	_, err = p.Parse(aws.Int64(1001), nil, 0)
+	requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "too large 1001")
+	for _, n := range []int64{4, 0, -1} {
+		_, err = p.Parse(aws.Int64(n), nil, 0)
+		requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, fmt.Sprintf("too small %d", n))
+	}
+
+	_, err = p.Parse(aws.Int64(2000), nil, 1)
+	requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "too large 2000")
+	_, err = p.Parse(aws.Int64(5), nil, 1)
+	requireAWSError(t, err, awserrors.ErrorInvalidParameterCombination, "ids and max")
+
+	_, err = p.Parse(nil, aws.String("garbage"), 0)
+	requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+		"Value ( garbage ) for parameter NextToken is invalid. The token is invalid.")
+
+	p.PaginationTokenError = true
+	_, err = p.Parse(nil, aws.String("garbage"), 0)
+	requireAWSError(t, err, awserrors.ErrorInvalidPaginationToken, "Next token 'garbage' is invalid")
+}