+ 
+
ValidationErrorInvalidParameterValue"+tc.code+"")
+ assert.Contains(t, w.Body.String(),
+ ""+tc.code+"")
+ assert.Contains(t, w.Body.String(), "InvalidParameterCombination")
+ assert.Contains(t, w.Body.String(), "Exception","message":""} body
-// for use by writeClusterUnavailable, writeThrottleError, and ErrorHandler.
-func GenerateEKSErrorResponse(code, message, _ string) []byte {
- return gateway_eks.GenerateEKSErrorResponse(code, message)
-}
-
// jsonErrorType derives the X-Amzn-Errortype header value for code, mirroring
-// GenerateEKSErrorResponse's own "Exception" suffixing so the header and the
+// gateway_eks.GenerateEKSErrorResponse's own "Exception" suffixing so the header and the
// body's __type always agree.
func jsonErrorType(code string) string {
if strings.HasSuffix(code, "Exception") {
@@ -305,6 +300,9 @@ func (gw *GatewayConfig) EKS_Request(w http.ResponseWriter, r *http.Request) err
if err := gw.checkPolicyResources(r, "eks", action, resources); err != nil {
return err
}
+ if err := gw.checkEKSPassRole(r, action, accountID, body); err != nil {
+ return err
+ }
if gw.NATSConn == nil {
return errors.New(awserrors.ErrorServerInternal)
@@ -322,6 +320,39 @@ func (gw *GatewayConfig) EKS_Request(w http.ResponseWriter, r *http.Request) err
return nil
}
+// errForeignRole marks a role ARN naming an account other than the caller's.
+var errForeignRole = errors.New("role is not in the caller's account")
+
+// checkEKSPassRole enforces iam:PassRole on each role action hands to EKS. Only
+// the exact ARN the store holds resolves, so the grant is evaluated against the
+// role's real path and an invented one cannot match a narrower Resource.
+func (gw *GatewayConfig) checkEKSPassRole(r *http.Request, action, accountID string, body []byte) error {
+ for _, roleARN := range gateway_eks.PassedRoleARNs(action, body) {
+ _, _, resolveErr := auth.ResolveRoleARN(roleARN, func(roleAccount, roleName string) (string, error) {
+ if roleAccount != accountID {
+ return "", errForeignRole
+ }
+ return gw.IAMService.CanonicalResourceARN(roleAccount, arn.IAMRole, roleName)
+ })
+ if errors.Is(resolveErr, auth.ErrInvalidRoleARN) || errors.Is(resolveErr, errForeignRole) {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "%s is not a role ARN in account %s", roleARN, accountID)
+ }
+ unknown := errors.Is(resolveErr, auth.ErrRoleARNMismatch) || awserrors.IsErrorCode(resolveErr, awserrors.ErrorIAMNoSuchEntity)
+ if resolveErr != nil && !unknown {
+ return resolveErr
+ }
+ // Evaluated before an unknown role is reported, so a caller without
+ // iam:PassRole cannot use the reply to probe which roles exist.
+ if err := gw.checkPassRole(r, roleARN, eksServicePrincipal); err != nil {
+ return err
+ }
+ if unknown {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "role %s does not exist", roleARN)
+ }
+ }
+ return nil
+}
+
// eksCaller reads the principal behind the request. The role name comes from the
// underlying role ARN, never the session name, which the caller picks at
// AssumeRole time and could name its way past the gate.
diff --git a/spinifex/gateway/eks/passrole.go b/spinifex/gateway/eks/passrole.go
new file mode 100644
index 000000000..8ed6b2962
--- /dev/null
+++ b/spinifex/gateway/eks/passrole.go
@@ -0,0 +1,31 @@
+package gateway_eks
+
+import (
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/eks"
+)
+
+// PassedRoleARNs returns the role ARNs action hands to EKS, each needing
+// iam:PassRole. An undecodable body passes nothing: the handler decodes the
+// same bytes the same way and refuses it before acting.
+func PassedRoleARNs(action string, body []byte) []string {
+ var roleARN *string
+ switch action {
+ case "CreateCluster":
+ input := new(eks.CreateClusterInput)
+ if unmarshalIfBody(body, input) != nil {
+ return nil
+ }
+ roleARN = input.RoleArn
+ case "CreateNodegroup":
+ input := new(eks.CreateNodegroupInput)
+ if unmarshalIfBody(body, input) != nil {
+ return nil
+ }
+ roleARN = input.NodeRole
+ }
+ if aws.StringValue(roleARN) == "" {
+ return nil
+ }
+ return []string{*roleARN}
+}
diff --git a/spinifex/gateway/eks_passrole_test.go b/spinifex/gateway/eks_passrole_test.go
new file mode 100644
index 000000000..1f1a21ab9
--- /dev/null
+++ b/spinifex/gateway/eks_passrole_test.go
@@ -0,0 +1,162 @@
+package gateway
+
+import (
+ "errors"
+ "net/http"
+ "testing"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const eksNodeRoleARN = "arn:aws:iam::" + authzAccountID + ":role/eks/node-role"
+
+const foreignAccountID = "999999999999"
+
+// roleStoreIAMService evaluates policy from docs and resolves role names to the
+// ARNs in roles, keyed by account then name, the way the IAM store does.
+type roleStoreIAMService struct {
+ policyMockIAMService
+
+ roles map[string]map[string]string
+ canonicalErr error
+}
+
+func (s *roleStoreIAMService) CanonicalResourceARN(account string, _ arn.IAMResourceType, name string) (string, error) {
+ if s.canonicalErr != nil {
+ return "", s.canonicalErr
+ }
+ if roleARN, ok := s.roles[account][name]; ok {
+ return roleARN, nil
+ }
+ return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+}
+
+// The foreign account holds a role of the same name, so only the gateway's
+// account check stops its ARN from resolving.
+func eksPassRoleService(statements ...handlers_iam.Statement) *roleStoreIAMService {
+ docs := []handlers_iam.PolicyDocument{{Version: "2012-10-17", Statement: statements}}
+ return &roleStoreIAMService{
+ policyMockIAMService: policyMockIAMService{
+ getUserPoliciesFn: func(_, _ string) ([]handlers_iam.PolicyDocument, error) { return docs, nil },
+ },
+ roles: map[string]map[string]string{
+ authzAccountID: {"node-role": eksNodeRoleARN},
+ foreignAccountID: {"node-role": "arn:aws:iam::" + foreignAccountID + ":role/eks/node-role"},
+ },
+ }
+}
+
+func eksPassRoleGateway(statements ...handlers_iam.Statement) *GatewayConfig {
+ return eksGatewayWithIAM(eksPassRoleService(statements...))
+}
+
+func eksGatewayWithIAM(iamService *roleStoreIAMService) *GatewayConfig {
+ return &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: iamService}
+}
+
+func createNodegroupBody(nodeRole string) string {
+ return `{"nodegroupName":"workers","nodeRole":"` + nodeRole + `"}`
+}
+
+func assertInvalidParameter(t *testing.T, err error) {
+ t.Helper()
+ code, ok := awserrors.ResolveErrorCode(err)
+ require.True(t, ok, "unresolvable error code: %v", err)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+}
+
+// eks:* alone must not let a caller launch nodes under a role it cannot pass.
+func TestEKSCreateNodegroup_DeniedWithoutPassRole(t *testing.T) {
+ gw := eksPassRoleGateway(statement("Allow", "eks:*", "*"))
+ assertDenied(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN)))
+}
+
+func TestEKSCreateNodegroup_AllowedWithPassRoleOnRole(t *testing.T) {
+ gw := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", eksNodeRoleARN),
+ )
+ assertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN)))
+}
+
+// A grant scoped to a path the role is not under must not be satisfied by
+// spelling the role's name under that path.
+func TestEKSCreateNodegroup_InventedPathIsRefused(t *testing.T) {
+ gw := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", "arn:aws:iam::"+authzAccountID+":role/sandbox/*"),
+ )
+ err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
+ createNodegroupBody("arn:aws:iam::"+authzAccountID+":role/sandbox/node-role"))
+ assertInvalidParameter(t, err)
+}
+
+func TestEKSCreateNodegroup_ForeignAccountRoleIsRefused(t *testing.T) {
+ gw := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", "*"),
+ )
+ err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
+ createNodegroupBody("arn:aws:iam::"+foreignAccountID+":role/eks/node-role"))
+ assertInvalidParameter(t, err)
+}
+
+func TestEKSCreateNodegroup_NonRoleARNIsRefused(t *testing.T) {
+ gw := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", "*"),
+ )
+ err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
+ createNodegroupBody("arn:aws:iam::"+authzAccountID+":user/node-role"))
+ assertInvalidParameter(t, err)
+}
+
+// A store fault is not evidence the role is absent, so it must not surface as
+// a caller error or fall through to dispatch.
+func TestEKSCreateNodegroup_LookupFaultPassesThrough(t *testing.T) {
+ storeErr := errors.New("get role: nats: timeout")
+ iamService := eksPassRoleService(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", "*"),
+ )
+ iamService.canonicalErr = storeErr
+ err := dispatchEKS(t, eksGatewayWithIAM(iamService), http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN))
+ require.ErrorIs(t, err, storeErr)
+}
+
+// An unknown role is reported only to a caller allowed to pass it, so the
+// reply cannot be used to enumerate roles.
+func TestEKSCreateNodegroup_UnknownRole(t *testing.T) {
+ body := createNodegroupBody("arn:aws:iam::" + authzAccountID + ":role/ghost")
+
+ denied := eksPassRoleGateway(statement("Allow", "eks:*", "*"))
+ assertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters/prod/node-groups", body))
+
+ allowed := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", "*"),
+ )
+ assertInvalidParameter(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters/prod/node-groups", body))
+}
+
+func TestEKSCreateNodegroup_NoRoleSkipsPassRole(t *testing.T) {
+ gw := eksPassRoleGateway(statement("Allow", "eks:*", "*"))
+ assertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", `{"nodegroupName":"workers"}`))
+}
+
+func TestEKSCreateCluster_PassRole(t *testing.T) {
+ body := `{"name":"prod","roleArn":"` + eksNodeRoleARN + `"}`
+
+ denied := eksPassRoleGateway(statement("Allow", "eks:*", "*"))
+ assertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters", body))
+
+ allowed := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ statement("Allow", "iam:PassRole", eksNodeRoleARN),
+ )
+ assertPermitted(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters", body))
+}
diff --git a/spinifex/gateway/gateway.go b/spinifex/gateway/gateway.go
index cb15467ab..de18144d2 100644
--- a/spinifex/gateway/gateway.go
+++ b/spinifex/gateway/gateway.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"log/slog"
+ "maps"
"net/http"
"net/url"
"strconv"
@@ -32,6 +33,7 @@ import (
gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance"
gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr"
gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth"
+ gateway_eks "github.com/mulgadc/spinifex/spinifex/gateway/eks"
"github.com/mulgadc/spinifex/spinifex/gateway/policy"
gateway_sts "github.com/mulgadc/spinifex/spinifex/gateway/sts"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -396,7 +398,7 @@ func (gw *GatewayConfig) writeClusterUnavailable(w http.ResponseWriter, r *http.
// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
if jsonErrorService(svc) {
- body := GenerateEKSErrorResponse(awserrors.ErrorServiceUnavailable, clusterUnavailableMsg, requestID)
+ body := gateway_eks.GenerateEKSErrorResponse(awserrors.ErrorServiceUnavailable, clusterUnavailableMsg)
w.Header().Set("Content-Type", eksJSONContentType)
w.Header().Set("X-Amzn-Errortype", jsonErrorType(awserrors.ErrorServiceUnavailable))
w.WriteHeader(http.StatusServiceUnavailable)
@@ -428,7 +430,7 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
if jsonErrorService(svc) {
- body := GenerateEKSErrorResponse(errorCode, errorMsg.Message, requestID)
+ body := gateway_eks.GenerateEKSErrorResponse(errorCode, errorMsg.Message)
w.Header().Set("Content-Type", eksJSONContentType)
w.Header().Set("X-Amzn-Errortype", jsonErrorType(errorCode))
w.WriteHeader(errorMsg.HTTPCode)
@@ -577,6 +579,26 @@ func (gw *GatewayConfig) checkPolicy(r *http.Request, service, action string) er
// checkPolicyResources evaluates every resource against one resolved policy
// snapshot. Used when one API request authorizes multiple resource ARNs.
func (gw *GatewayConfig) checkPolicyResources(r *http.Request, service, action string, resources []string) error {
+ return gw.checkPolicyResourcesWithKeys(r, service, action, resources, nil)
+}
+
+// Service principals an iam:PassRole check reports as iam:PassedToService.
+const (
+ ec2ServicePrincipal = "ec2.amazonaws.com"
+ ecsTasksServicePrincipal = "ecs-tasks.amazonaws.com"
+ eksServicePrincipal = "eks.amazonaws.com"
+)
+
+// checkPassRole enforces iam:PassRole on roleARN. iam:PassedToService is fixed
+// by the call site handing the role over, never read from the request.
+func (gw *GatewayConfig) checkPassRole(r *http.Request, roleARN, passedTo string) error {
+ return gw.checkPolicyResourcesWithKeys(r, "iam", "PassRole", []string{roleARN},
+ iampolicy.ConditionKeys{iampolicy.KeyPassedToService: passedTo})
+}
+
+// checkPolicyResourcesWithKeys is checkPolicyResources with action-scoped
+// condition keys layered over the request's own.
+func (gw *GatewayConfig) checkPolicyResourcesWithKeys(r *http.Request, service, action string, resources []string, actionKeys iampolicy.ConditionKeys) error {
// Every dispatcher — query-protocol and REST-JSON alike — reaches this
// point with its resolved action, so telemetry enrichment lives here.
recordResolvedAction(r.Context(), service, action)
@@ -616,8 +638,9 @@ func (gw *GatewayConfig) checkPolicyResources(r *http.Request, service, action s
underlyingRoleARN: mustCtxString(r, ctxUnderlyingRoleARN),
userID: mustCtxString(r, ctxUserID),
}
- return gw.evaluatePrincipalPolicyResources(principal, policy.IAMAction(service, action), resources,
- requestConditionKeys(r, principal))
+ keys := requestConditionKeys(r, principal)
+ maps.Copy(keys, actionKeys)
+ return gw.evaluatePrincipalPolicyResources(principal, policy.IAMAction(service, action), resources, keys)
}
// requestConditionKeys resolves the IAM condition context keys available on the
@@ -656,6 +679,8 @@ func requestConditionKeys(r *http.Request, principal principalContext) iampolicy
if v, ok := principalTypeCondition(principal.principalType); ok {
keys[iampolicy.KeyPrincipalType] = v
}
+ // The server clock, never a request header, as aws:CurrentTime is in AWS.
+ keys.SetRequestTime(time.Now())
return keys
}
@@ -855,7 +880,7 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
// EKS, ECR, ACM, ECS, tagging, and the bedrock family use AWS JSON 1.1;
// query/XML services fall through.
if jsonErrorService(svc) {
- body := GenerateEKSErrorResponse(code, errorMsg.Message, requestId)
+ body := gateway_eks.GenerateEKSErrorResponse(code, errorMsg.Message)
slog.Debug("Generated JSON error response", "service", svc, "error", err, "code", code, "json", string(body), "requestId", requestId)
w.Header().Set("Content-Type", eksJSONContentType)
w.Header().Set("X-Amzn-Errortype", jsonErrorType(code))
diff --git a/spinifex/gateway/gateway_test.go b/spinifex/gateway/gateway_test.go
index b573c51ac..83e61da83 100644
--- a/spinifex/gateway/gateway_test.go
+++ b/spinifex/gateway/gateway_test.go
@@ -22,6 +22,7 @@ import (
"github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+ "github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -1124,9 +1125,10 @@ func TestEC2Request_MissingAccountID(t *testing.T) {
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
}
+// Static attributes are served without NATS; only default-vpc needs a lookup.
func TestEC2Request_DescribeAccountAttributes(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()}
- req := setupEC2Request("Action=DescribeAccountAttributes", "123456789012")
+ req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=max-instances", "123456789012")
w := httptest.NewRecorder()
err := gw.EC2_Request(w, req)
@@ -1140,6 +1142,32 @@ func TestEC2Request_DescribeAccountAttributes(t *testing.T) {
assert.Contains(t, string(body), "DescribeAccountAttributesResponse")
}
+// default-vpc is the caller's own default VPC, asked for with the is-default
+// filter so a deleted default VPC reads as none.
+func TestEC2Request_DescribeAccountAttributes_DefaultVPC(t *testing.T) {
+ nc := startTestNATS(t)
+ sub, err := nc.Subscribe("ec2.DescribeVpcs", func(msg *nats.Msg) {
+ var in awsec2.DescribeVpcsInput
+ if err := json.Unmarshal(msg.Data, &in); err != nil || len(in.Filters) != 1 ||
+ aws.StringValue(in.Filters[0].Name) != "is-default" || msg.Header.Get(utils.AccountIDHeader) != "123456789012" {
+ _ = msg.Respond([]byte(`{}`))
+ return
+ }
+ data, _ := json.Marshal(awsec2.DescribeVpcsOutput{Vpcs: []*awsec2.Vpc{{VpcId: aws.String("vpc-0default"), IsDefault: aws.Bool(true)}}})
+ _ = msg.Respond(data)
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+ gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()}
+ req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=default-vpc", "123456789012")
+ w := httptest.NewRecorder()
+
+ require.NoError(t, gw.EC2_Request(w, req))
+ body, _ := io.ReadAll(w.Result().Body)
+ assert.Contains(t, string(body), "vpc-0default ")
+}
+
func TestEC2Request_DescribeAvailabilityZones(t *testing.T) {
gw := &GatewayConfig{
DisableLogging: true,
diff --git a/spinifex/gateway/iam/group.go b/spinifex/gateway/iam/group.go
index e5f3e9dba..352f88989 100644
--- a/spinifex/gateway/iam/group.go
+++ b/spinifex/gateway/iam/group.go
@@ -34,6 +34,9 @@ func GetGroup(accountID string, input *iam.GetGroupInput, svc handlers_iam.IAMSe
}
func ListGroups(accountID string, input *iam.ListGroupsInput, svc handlers_iam.IAMService) (*iam.ListGroupsOutput, error) {
+ if err := validatePathPrefix(input.PathPrefix, identityPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
@@ -115,6 +118,9 @@ func ListAttachedGroupPolicies(accountID string, input *iam.ListAttachedGroupPol
if input.GroupName == nil || *input.GroupName == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
+ if err := validatePathPrefix(input.PathPrefix, policyPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
diff --git a/spinifex/gateway/iam/instance_profile.go b/spinifex/gateway/iam/instance_profile.go
index 860395249..e07ea57b4 100644
--- a/spinifex/gateway/iam/instance_profile.go
+++ b/spinifex/gateway/iam/instance_profile.go
@@ -34,6 +34,9 @@ func GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput, sv
}
func ListInstanceProfiles(accountID string, input *iam.ListInstanceProfilesInput, svc handlers_iam.IAMService) (*iam.ListInstanceProfilesOutput, error) {
+ if err := validatePathPrefix(input.PathPrefix, identityPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
diff --git a/spinifex/gateway/iam/pagination_test.go b/spinifex/gateway/iam/pagination_test.go
index 877114560..c246ccc3c 100644
--- a/spinifex/gateway/iam/pagination_test.go
+++ b/spinifex/gateway/iam/pagination_test.go
@@ -148,12 +148,12 @@ func TestListPolicyVersions_PagesNewestFirst(t *testing.T) {
return got
}
- first, err := gateway_iam.ListPolicyVersions(testAccountID, &iam.ListPolicyVersionsInput{PolicyArn: aws.String("arn"), MaxItems: aws.Int64(2)}, svc)
+ first, err := gateway_iam.ListPolicyVersions(testAccountID, &iam.ListPolicyVersionsInput{PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), MaxItems: aws.Int64(2)}, svc)
require.NoError(t, err)
assert.Equal(t, []string{"v10", "v9"}, ids(first))
assert.True(t, aws.BoolValue(first.IsTruncated))
- second, err := gateway_iam.ListPolicyVersions(testAccountID, &iam.ListPolicyVersionsInput{PolicyArn: aws.String("arn"), MaxItems: aws.Int64(2), Marker: first.Marker}, svc)
+ second, err := gateway_iam.ListPolicyVersions(testAccountID, &iam.ListPolicyVersionsInput{PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), MaxItems: aws.Int64(2), Marker: first.Marker}, svc)
require.NoError(t, err)
assert.Equal(t, []string{"v2"}, ids(second))
assert.False(t, aws.BoolValue(second.IsTruncated))
@@ -169,7 +169,7 @@ func TestListEntitiesForPolicy_PagesAcrossAllThreeLists(t *testing.T) {
PolicyGroups: []*iam.PolicyGroup{{GroupName: aws.String("g1")}},
}
}}
- input := &iam.ListEntitiesForPolicyInput{PolicyArn: aws.String("arn"), MaxItems: aws.Int64(2)}
+ input := &iam.ListEntitiesForPolicyInput{PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), MaxItems: aws.Int64(2)}
first, err := gateway_iam.ListEntitiesForPolicy(testAccountID, input, svc)
require.NoError(t, err)
diff --git a/spinifex/gateway/iam/pathprefix.go b/spinifex/gateway/iam/pathprefix.go
new file mode 100644
index 000000000..51b456554
--- /dev/null
+++ b/spinifex/gateway/iam/pathprefix.go
@@ -0,0 +1,34 @@
+package gateway_iam
+
+import (
+ "regexp"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+const maxPathPrefixLength = 512
+
+// The three PathPrefix shapes in the IAM model: identity lists, policy lists
+// (a whole path, trailing slash included), and ListEntitiesForPolicy.
+var (
+ identityPathPrefix = regexp.MustCompile(`^/[\x21-\x7F]*$`)
+ policyPathPrefix = regexp.MustCompile(`^(/[A-Za-z0-9.,+@=_-]+)*/$`)
+ entityPathPrefix = regexp.MustCompile(`^(/|/[\x21-\x7E]+/)$`)
+)
+
+// validatePathPrefix checks an optional PathPrefix as AWS does: the pattern
+// first, with IAM's own message, then the model's length limit.
+func validatePathPrefix(prefix *string, pattern *regexp.Regexp) error {
+ if prefix == nil {
+ return nil
+ }
+ if !pattern.MatchString(*prefix) {
+ return awserrors.Errorf(awserrors.ErrorValidationError,
+ "The specified value for pathPrefix is invalid. It must begin with the / character and contain only alphanumeric characters and/or / characters.")
+ }
+ if len(*prefix) > maxPathPrefixLength {
+ return awserrors.Errorf(awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'pathPrefix' failed to satisfy constraint: Member must have length less than or equal to %d", maxPathPrefixLength)
+ }
+ return nil
+}
diff --git a/spinifex/gateway/iam/pathprefix_test.go b/spinifex/gateway/iam/pathprefix_test.go
new file mode 100644
index 000000000..36d89d620
--- /dev/null
+++ b/spinifex/gateway/iam/pathprefix_test.go
@@ -0,0 +1,118 @@
+package gateway_iam_test
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/iam"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ gateway_iam "github.com/mulgadc/spinifex/spinifex/gateway/iam"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// The messages are the ones AWS returned for the same PathPrefix.
+const (
+ badPathPrefix = "The specified value for pathPrefix is invalid. It must begin with the / character and contain only alphanumeric characters and/or / characters."
+ longPathPrefix = "1 validation error detected: Value at 'pathPrefix' failed to satisfy constraint: Member must have length less than or equal to 512"
+)
+
+// Each list is refused before the service runs; the stub panics if called.
+func TestListCalls_RejectPathPrefixBeforeListing(t *testing.T) {
+ lists := map[string]func(prefix string) error{
+ "ListUsers": func(p string) error {
+ _, err := gateway_iam.ListUsers(testAccountID, &iam.ListUsersInput{PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListRoles": func(p string) error {
+ _, err := gateway_iam.ListRoles(testAccountID, &iam.ListRolesInput{PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListGroups": func(p string) error {
+ _, err := gateway_iam.ListGroups(testAccountID, &iam.ListGroupsInput{PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListInstanceProfiles": func(p string) error {
+ _, err := gateway_iam.ListInstanceProfiles(testAccountID, &iam.ListInstanceProfilesInput{PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListPolicies": func(p string) error {
+ _, err := gateway_iam.ListPolicies(testAccountID, &iam.ListPoliciesInput{PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListEntitiesForPolicy": func(p string) error {
+ _, err := gateway_iam.ListEntitiesForPolicy(testAccountID, &iam.ListEntitiesForPolicyInput{
+ PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), PathPrefix: aws.String(p),
+ }, &listStub{})
+ return err
+ },
+ "ListAttachedUserPolicies": func(p string) error {
+ _, err := gateway_iam.ListAttachedUserPolicies(testAccountID, &iam.ListAttachedUserPoliciesInput{UserName: aws.String("u"), PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListAttachedRolePolicies": func(p string) error {
+ _, err := gateway_iam.ListAttachedRolePolicies(testAccountID, &iam.ListAttachedRolePoliciesInput{RoleName: aws.String("r"), PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ "ListAttachedGroupPolicies": func(p string) error {
+ _, err := gateway_iam.ListAttachedGroupPolicies(testAccountID, &iam.ListAttachedGroupPoliciesInput{GroupName: aws.String("g"), PathPrefix: aws.String(p)}, &listStub{})
+ return err
+ },
+ }
+ cases := []struct {
+ name string
+ prefix string
+ wantMsg string
+ }{
+ {"empty", "", badPathPrefix},
+ {"no leading slash", "awsdiff", badPathPrefix},
+ {"space", "/bad path/", badPathPrefix},
+ {"over 512 characters", "/" + strings.Repeat("a", 511) + "/", longPathPrefix},
+ }
+ for op, list := range lists {
+ for _, tc := range cases {
+ t.Run(op+"/"+tc.name, func(t *testing.T) {
+ err := list(tc.prefix)
+ require.Error(t, err)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error must carry a registered code: %v", err)
+ assert.Equal(t, awserrors.ErrorValidationError, code)
+ assert.Equal(t, tc.wantMsg, msg)
+ })
+ }
+ }
+}
+
+func TestListUsers_PathPrefixAccepted(t *testing.T) {
+ for _, prefix := range []string{"/", "/~/", "/team", "/" + strings.Repeat("a", 511)} {
+ svc := &listStub{users: []string{"a"}}
+ _, err := gateway_iam.ListUsers(testAccountID, &iam.ListUsersInput{PathPrefix: aws.String(prefix)}, svc)
+ require.NoError(t, err, prefix)
+ assert.Equal(t, 1, svc.calls)
+ }
+}
+
+// Policy paths end in a slash, so a prefix without one breaks the pattern
+// before its length is checked.
+func TestListPolicies_PathPrefixPatternBeforeLength(t *testing.T) {
+ _, err := gateway_iam.ListPolicies(testAccountID, &iam.ListPoliciesInput{PathPrefix: aws.String("/" + strings.Repeat("a", 512))}, &listStub{})
+ _, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error must carry a registered code: %v", err)
+ assert.Equal(t, badPathPrefix, msg)
+}
+
+// The messages are the ones AWS returned for a 19- and a 2049-character ARN.
+func TestListEntitiesForPolicy_PolicyArnLength(t *testing.T) {
+ cases := map[string]string{
+ "arn:aws:iam::aws:x": "1 validation error detected: Value at 'policyArn' failed to satisfy constraint: Member must have length greater than or equal to 20",
+ "arn:aws:iam::aws:policy/" + strings.Repeat("p", 2025): "1 validation error detected: Value at 'policyArn' failed to satisfy constraint: Member must have length less than or equal to 2048",
+ }
+ for arn, want := range cases {
+ _, err := gateway_iam.ListEntitiesForPolicy(testAccountID, &iam.ListEntitiesForPolicyInput{PolicyArn: aws.String(arn)}, &listStub{})
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error must carry a registered code: %v", err)
+ assert.Equal(t, awserrors.ErrorValidationError, code)
+ assert.Equal(t, want, msg)
+ }
+}
diff --git a/spinifex/gateway/iam/policy.go b/spinifex/gateway/iam/policy.go
index 8df2d3051..2a20fa68e 100644
--- a/spinifex/gateway/iam/policy.go
+++ b/spinifex/gateway/iam/policy.go
@@ -2,6 +2,7 @@ package gateway_iam
import (
"errors"
+ "fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
@@ -9,6 +10,12 @@ import (
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
)
+// The model's arnType length bounds.
+const (
+ minARNLength = 20
+ maxARNLength = 2048
+)
+
func CreatePolicy(accountID string, input *iam.CreatePolicyInput, svc handlers_iam.IAMService) (*iam.CreatePolicyOutput, error) {
if input.PolicyName == nil || *input.PolicyName == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -91,6 +98,9 @@ func DeletePolicyVersion(accountID string, input *iam.DeletePolicyVersionInput,
}
func ListPolicies(accountID string, input *iam.ListPoliciesInput, svc handlers_iam.IAMService) (*iam.ListPoliciesOutput, error) {
+ if err := validatePathPrefix(input.PathPrefix, policyPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
@@ -115,6 +125,17 @@ func ListEntitiesForPolicy(accountID string, input *iam.ListEntitiesForPolicyInp
if input.PolicyArn == nil || *input.PolicyArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
+ if n := len(*input.PolicyArn); n < minARNLength || n > maxARNLength {
+ bound := fmt.Sprintf("greater than or equal to %d", minARNLength)
+ if n > maxARNLength {
+ bound = fmt.Sprintf("less than or equal to %d", maxARNLength)
+ }
+ return nil, awserrors.Errorf(awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'policyArn' failed to satisfy constraint: Member must have length %s", bound)
+ }
+ if err := validatePathPrefix(input.PathPrefix, entityPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
@@ -187,6 +208,9 @@ func ListAttachedUserPolicies(accountID string, input *iam.ListAttachedUserPolic
if input.UserName == nil || *input.UserName == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
+ if err := validatePathPrefix(input.PathPrefix, policyPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
diff --git a/spinifex/gateway/iam/role.go b/spinifex/gateway/iam/role.go
index 7d86e7a07..237efa6fc 100644
--- a/spinifex/gateway/iam/role.go
+++ b/spinifex/gateway/iam/role.go
@@ -37,6 +37,9 @@ func GetRole(accountID string, input *iam.GetRoleInput, svc handlers_iam.IAMServ
}
func ListRoles(accountID string, input *iam.ListRolesInput, svc handlers_iam.IAMService) (*iam.ListRolesOutput, error) {
+ if err := validatePathPrefix(input.PathPrefix, identityPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
@@ -99,6 +102,9 @@ func ListAttachedRolePolicies(accountID string, input *iam.ListAttachedRolePolic
if input.RoleName == nil || *input.RoleName == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
+ if err := validatePathPrefix(input.PathPrefix, policyPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
diff --git a/spinifex/gateway/iam/user.go b/spinifex/gateway/iam/user.go
index be5042d91..22f8d8cc7 100644
--- a/spinifex/gateway/iam/user.go
+++ b/spinifex/gateway/iam/user.go
@@ -24,6 +24,9 @@ func GetUser(accountID string, input *iam.GetUserInput, svc handlers_iam.IAMServ
}
func ListUsers(accountID string, input *iam.ListUsersInput, svc handlers_iam.IAMService) (*iam.ListUsersOutput, error) {
+ if err := validatePathPrefix(input.PathPrefix, identityPathPrefix); err != nil {
+ return nil, err
+ }
p, err := newPager(input.Marker, input.MaxItems)
if err != nil {
return nil, err
diff --git a/spinifex/gateway/notselectors_test.go b/spinifex/gateway/notselectors_test.go
new file mode 100644
index 000000000..86f4b673e
--- /dev/null
+++ b/spinifex/gateway/notselectors_test.go
@@ -0,0 +1,37 @@
+//test:in-package — drives EC2_Request through the gateway's unexported test
+// helpers (scopedPolicyGateway, dispatchEC2).
+
+package gateway
+
+import (
+ "testing"
+
+ handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+)
+
+// The PowerUserAccess shape: an Allow on everything except the listed actions.
+func TestEC2Request_AllowNotActionGrantsTheComplement(t *testing.T) {
+ gw := scopedPolicyGateway(handlers_iam.Statement{
+ Effect: "Allow",
+ NotAction: handlers_iam.StringOrArr{"iam:*", "ec2:TerminateInstances"},
+ Resource: handlers_iam.StringOrArr{"*"},
+ })
+
+ assertPermitted(t, dispatchEC2(t, gw, "Action=StopInstances&InstanceId.1=i-dev"))
+ assertUnauthorized(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-dev"))
+}
+
+// A Deny on NotResource fences every instance but the named one.
+func TestEC2Request_DenyNotResourceFencesTheComplement(t *testing.T) {
+ gw := scopedPolicyGateway(
+ statement("Allow", "ec2:*", "*"),
+ handlers_iam.Statement{
+ Effect: "Deny",
+ Action: handlers_iam.StringOrArr{"ec2:TerminateInstances"},
+ NotResource: handlers_iam.StringOrArr{"arn:aws:ec2:*:*:instance/i-dev"},
+ },
+ )
+
+ assertPermitted(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-dev"))
+ assertUnauthorized(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-prod"))
+}
diff --git a/spinifex/gateway/oidc_discovery_test.go b/spinifex/gateway/oidc_discovery_test.go
index a86a23cac..ad1d370ac 100644
--- a/spinifex/gateway/oidc_discovery_test.go
+++ b/spinifex/gateway/oidc_discovery_test.go
@@ -23,7 +23,7 @@ func seedDiscoveryCluster(t *testing.T) http.Handler {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, testDiscAccount, 1)
+ kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, testDiscAccount)
if err != nil {
t.Fatalf("account bucket: %v", err)
}
diff --git a/spinifex/gateway/passedtoservice_test.go b/spinifex/gateway/passedtoservice_test.go
new file mode 100644
index 000000000..a8ad74056
--- /dev/null
+++ b/spinifex/gateway/passedtoservice_test.go
@@ -0,0 +1,143 @@
+package gateway
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/mulgadc/bluebottle/pkg/iampolicy"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// passRoleTo is a PassRole statement on every role, conditioned on the
+// consuming service the way AWS's documented examples scope it.
+func passRoleTo(effect, operator, service string) handlers_iam.Statement {
+ s := statement(effect, "iam:PassRole", "*")
+ s.Condition = map[string]map[string]handlers_iam.ConditionValue{
+ operator: {iampolicy.KeyPassedToService: {service}},
+ }
+ return s
+}
+
+func passRoleToGateway(t *testing.T, statements ...handlers_iam.Statement) *GatewayConfig {
+ t.Helper()
+ docs := []handlers_iam.PolicyDocument{{Version: "2012-10-17", Statement: statements}}
+ return &GatewayConfig{
+ DisableLogging: true,
+ Region: authzRegion,
+ NATSConn: startTestNATS(t),
+ IAMService: &profileIAMService{
+ policyMockIAMService: policyMockIAMService{
+ getUserPoliciesFn: func(_, _ string) ([]handlers_iam.PolicyDocument, error) { return docs, nil },
+ },
+ },
+ }
+}
+
+func associateProfile(t *testing.T, gw *GatewayConfig) error {
+ t.Helper()
+ return dispatchEC2(t, gw, "Action=AssociateIamInstanceProfile&InstanceId=i-dev&IamInstanceProfile.Name=web")
+}
+
+func registerTaskDefinition(t *testing.T, gw *GatewayConfig) error {
+ t.Helper()
+ req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
+ registerTaskDefinitionBody(t, testECSPassRoleTargetARN))
+ return gw.ECS_Request(httptest.NewRecorder(), req)
+}
+
+func createNodegroup(t *testing.T, gw *GatewayConfig) error {
+ t.Helper()
+ return dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN))
+}
+
+func assertNotUnauthorized(t *testing.T, err error) {
+ t.Helper()
+ require.Error(t, err)
+ assert.NotEqual(t, awserrors.ErrorUnauthorizedOperation, err.Error())
+}
+
+// Each EC2 action that hands over an instance profile builds its own PassRole
+// check, so every one is pinned to ec2.amazonaws.com.
+func TestPassedToService_EC2(t *testing.T) {
+ tests := []struct {
+ action string
+ body string
+ }{
+ {"RunInstances", "Action=RunInstances&ImageId=ami-1&InstanceType=t3.micro&MinCount=1&MaxCount=1&IamInstanceProfile.Name=web"},
+ {"AssociateIamInstanceProfile", "Action=AssociateIamInstanceProfile&InstanceId=i-dev&IamInstanceProfile.Name=web"},
+ {"ReplaceIamInstanceProfileAssociation", "Action=ReplaceIamInstanceProfileAssociation" +
+ "&AssociationId=iip-assoc-dev&IamInstanceProfile.Name=web"},
+ {"RequestSpotInstances", "Action=RequestSpotInstances&LaunchSpecification.ImageId=ami-1" +
+ "&LaunchSpecification.InstanceType=t3.micro&LaunchSpecification.IamInstanceProfile.Name=web"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.action, func(t *testing.T) {
+ assertNotUnauthorized(t, dispatchEC2(t, passRoleToGateway(t,
+ statement("Allow", "ec2:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringEquals, "ec2.amazonaws.com"),
+ ), tt.body))
+ assertUnauthorized(t, dispatchEC2(t, passRoleToGateway(t,
+ statement("Allow", "ec2:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringEquals, "ecs-tasks.amazonaws.com"),
+ ), tt.body))
+ })
+ }
+}
+
+func TestPassedToService_ECS(t *testing.T) {
+ assertNotDenied(t, registerTaskDefinition(t, passRoleToGateway(t,
+ statement("Allow", "ecs:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringLike, "ecs-tasks.*"),
+ )))
+ assertDenied(t, registerTaskDefinition(t, passRoleToGateway(t,
+ statement("Allow", "ecs:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringEquals, "ec2.amazonaws.com"),
+ )))
+}
+
+// AWS reports a nodegroup's node role as passed to EKS, not to the EC2
+// instances that end up running under it.
+func TestPassedToService_EKS(t *testing.T) {
+ allowed := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringEquals, "eks.amazonaws.com"),
+ )
+ assertPermitted(t, createNodegroup(t, allowed))
+ assertPermitted(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters",
+ `{"name":"prod","roleArn":"`+eksNodeRoleARN+`"}`))
+
+ denied := eksPassRoleGateway(
+ statement("Allow", "eks:*", "*"),
+ passRoleTo("Allow", iampolicy.OpStringEquals, "ec2.amazonaws.com"),
+ )
+ assertDenied(t, createNodegroup(t, denied))
+ assertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters",
+ `{"name":"prod","roleArn":"`+eksNodeRoleARN+`"}`))
+}
+
+// A Deny scoped to one service fences that service and leaves the others to
+// the unconditional grant.
+func TestPassedToService_DenyFencesOneService(t *testing.T) {
+ statements := []handlers_iam.Statement{
+ statement("Allow", "ec2:*", "*"),
+ statement("Allow", "ecs:*", "*"),
+ statement("Allow", "iam:PassRole", "*"),
+ passRoleTo("Deny", iampolicy.OpStringEquals, "ecs-tasks.amazonaws.com"),
+ }
+ assertDenied(t, registerTaskDefinition(t, passRoleToGateway(t, statements...)))
+ assertNotUnauthorized(t, associateProfile(t, passRoleToGateway(t, statements...)))
+}
+
+// The key exists only on a PassRole check, so a condition on it attached to
+// any other action does not hold.
+func TestPassedToService_AbsentOutsidePassRole(t *testing.T) {
+ grant := statement("Allow", "ec2:*", "*")
+ grant.Condition = map[string]map[string]handlers_iam.ConditionValue{
+ iampolicy.OpStringEquals: {iampolicy.KeyPassedToService: {"ec2.amazonaws.com"}},
+ }
+ assertUnauthorized(t, dispatchEC2(t, scopedPolicyGateway(grant), "Action=DeleteVolume"))
+}
diff --git a/spinifex/gateway/rds/enginecatalog.go b/spinifex/gateway/rds/enginecatalog.go
index dec2f7410..0a4fc4fbb 100644
--- a/spinifex/gateway/rds/enginecatalog.go
+++ b/spinifex/gateway/rds/enginecatalog.go
@@ -17,9 +17,8 @@ import (
"github.com/nats-io/nats.go"
)
-// The filter names each action recognises. Rejecting an unknown name is only
-// implementable against a closed vocabulary, and the two actions take disjoint
-// typed parameters, so each keeps its own list rather than sharing one.
+// The filter names each action recognises, compared case-sensitively as AWS
+// does. The two actions take disjoint typed parameters, so each keeps its own list.
const (
filterNameEngine = "engine"
filterNameEngineVersion = "engine-version"
@@ -43,46 +42,42 @@ var (
// filter here rather than a required parameter, and an unknown one is an empty
// list rather than the rejection create-db-instance gives it.
func DescribeDBEngineVersions(ctx context.Context, input *rds.DescribeDBEngineVersionsInput, _ *nats.Conn, _ Caller) (any, error) {
- if err := rejectMarker(input.Marker, "DescribeDBEngineVersions"); err != nil {
- return nil, err
- }
-
filter := handlers_rds.EngineVersionFilter{}
filter.Engine.AddParam(aws.StringValue(input.Engine))
filter.EngineVersion.AddParam(aws.StringValue(input.EngineVersion))
filter.ParameterGroupFamily.AddParam(aws.StringValue(input.DBParameterGroupFamily))
- for _, entry := range input.Filters {
- name, values, err := filterEntry(entry)
- if err != nil {
- return nil, err
- }
- switch name {
+ entries, err := handlers_rds.ReadFilters(input.Filters, engineVersionFilterNames...)
+ if err != nil {
+ return nil, err
+ }
+ for _, entry := range entries {
+ switch entry.Name {
case filterNameEngine:
- filter.Engine.AddFilter(values)
+ filter.Engine.AddFilter(entry.Values)
case filterNameEngineVersion:
- filter.EngineVersion.AddFilter(values)
+ filter.EngineVersion.AddFilter(entry.Values)
case filterNameParameterGroupFamily:
- filter.ParameterGroupFamily.AddFilter(values)
+ filter.ParameterGroupFamily.AddFilter(entry.Values)
case filterNameStatus:
- filter.Status.AddFilter(values)
- default:
- return nil, unknownFilterName(name, engineVersionFilterNames)
+ filter.Status.AddFilter(entry.Values)
}
}
// DefaultOnly, IncludeAll, ListSupportedCharacterSets and ListSupportedTimezones
// are accepted and not read: each is an identity on a catalog of one available
// version per engine with no character-set or timezone list to populate.
- return &rds.DescribeDBEngineVersionsOutput{DBEngineVersions: handlers_rds.EngineVersions(filter)}, nil
+ versions, marker, err := handlers_rds.Page(handlers_rds.EngineVersions(filter),
+ handlers_rds.EngineVersionPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBEngineVersionsOutput{DBEngineVersions: versions, Marker: marker}, nil
}
// Engine is required, so an absent one is MissingParameter and an unknown one is
// the InvalidParameterValue LookupEngine already words. Everything else narrows.
func DescribeOrderableDBInstanceOptions(ctx context.Context, input *rds.DescribeOrderableDBInstanceOptionsInput, nc *nats.Conn, _ Caller, env Env) (any, error) {
- if err := rejectMarker(input.Marker, "DescribeOrderableDBInstanceOptions"); err != nil {
- return nil, err
- }
if aws.StringValue(input.AvailabilityZoneGroup) != "" {
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"AvailabilityZoneGroup is not supported: this platform exposes a single zone and names none")
@@ -104,28 +99,26 @@ func DescribeOrderableDBInstanceOptions(ctx context.Context, input *rds.Describe
filter.Vpc.AddParam(strconv.FormatBool(aws.BoolValue(input.Vpc)))
}
- for _, entry := range input.Filters {
- name, values, err := filterEntry(entry)
- if err != nil {
- return nil, err
- }
- switch name {
+ entries, err := handlers_rds.ReadFilters(input.Filters, orderableFilterNames...)
+ if err != nil {
+ return nil, err
+ }
+ for _, entry := range entries {
+ switch entry.Name {
case filterNameEngine:
- filter.Engine.AddFilter(values)
+ filter.Engine.AddFilter(entry.Values)
case filterNameEngineVersion:
- filter.EngineVersion.AddFilter(values)
+ filter.EngineVersion.AddFilter(entry.Values)
case filterNameDBInstanceClass:
- filter.DBInstanceClass.AddFilter(values)
+ filter.DBInstanceClass.AddFilter(entry.Values)
case filterNameLicenseModel:
- filter.LicenseModel.AddFilter(values)
+ filter.LicenseModel.AddFilter(entry.Values)
case filterNameVpc:
- parsed, perr := boolFilterValues(name, values)
+ parsed, perr := boolFilterValues(entry.Name, entry.Values)
if perr != nil {
return nil, perr
}
filter.Vpc.AddFilter(parsed)
- default:
- return nil, unknownFilterName(name, orderableFilterNames)
}
}
@@ -133,9 +126,12 @@ func DescribeOrderableDBInstanceOptions(ctx context.Context, input *rds.Describe
if err != nil {
return nil, err
}
- return &rds.DescribeOrderableDBInstanceOptionsOutput{
- OrderableDBInstanceOptions: handlers_rds.OrderableOptions(filter, runnable),
- }, nil
+ options, marker, err := handlers_rds.Page(handlers_rds.OrderableOptions(filter, runnable),
+ handlers_rds.OrderableOptionPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeOrderableDBInstanceOptionsOutput{OrderableDBInstanceOptions: options, Marker: marker}, nil
}
// Which EC2 instance types the cluster's nodes report they can run, as a
@@ -150,7 +146,7 @@ func DescribeOrderableDBInstanceOptions(ctx context.Context, input *rds.Describe
// collapse both onto the same answer.
func clusterRunnableTypes(ctx context.Context, nc *nats.Conn, env Env) (func(string) bool, error) {
out, err := gateway_ec2_instance.DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{},
- nc, env.ExpectedNodes, utils.GlobalAccountID)
+ nc, env.ExpectedNodes, nil, utils.GlobalAccountID)
if err != nil {
slog.ErrorContext(ctx, "RDS: instance-type capability probe failed", "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
@@ -171,31 +167,6 @@ func clusterRunnableTypes(ctx context.Context, nc *nats.Conn, env Env) (func(str
return func(instanceType string) bool { return supported[instanceType] }, nil
}
-// Neither action ever issues a Marker, so one in a request can only have been
-// fabricated. Answering it as page one would report the whole catalog as if it
-// were a later page.
-func rejectMarker(marker *string, action string) error {
- if aws.StringValue(marker) == "" {
- return nil
- }
- return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
- "Marker is not supported: %s returns every row in a single page", action)
-}
-
-// Both members are required by the shape, and treating either as absent would
-// silently widen the result rather than narrow it.
-func filterEntry(entry *rds.Filter) (string, []string, error) {
- if entry == nil || strings.TrimSpace(aws.StringValue(entry.Name)) == "" {
- return "", nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "each filter must carry a name")
- }
- name := strings.ToLower(strings.TrimSpace(aws.StringValue(entry.Name)))
- if len(entry.Values) == 0 {
- return "", nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
- "filter %q must carry at least one value", name)
- }
- return name, aws.StringValueSlice(entry.Values), nil
-}
-
// A bool-shaped filter is parsed rather than compared as text, so it accepts
// every spelling the typed parameter does. Matching only "true" and "false"
// would answer "1" with an empty catalog the caller cannot tell from a real one.
@@ -211,10 +182,3 @@ func boolFilterValues(name string, values []string) ([]string, error) {
}
return parsed, nil
}
-
-// Rejected rather than ignored: these two actions exist only to be filtered, so
-// a dropped filter returns rows the caller asked not to see and cannot detect.
-func unknownFilterName(name string, recognised []string) error {
- return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
- "filter %q is not recognised; supported filters are %s", name, strings.Join(recognised, ", "))
-}
diff --git a/spinifex/gateway/rds/enginecatalog_test.go b/spinifex/gateway/rds/enginecatalog_test.go
index 11738190f..8e11b2acb 100644
--- a/spinifex/gateway/rds/enginecatalog_test.go
+++ b/spinifex/gateway/rds/enginecatalog_test.go
@@ -219,13 +219,25 @@ func TestDescribeOrderableDBInstanceOptions_RejectANonBooleanVpcFilter(t *testin
}
}
+// AWS's answers: names are compared case-sensitively and checked before the
+// values, and a missing name is reported as "null".
func TestDescribeCatalogs_RejectMalformedFilters(t *testing.T) {
cases := []struct {
name string
query map[string]string
+ code string
+ msg string
}{
- {"no values", map[string]string{"Filters.Filter.1.Name": "engine"}},
- {"no name", map[string]string{"Filters.Filter.1.Values.Value.1": "postgres"}},
+ {"no values", map[string]string{"Filters.Filter.1.Name": "engine"},
+ awserrors.ErrorInvalidParameterCombination, "The values list cannot be null for the filter engine."},
+ {"empty values", map[string]string{"Filters.Filter.1.Name": "engine", "Filters.Filter.1.Values": ""},
+ awserrors.ErrorInvalidParameterCombination, "The values list cannot be null for the filter engine."},
+ {"no name", map[string]string{"Filters.Filter.1.Values.Value.1": "postgres"},
+ awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: null"},
+ {"name case", map[string]string{"Filters.Filter.1.Name": "Engine", "Filters.Filter.1.Values.Value.1": "postgres"},
+ awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: Engine"},
+ {"unknown name without values", map[string]string{"Filters.Filter.1.Name": "bogus"},
+ awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: bogus"},
}
for _, tc := range cases {
@@ -233,15 +245,38 @@ func TestDescribeCatalogs_RejectMalformedFilters(t *testing.T) {
tc.query["Action"] = "DescribeDBEngineVersions"
_, err := Dispatch(t.Context(), "DescribeDBEngineVersions", tc.query, nil, testCaller, testEnv)
require.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
+ assert.Contains(t, err.Error(), tc.code)
+ assert.Contains(t, err.Error(), tc.msg)
})
}
}
-// Neither action ever issues a Marker, so one in a request was fabricated by the
-// caller. This diverges from the other RDS describes, which parse and ignore it,
-// because those may one day paginate and these two never will.
-func TestDescribeCatalogs_RejectAMarker(t *testing.T) {
+// Following each Marker returns the whole catalog once, in the unpaged order,
+// with the Marker surviving the XML round trip.
+func TestDescribeOrderableDBInstanceOptions_PagesTheCatalog(t *testing.T) {
+ nc := newStubbedNATS(t)
+ var classes []string
+ marker := ""
+ for {
+ query := map[string]string{"Action": "DescribeOrderableDBInstanceOptions", "Engine": "postgres", "MaxRecords": "2"}
+ if marker != "" {
+ query["Marker"] = marker
+ }
+ body, err := Dispatch(t.Context(), "DescribeOrderableDBInstanceOptions", query, nc, testCaller, testEnv)
+ require.NoError(t, err)
+ var out rds.DescribeOrderableDBInstanceOptionsOutput
+ require.NoError(t, xmlutil.UnmarshalXML(&out, xml.NewDecoder(bytes.NewReader(body)),
+ "DescribeOrderableDBInstanceOptionsResult"))
+ require.LessOrEqual(t, len(out.OrderableDBInstanceOptions), 2)
+ classes = append(classes, classNames(out.OrderableDBInstanceOptions)...)
+ if marker = aws.StringValue(out.Marker); marker == "" {
+ break
+ }
+ }
+ assert.Equal(t, handlers_rds.SupportedInstanceClasses(), classes)
+}
+
+func TestDescribeCatalogs_RejectAMarkerTheyDidNotIssue(t *testing.T) {
nc := newStubbedNATS(t)
for _, action := range []string{"DescribeDBEngineVersions", "DescribeOrderableDBInstanceOptions"} {
t.Run(action, func(t *testing.T) {
@@ -250,7 +285,6 @@ func TestDescribeCatalogs_RejectAMarker(t *testing.T) {
}, nc, testCaller, testEnv)
require.Error(t, err)
assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
- assert.Contains(t, err.Error(), "Marker")
})
}
}
diff --git a/spinifex/gateway/spinifex_modelaccess_test.go b/spinifex/gateway/spinifex_modelaccess_test.go
index 2b561e06c..b357fe423 100644
--- a/spinifex/gateway/spinifex_modelaccess_test.go
+++ b/spinifex/gateway/spinifex_modelaccess_test.go
@@ -48,7 +48,7 @@ func accessGateway(t *testing.T) *GatewayConfig {
_, _, js := testutil.StartTestJetStream(t)
return &GatewayConfig{
DisableLogging: true,
- BedrockAccessAdmin: gateway_bedrock.NewModelAccessStore(js, 1),
+ BedrockAccessAdmin: gateway_bedrock.NewModelAccessStore(js),
IAMService: allowAllIAMService(),
}
}
diff --git a/spinifex/gateway/tagging/GetResources.go b/spinifex/gateway/tagging/GetResources.go
index e9ebf944c..b686668ec 100644
--- a/spinifex/gateway/tagging/GetResources.go
+++ b/spinifex/gateway/tagging/GetResources.go
@@ -250,14 +250,9 @@ func matchesAllTagFilters(tags []*rgt.Tag, filters []*rgt.TagFilter) bool {
if len(f.Values) == 0 {
continue
}
- matched := false
- for _, v := range f.Values {
- if aws.StringValue(v) == val {
- matched = true
- break
- }
- }
- if !matched {
+ if !slices.ContainsFunc(f.Values, func(v *string) bool {
+ return aws.StringValue(v) == val
+ }) {
return false
}
}
diff --git a/spinifex/handlers/bedrock/service.go b/spinifex/handlers/bedrock/service.go
index 14bf45780..2fff5c196 100644
--- a/spinifex/handlers/bedrock/service.go
+++ b/spinifex/handlers/bedrock/service.go
@@ -90,7 +90,7 @@ func NewService(nc *nats.Conn, deps ServiceDeps) *Service {
return &Service{
nc: nc,
deps: deps,
- store: newEndpointStore(js, deps.Replicas),
+ store: newEndpointStore(js),
leader: newLeaderBucket(js),
}
}
diff --git a/spinifex/handlers/bedrock/store.go b/spinifex/handlers/bedrock/store.go
index 5026a5e0f..bebce6cb6 100644
--- a/spinifex/handlers/bedrock/store.go
+++ b/spinifex/handlers/bedrock/store.go
@@ -66,15 +66,13 @@ type endpointStore struct {
*kvstore.Store[EndpointRecord]
}
-// newEndpointStore returns the endpoints store over js, replicated across
-// replicas nodes. A nil js is permitted: every accessor then reports
-// missingJetStream rather than panicking.
-func newEndpointStore(js jetstream.JetStream, replicas int) *endpointStore {
+// newEndpointStore returns the endpoints store over js. A nil js is permitted:
+// every accessor then reports missingJetStream rather than panicking.
+func newEndpointStore(js jetstream.JetStream) *endpointStore {
return &endpointStore{Store: kvstore.New[EndpointRecord](js, kvstore.Config{
- Name: KVBucketEndpoints,
- History: KVBucketEndpointsHistory,
- Replicas: replicas,
- Missing: missingJetStream,
+ Name: KVBucketEndpoints,
+ History: KVBucketEndpointsHistory,
+ Missing: missingJetStream,
})}
}
diff --git a/spinifex/handlers/bedrock/store_test.go b/spinifex/handlers/bedrock/store_test.go
index c6e511015..2715a1c58 100644
--- a/spinifex/handlers/bedrock/store_test.go
+++ b/spinifex/handlers/bedrock/store_test.go
@@ -17,7 +17,7 @@ func TestEndpointKey_Base64EncodesColonBearingModelID(t *testing.T) {
func newTestBucket(t *testing.T) *endpointStore {
t.Helper()
_, _, js := testutil.StartTestJetStream(t)
- return newEndpointStore(js, 1)
+ return newEndpointStore(js)
}
func TestStore_CreateGetUpdateDelete(t *testing.T) {
diff --git a/spinifex/handlers/bedrock/userdata.go b/spinifex/handlers/bedrock/userdata.go
index b00c6a0be..41f7903a2 100644
--- a/spinifex/handlers/bedrock/userdata.go
+++ b/spinifex/handlers/bedrock/userdata.go
@@ -18,15 +18,11 @@ const bundleAgentEnvDir = "/etc/conf.d/bedrock-bundle"
// instance reads from. A model id's ':' and '.' are not valid systemd
// instance-name characters, so the id is folded to a safe one first.
func memberEnvPath(modelID string) string {
- return bundleAgentEnvDir + "/" + sanitizeMemberInstanceName(modelID) + ".env"
+ return bundleAgentEnvDir + "/" + memberInstanceNameReplacer.Replace(modelID) + ".env"
}
var memberInstanceNameReplacer = strings.NewReplacer(":", "_", "/", "_", ".", "_")
-func sanitizeMemberInstanceName(modelID string) string {
- return memberInstanceNameReplacer.Replace(modelID)
-}
-
// engineForFamily selects which serving engine a family runs under: familyMeta
// always runs vLLM, every other family (familyTEI included) runs TEI. Mirrors
// the invoke-side dispatch in gateway_bedrock, so a family unknown there is
diff --git a/spinifex/handlers/ec2/eigw/service_impl.go b/spinifex/handlers/ec2/eigw/service_impl.go
index 0efabf2c0..a722e8291 100644
--- a/spinifex/handlers/ec2/eigw/service_impl.go
+++ b/spinifex/handlers/ec2/eigw/service_impl.go
@@ -2,6 +2,7 @@ package handlers_ec2_eigw
import (
"context"
+ "encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -89,7 +90,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
}
if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, *input.VpcId)); err != nil {
slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID)
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", *input.VpcId)
}
eigwID := utils.GenerateResourceID("eigw")
@@ -129,7 +130,7 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
// Verify the EIGW exists before deleting
if _, err := s.eigwKV.Get(ctx, key); err != nil {
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidEgressOnlyInternetGatewayIdNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidGatewayIDNotFound, "The eigw ID '%s' does not exist", eigwID)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -147,6 +148,8 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
// describeEIGWValidFilters defines the set of filter names accepted by DescribeEgressOnlyInternetGateways.
var describeEIGWValidFilters = map[string]bool{
"egress-only-internet-gateway-id": true,
+ "tag-key": true,
+ "tag-value": true,
}
// DescribeEgressOnlyInternetGateways describes Egress-only Internet Gateways.
@@ -165,6 +168,11 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
slog.WarnContext(ctx, "DescribeEgressOnlyInternetGateways: invalid filter", "err", err)
return nil, err
}
+ for _, id := range input.EgressOnlyInternetGatewayIds {
+ if id != nil && !eigwIDWellFormed(*id) {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidEgressOnlyInternetGatewayIdMalformed, "The eigw ID %s is malformed", *id)
+ }
+ }
prefix := accountID + "."
keys, err := s.eigwKV.Keys(ctx)
@@ -210,10 +218,22 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
}, nil
}
+// eigwIDWellFormed reports whether AWS accepts id as an egress-only gateway ID:
+// "eigw-" and exactly 17 hex digits of either case. A well-formed unknown ID is
+// not an error; the describe returns no gateway for it.
+func eigwIDWellFormed(id string) bool {
+ suffix, ok := strings.CutPrefix(id, "eigw-")
+ if !ok || len(suffix) != 17 {
+ return false
+ }
+ _, err := hex.DecodeString("0" + suffix)
+ return err == nil
+}
+
// eigwMatchesFilters checks whether an EgressOnlyIGWRecord satisfies all parsed awsfilters.
func eigwMatchesFilters(record *EgressOnlyIGWRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
diff --git a/spinifex/handlers/ec2/eigw/service_impl_test.go b/spinifex/handlers/ec2/eigw/service_impl_test.go
index 76b48420f..b49b45d8d 100644
--- a/spinifex/handlers/ec2/eigw/service_impl_test.go
+++ b/spinifex/handlers/ec2/eigw/service_impl_test.go
@@ -6,8 +6,9 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
- handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -133,7 +134,10 @@ func TestDeleteEgressOnlyInternetGateway_NotFound(t *testing.T) {
_, err := svc.DeleteEgressOnlyInternetGateway(context.Background(), &ec2.DeleteEgressOnlyInternetGatewayInput{
EgressOnlyInternetGatewayId: aws.String("eigw-nonexistent"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidEgressOnlyInternetGatewayId.NotFound")
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidGatewayIDNotFound, code)
+ assert.Equal(t, "The eigw ID 'eigw-nonexistent' does not exist", msg)
}
func TestDeleteEgressOnlyInternetGateway_MissingID(t *testing.T) {
@@ -270,6 +274,36 @@ func TestDescribeEgressOnlyInternetGateways_FilterByTag(t *testing.T) {
assert.Equal(t, *out.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId, *desc.EgressOnlyInternetGateways[0].EgressOnlyInternetGatewayId)
}
+func TestDescribeEgressOnlyInternetGateways_FilterByTagKeyAndTagValue(t *testing.T) {
+ svc := setupTestEIGWService(t)
+ out, err := svc.CreateEgressOnlyInternetGateway(context.Background(), &ec2.CreateEgressOnlyInternetGatewayInput{
+ VpcId: aws.String("vpc-tagged"),
+ TagSpecifications: []*ec2.TagSpecification{
+ {
+ ResourceType: aws.String("egress-only-internet-gateway"),
+ Tags: []*ec2.Tag{{Key: aws.String("Env"), Value: aws.String("prod")}},
+ },
+ },
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestEIGW(t, svc) // untagged
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Env"}, {"tag-value", "prod"}} {
+ desc, err := svc.DescribeEgressOnlyInternetGateways(context.Background(), &ec2.DescribeEgressOnlyInternetGatewaysInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.EgressOnlyInternetGateways, 1, tc.name)
+ assert.Equal(t, *out.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId, *desc.EgressOnlyInternetGateways[0].EgressOnlyInternetGatewayId, tc.name)
+
+ desc, err = svc.DescribeEgressOnlyInternetGateways(context.Background(), &ec2.DescribeEgressOnlyInternetGatewaysInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.EgressOnlyInternetGateways, tc.name)
+ }
+}
+
// TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected tests that creating an EIGW in another account's VPC is rejected.
func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
// Set up with manual NATS to get VPC KV access
@@ -303,3 +337,24 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
require.NoError(t, err)
assert.NotNil(t, out.EgressOnlyInternetGateway)
}
+
+func TestDescribeEgressOnlyInternetGateways_MalformedID(t *testing.T) {
+ svc := setupTestEIGWService(t)
+ describe := func(ids ...string) error {
+ _, err := svc.DescribeEgressOnlyInternetGateways(context.Background(), &ec2.DescribeEgressOnlyInternetGatewaysInput{
+ EgressOnlyInternetGatewayIds: aws.StringSlice(ids),
+ }, testAccountID)
+ return err
+ }
+
+ for _, id := range []string{"eigw-xyz", "eigw-12345", "eigw-0000000000000001", "eigw-000000000000000001", "foo"} {
+ err := describe(id)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidEgressOnlyInternetGatewayIdMalformed, code)
+ assert.Equal(t, "The eigw ID "+id+" is malformed", msg)
+ }
+ // Exactly 17 hex digits of either case is well formed, and unknown is not an error.
+ assert.NoError(t, describe("eigw-00000000000000001"))
+ assert.NoError(t, describe("eigw-0AAAAAAAAAAAAAAA1"))
+}
diff --git a/spinifex/handlers/ec2/eip/associate_test.go b/spinifex/handlers/ec2/eip/associate_test.go
index b54a7e3eb..865245bfa 100644
--- a/spinifex/handlers/ec2/eip/associate_test.go
+++ b/spinifex/handlers/ec2/eip/associate_test.go
@@ -117,10 +117,11 @@ func TestEIP_AssociateByNetworkInterfaceID_NotFound(t *testing.T) {
_, err := svc.AssociateAddress(context.Background(), &ec2.AssociateAddressInput{
AllocationId: aws.String(allocID),
- NetworkInterfaceId: aws.String("eni-doesnotexist"),
+ NetworkInterfaceId: aws.String("eni-0000000000000dead"),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidNetworkInterfaceIDNotFound, err.Error())
+ code, _ := awserrors.ResolveErrorCode(err)
+ assert.Equal(t, awserrors.ErrorInvalidNetworkInterfaceIDNotFound, code)
// The allocation must survive a failed association.
addr := describeEIP(t, svc, allocID)
diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go
index a6025d832..9ae615e8f 100644
--- a/spinifex/handlers/ec2/eip/service_impl.go
+++ b/spinifex/handlers/ec2/eip/service_impl.go
@@ -596,9 +596,21 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr
found[*addr.AllocationId] = true
}
}
- for id := range allocIDs {
- if !found[id] {
- return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound)
+ for _, id := range input.AllocationIds {
+ if id != nil && !found[*id] {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidAllocationIDNotFound, "The allocation ID '%s' does not exist", *id)
+ }
+ }
+ }
+
+ if len(publicIPs) > 0 {
+ found := make(map[string]bool)
+ for _, addr := range addresses {
+ found[aws.StringValue(addr.PublicIp)] = true
+ }
+ for _, ip := range input.PublicIps {
+ if ip != nil && !found[*ip] {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidAddressNotFound, "Address '%s' not found.", *ip)
}
}
}
diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go
index 87ea427fb..879e1ae1a 100644
--- a/spinifex/handlers/ec2/eip/service_impl_test.go
+++ b/spinifex/handlers/ec2/eip/service_impl_test.go
@@ -544,6 +544,52 @@ func TestEIP_DescribeAddresses(t *testing.T) {
_ = out3
}
+// AWS answers a named public IP the account does not hold with
+// InvalidAddress.NotFound, including one it has just released.
+func TestEIP_DescribeAddresses_PublicIpNotFound(t *testing.T) {
+ svc, _, _ := setupTestEIP(t)
+
+ kept, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+ require.NoError(t, err)
+ released, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+ require.NoError(t, err)
+ _, err = svc.ReleaseAddress(context.Background(), &ec2.ReleaseAddressInput{AllocationId: released.AllocationId}, testAccountID)
+ require.NoError(t, err)
+
+ desc, err := svc.DescribeAddresses(context.Background(), &ec2.DescribeAddressesInput{
+ PublicIps: []*string{kept.PublicIp},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, desc.Addresses, 1)
+
+ _, err = svc.DescribeAddresses(context.Background(), &ec2.DescribeAddressesInput{
+ PublicIps: []*string{kept.PublicIp, released.PublicIp},
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidAddressNotFound, code)
+ assert.Equal(t, "Address '"+*released.PublicIp+"' not found.", msg)
+}
+
+func TestEIP_DescribeAddresses_AllocationIdNotFound(t *testing.T) {
+ svc, _, _ := setupTestEIP(t)
+
+ kept, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+ require.NoError(t, err)
+ released, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
+ require.NoError(t, err)
+ _, err = svc.ReleaseAddress(context.Background(), &ec2.ReleaseAddressInput{AllocationId: released.AllocationId}, testAccountID)
+ require.NoError(t, err)
+
+ _, err = svc.DescribeAddresses(context.Background(), &ec2.DescribeAddressesInput{
+ AllocationIds: []*string{kept.AllocationId, released.AllocationId},
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidAllocationIDNotFound, code)
+ assert.Equal(t, "The allocation ID '"+*released.AllocationId+"' does not exist", msg)
+}
+
func TestEIP_DescribeAddresses_FilterByAllocationId(t *testing.T) {
svc, _, _ := setupTestEIP(t)
diff --git a/spinifex/handlers/ec2/igw/service_impl.go b/spinifex/handlers/ec2/igw/service_impl.go
index 907624822..76521db17 100644
--- a/spinifex/handlers/ec2/igw/service_impl.go
+++ b/spinifex/handlers/ec2/igw/service_impl.go
@@ -18,6 +18,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+ "github.com/mulgadc/spinifex/spinifex/paging"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -136,7 +137,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne
slog.InfoContext(ctx, "CreateInternetGateway completed", "internetGatewayId", igwID, "accountID", accountID)
return &ec2.CreateInternetGatewayOutput{
- InternetGateway: s.recordToEC2(&record),
+ InternetGateway: s.recordToEC2(&record, accountID),
}, nil
}
@@ -155,7 +156,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
// tolerates it on destroy); destroy orchestration tolerates it too.
// A transient read error stays a server error.
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
+ return nil, igwNotFoundError(igwID)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -165,12 +166,11 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
return nil, errors.New(awserrors.ErrorServerInternal)
}
- // Cannot delete an attached IGW. The VPC is named because a pending
- // attachment is hidden from describes, so the caller has no other way to
- // learn what it must detach from.
+ // Cannot delete an attached IGW. AWS's message does not name the VPC, and
+ // a pending attachment is hidden from describes, so the log names it.
if record.VpcId != "" {
- return nil, awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the internet gateway is attached to %s and must be detached first", record.VpcId)
+ slog.WarnContext(ctx, "DeleteInternetGateway: gateway is attached", "internetGatewayId", igwID, "vpcId", record.VpcId)
+ return nil, awserrors.HasDependencies("internetGateway", igwID)
}
if err := s.igwKV.Delete(ctx, key); err != nil {
@@ -182,17 +182,31 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
return &ec2.DeleteInternetGatewayOutput{}, nil
}
+var describeIGWPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+ TooSmall: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter InternetGatewayIds cannot be used with the parameter MaxResults",
+}
+
// describeIGWValidFilters defines the set of filter names accepted by DescribeInternetGateways.
var describeIGWValidFilters = map[string]bool{
"internet-gateway-id": true,
"attachment.vpc-id": true,
"attachment.state": true,
+ "tag-key": true,
+ "tag-value": true,
}
// DescribeInternetGateways lists Internet Gateways, optionally filtered by ID.
func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, accountID string) (*ec2.DescribeInternetGatewaysOutput, error) {
var igws []*ec2.InternetGateway
+ // Validated but not paged: AWS's paging of DescribeInternetGateways is unobserved.
+ if _, err := describeIGWPaging.Parse(input.MaxResults, input.NextToken, len(input.InternetGatewayIds)); err != nil {
+ return nil, err
+ }
+
igwIDs := make(map[string]bool)
for _, id := range input.InternetGatewayIds {
if id != nil {
@@ -242,14 +256,14 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
continue
}
- igws = append(igws, s.recordToEC2(&record))
+ igws = append(igws, s.recordToEC2(&record, accountID))
foundIDs[record.InternetGatewayId] = true
}
// Return error if specific IDs were requested but not found
- for id := range igwIDs {
- if !foundIDs[id] {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
+ for _, id := range input.InternetGatewayIds {
+ if id != nil && !foundIDs[*id] {
+ return nil, igwNotFoundError(*id)
}
}
@@ -263,7 +277,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
// igwMatchesFilters checks whether an IGWRecord satisfies all parsed awsfilters.
func igwMatchesFilters(record *IGWRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
@@ -309,7 +323,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
entry, err := s.igwKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
+ return nil, igwNotFoundError(igwID)
}
var record IGWRecord
@@ -318,7 +332,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
}
if record.VpcId != "" {
- return nil, errors.New(awserrors.ErrorResourceAlreadyAssociated)
+ return nil, awserrors.Errorf(awserrors.ErrorResourceAlreadyAssociated, "resource %s is already attached to network %s", igwID, record.VpcId)
}
// Verify the caller owns the target VPC (fail-closed if KV unavailable)
@@ -328,7 +342,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
}
if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID)); err != nil {
slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID)
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
}
record.VpcId = vpcID
@@ -371,7 +385,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
entry, err := s.igwKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
+ return nil, igwNotFoundError(igwID)
}
var record IGWRecord
@@ -443,7 +457,7 @@ func (s *IGWServiceImpl) CreateAttachedInternetGateway(ctx context.Context, acco
return false, errors.New(awserrors.ErrorServerInternal)
}
if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID)); err != nil {
- return false, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return false, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
}
key := utils.AccountKey(accountID, igwID)
@@ -569,9 +583,10 @@ func MarkAttached(ctx context.Context, kv jetstream.KeyValue, recordKey, vpcID s
return nil
}
-func (s *IGWServiceImpl) recordToEC2(record *IGWRecord) *ec2.InternetGateway {
+func (s *IGWServiceImpl) recordToEC2(record *IGWRecord, accountID string) *ec2.InternetGateway {
igw := &ec2.InternetGateway{
InternetGatewayId: aws.String(record.InternetGatewayId),
+ OwnerId: aws.String(accountID),
}
// AWS returns no attachment at all unless one exists, so a requested but
@@ -612,3 +627,7 @@ func (s *IGWServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID
func(r *IGWRecord) *map[string]string { return &r.Tags },
utils.RemoveTagsMut(input))
}
+
+func igwNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidInternetGatewayIDNotFound, "internetGateway", id)
+}
diff --git a/spinifex/handlers/ec2/igw/service_impl_test.go b/spinifex/handlers/ec2/igw/service_impl_test.go
index 34de1a585..324b9be68 100644
--- a/spinifex/handlers/ec2/igw/service_impl_test.go
+++ b/spinifex/handlers/ec2/igw/service_impl_test.go
@@ -9,8 +9,9 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
- handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
@@ -84,6 +85,7 @@ func TestCreateInternetGateway(t *testing.T) {
assert.Equal(t, "igw-", (*out.InternetGateway.InternetGatewayId)[:4])
// Should not have attachments when created
assert.Empty(t, out.InternetGateway.Attachments)
+ assert.Equal(t, testAccountID, aws.StringValue(out.InternetGateway.OwnerId))
}
func TestCreateInternetGateway_WithTags(t *testing.T) {
@@ -171,7 +173,37 @@ func TestDeleteInternetGateway_WhileAttached(t *testing.T) {
_, err = svc.DeleteInternetGateway(context.Background(), &ec2.DeleteInternetGatewayInput{
InternetGatewayId: aws.String(igwID),
}, testAccountID)
- assert.ErrorContains(t, err, "DependencyViolation")
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The internetGateway '"+igwID+"' has dependencies and cannot be deleted.")
+
+ // Attaching again names the gateway and the network it is on
+ _, err = svc.AttachInternetGateway(context.Background(), &ec2.AttachInternetGatewayInput{
+ InternetGatewayId: aws.String(igwID),
+ VpcId: aws.String("vpc-test123"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorResourceAlreadyAssociated, "resource "+igwID+" is already attached to network vpc-test123")
+}
+
+func TestInternetGateway_NotFoundNamesTheID(t *testing.T) {
+ svc, _ := setupTestIGWService(t)
+ createTestIGW(t, svc)
+
+ _, err := svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+ InternetGatewayIds: []*string{aws.String("igw-0000000000000dead")},
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
+
+ _, err = svc.DeleteInternetGateway(context.Background(), &ec2.DeleteInternetGatewayInput{
+ InternetGatewayId: aws.String("igw-0000000000000dead"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidInternetGatewayIDNotFound, "The internetGateway ID 'igw-0000000000000dead' does not exist")
+}
+
+func requireAWSError(t *testing.T, err error, code, message string) {
+ t.Helper()
+ got, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, code, got)
+ assert.Equal(t, message, msg)
}
func TestDescribeInternetGateways_All(t *testing.T) {
@@ -195,6 +227,7 @@ func TestDescribeInternetGateways_ByID(t *testing.T) {
require.NoError(t, err)
require.Len(t, desc.InternetGateways, 1)
assert.Equal(t, igwID, *desc.InternetGateways[0].InternetGatewayId)
+ assert.Equal(t, testAccountID, aws.StringValue(desc.InternetGateways[0].OwnerId))
}
func TestDescribeInternetGateways_Empty(t *testing.T) {
@@ -948,6 +981,35 @@ func TestDescribeInternetGateways_FilterByTag(t *testing.T) {
assert.Equal(t, *out.InternetGateway.InternetGatewayId, *desc.InternetGateways[0].InternetGatewayId)
}
+func TestDescribeInternetGateways_FilterByTagKeyAndTagValue(t *testing.T) {
+ svc, _ := setupTestIGWService(t)
+ out, err := svc.CreateInternetGateway(context.Background(), &ec2.CreateInternetGatewayInput{
+ TagSpecifications: []*ec2.TagSpecification{
+ {
+ ResourceType: aws.String("internet-gateway"),
+ Tags: []*ec2.Tag{{Key: aws.String("Env"), Value: aws.String("prod")}},
+ },
+ },
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestIGW(t, svc) // untagged
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Env"}, {"tag-value", "prod"}} {
+ desc, err := svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.InternetGateways, 1, tc.name)
+ assert.Equal(t, *out.InternetGateway.InternetGatewayId, *desc.InternetGateways[0].InternetGatewayId, tc.name)
+
+ desc, err = svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.InternetGateways, tc.name)
+ }
+}
+
func TestDeleteInternetGateway_PublishesNoEvent(t *testing.T) {
svc, nc := setupTestIGWService(t)
igwID := createTestIGW(t, svc)
@@ -1029,3 +1091,22 @@ func TestAttachInternetGateway_NoGatePublisher_NoOp(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
}
+
+func TestDescribeInternetGateways_PagingValidation(t *testing.T) {
+ svc, _ := setupTestIGWService(t)
+
+ _, err := svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+ MaxResults: aws.Int64(5), InternetGatewayIds: []*string{aws.String("igw-0123456789abcdef0")},
+ }, testAccountID)
+ code, message, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+ assert.Equal(t, "The parameter InternetGatewayIds cannot be used with the parameter MaxResults", message)
+
+ _, err = svc.DescribeInternetGateways(context.Background(), &ec2.DescribeInternetGatewaysInput{
+ NextToken: aws.String("garbage"),
+ }, testAccountID)
+ code, _, ok = awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+}
diff --git a/spinifex/handlers/ec2/image/describe_byid_test.go b/spinifex/handlers/ec2/image/describe_byid_test.go
index 049596882..eb539aef9 100644
--- a/spinifex/handlers/ec2/image/describe_byid_test.go
+++ b/spinifex/handlers/ec2/image/describe_byid_test.go
@@ -160,6 +160,41 @@ func TestDescribeImagesByID_SystemAMIIsVisibleToEveryAccount(t *testing.T) {
assert.Equal(t, "amazon", aws.StringValue(out.Images[0].ImageOwnerAlias))
}
+// A system image renders as AWS's public, Amazon-owned images do, and the
+// is-public filter reads the rendered value; a private image stays private.
+func TestDescribeImagesByID_SystemAMIRendersPublicAmazon(t *testing.T) {
+ tests := []struct {
+ name string
+ ownerAlias string
+ wantPublic bool
+ wantAlias string
+ }{
+ {name: "system image", ownerAlias: "system", wantPublic: true, wantAlias: "amazon"},
+ {name: "own image", ownerAlias: byIDAccount, wantPublic: false, wantAlias: byIDAccount},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ store := recordingstore.New()
+ svc := handlers_ec2_image.NewImageServiceImplWithStore(store, byIDBucket)
+ putByIDAMI(t, store, byIDAMIDoc(byIDImage, tt.ownerAlias))
+
+ out, err := svc.DescribeImages(t.Context(), &ec2.DescribeImagesInput{
+ ImageIds: aws.StringSlice([]string{byIDImage}),
+ Filters: []*ec2.Filter{{
+ Name: aws.String("is-public"), Values: aws.StringSlice([]string{strconv.FormatBool(tt.wantPublic)}),
+ }},
+ }, byIDAccount)
+
+ require.NoError(t, err)
+ require.Len(t, out.Images, 1)
+ img := out.Images[0]
+ assert.Equal(t, tt.wantPublic, aws.BoolValue(img.Public))
+ assert.Equal(t, tt.wantAlias, aws.StringValue(img.ImageOwnerAlias))
+ assert.Equal(t, "RunInstances", aws.StringValue(img.UsageOperation))
+ })
+ }
+}
+
// The Owners parameter is matched against the resolved owner, and a named image
// that no owner selects is not-found rather than an empty success.
func TestDescribeImagesByID_OwnerFilters(t *testing.T) {
diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go
index 29d5cb41d..479a84340 100644
--- a/spinifex/handlers/ec2/image/service_impl.go
+++ b/spinifex/handlers/ec2/image/service_impl.go
@@ -295,6 +295,7 @@ func renderVisibleImage(amiMeta ebsmetadata.AMI, input *ec2.DescribeImagesInput,
Architecture: aws.String(amiMeta.Architecture),
PlatformDetails: aws.String(amiMeta.PlatformDetails),
Platform: utils.PlatformFromDetails(amiMeta.PlatformDetails),
+ UsageOperation: utils.UsageOperationFromDetails(amiMeta.PlatformDetails),
CreationDate: aws.String(amiMeta.CreationDate.UTC().Format("2006-01-02T15:04:05.000Z")),
RootDeviceType: aws.String(amiMeta.RootDeviceType),
VirtualizationType: aws.String(amiMeta.Virtualization),
@@ -306,6 +307,13 @@ func renderVisibleImage(amiMeta ebsmetadata.AMI, input *ec2.DescribeImagesInput,
Hypervisor: aws.String("xen"),
}
+ // A system image stands where AWS's public, Amazon-owned images do. Only the
+ // rendered values change: the stored alias still identifies it as system.
+ if isSystemAMI {
+ image.Public = aws.Bool(true)
+ image.ImageOwnerAlias = aws.String("amazon")
+ }
+
// RegisterImage takes BootMode as an optional input, so an AMI can
// record none. The enum has no empty member: omit it, don't emit "".
if amiMeta.BootMode != "" {
diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go
index e76966a1a..739f2d80b 100644
--- a/spinifex/handlers/ec2/instance/service_impl.go
+++ b/spinifex/handlers/ec2/instance/service_impl.go
@@ -9,6 +9,7 @@ import (
"log/slog"
"maps"
"os"
+ "slices"
"sort"
"strconv"
"strings"
@@ -2031,22 +2032,16 @@ func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]
// matchTagKey returns true if any tag key on the resource matches any of the filter values.
func matchTagKey(tags []*ec2.Tag, values []string) bool {
- for _, t := range tags {
- if t.Key != nil && awsfilters.MatchesAny(values, *t.Key) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(tags, func(t *ec2.Tag) bool {
+ return t.Key != nil && awsfilters.MatchesAny(values, *t.Key)
+ })
}
// matchTagValue returns true if any tag value on the resource matches any of the filter values.
func matchTagValue(tags []*ec2.Tag, values []string) bool {
- for _, t := range tags {
- if t.Value != nil && awsfilters.MatchesAny(values, *t.Value) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(tags, func(t *ec2.Tag) bool {
+ return t.Value != nil && awsfilters.MatchesAny(values, *t.Value)
+ })
}
// DescribeInstances returns instances on this node visible to the caller's account.
@@ -2123,10 +2118,7 @@ func (s *InstanceServiceImpl) DescribeInstances(ctx context.Context, input *ec2.
}
})
- reservations := make([]*ec2.Reservation, 0, len(reservationMap))
- for _, reservation := range reservationMap {
- reservations = append(reservations, reservation)
- }
+ reservations := slices.Collect(maps.Values(reservationMap))
slog.InfoContext(ctx, "DescribeInstances completed", "count", len(reservations))
return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
@@ -2249,10 +2241,7 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(ctx context.Context, input
reservationMap[resID].Instances = append(reservationMap[resID].Instances, projected)
}
- reservations := make([]*ec2.Reservation, 0, len(reservationMap))
- for _, reservation := range reservationMap {
- reservations = append(reservations, reservation)
- }
+ reservations := slices.Collect(maps.Values(reservationMap))
slog.InfoContext(ctx, opName+" completed", "count", len(reservations))
return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go
index e94fee1ca..9fce80de7 100644
--- a/spinifex/handlers/ec2/instance/service_impl_test.go
+++ b/spinifex/handlers/ec2/instance/service_impl_test.go
@@ -3883,6 +3883,7 @@ func TestPrepareRunInstances_NATFailureRollsBackPublicIP(t *testing.T) {
// half-committed rule from a waitForFlowsHV timeout.
deleteNATCh := make(chan natWirePayload, 1)
delSub, err := svc.natsConn.Subscribe("vpc.delete-nat", func(msg *nats.Msg) {
+ _ = msg.Respond([]byte(`{"success":true}`))
var p natWirePayload
_ = json.Unmarshal(msg.Data, &p)
select {
@@ -3990,6 +3991,7 @@ func TestPrepareRunInstances_PartialLaunchReleasesSucceededInstances(t *testing.
deleteNATCh := make(chan natWirePayload, 2)
delSub, err := svc.natsConn.Subscribe("vpc.delete-nat", func(msg *nats.Msg) {
+ _ = msg.Respond([]byte(`{"success":true}`))
var p natWirePayload
_ = json.Unmarshal(msg.Data, &p)
select {
diff --git a/spinifex/handlers/ec2/key/service_impl.go b/spinifex/handlers/ec2/key/service_impl.go
index 298da0017..7a0e44953 100644
--- a/spinifex/handlers/ec2/key/service_impl.go
+++ b/spinifex/handlers/ec2/key/service_impl.go
@@ -19,6 +19,7 @@ import (
"fmt"
"io"
"log/slog"
+ "slices"
"strings"
"github.com/aws/aws-sdk-go/aws"
@@ -41,6 +42,45 @@ type KeyServiceImpl struct {
config *config.Config
store objectstore.ObjectStore
bucketName string
+
+ // Optional: injected after construction. Nil leaves the central tag store
+ // untouched by create, import and delete.
+ centralTags CentralTagStore
+}
+
+// CentralTagStore keeps the central tag index in step with key-pair records, so
+// DescribeTags sees creation tags. Implemented by handlers/ec2/tags.TagsServiceImpl.
+type CentralTagStore interface {
+ PutResourceTags(ctx context.Context, accountID, resourceID string, tags map[string]string) error
+ DeleteAllTags(ctx context.Context, accountID, resourceID string) error
+}
+
+// SetCentralTagStore injects the central tag store that create and import
+// project their tags into and delete clears.
+func (s *KeyServiceImpl) SetCentralTagStore(st CentralTagStore) {
+ s.centralTags = st
+}
+
+// projectRecordTags writes creation tags into the central tag store. A failure
+// is logged, not returned: the key pair already exists.
+func (s *KeyServiceImpl) projectRecordTags(ctx context.Context, accountID, keyPairID string, tags map[string]string) {
+ if s.centralTags == nil || len(tags) == 0 {
+ return
+ }
+ if err := s.centralTags.PutResourceTags(ctx, accountID, keyPairID, tags); err != nil {
+ slog.ErrorContext(ctx, "central tag store write failed", "keyPairId", keyPairID, "err", err)
+ }
+}
+
+// clearRecordTags drops a deleted key pair's central tags. Call it only once the
+// delete has succeeded; a failure is logged, since the key pair is already gone.
+func (s *KeyServiceImpl) clearRecordTags(ctx context.Context, accountID, keyPairID string) {
+ if s.centralTags == nil {
+ return
+ }
+ if err := s.centralTags.DeleteAllTags(ctx, accountID, keyPairID); err != nil {
+ slog.ErrorContext(ctx, "central tag store clear failed", "keyPairId", keyPairID, "err", err)
+ }
}
// NewKeyServiceImpl creates a new daemon-side key service.
@@ -93,7 +133,7 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
if err == nil {
// Object exists - return duplicate error
slog.ErrorContext(ctx, "Key pair already exists", "keyName", keyName)
- return nil, errors.New(awserrors.ErrorInvalidKeyPairDuplicate)
+ return nil, errDuplicateKeyPair()
}
// Determine key type (default: ed25519, optional: rsa). This deviates from
@@ -112,6 +152,20 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
}
}
+ // A declared divergence: AWS returns a PuTTY key for ppk, Spinifex refuses
+ // it rather than silently returning a PEM the caller did not ask for.
+ if input.KeyFormat != nil {
+ switch *input.KeyFormat {
+ case ec2.KeyFormatPem:
+ case ec2.KeyFormatPpk:
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "The ppk key format is not supported; use pem.")
+ default:
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "1 validation error detected: Value '%s' at 'keyFormat' failed to satisfy constraint: Member must satisfy enum value set: [pem, ppk]", *input.KeyFormat)
+ }
+ }
+
// Generate the key in process; the private half never reaches local disk.
privateKeyData, publicKey, err := generateKeyPair(keyType)
if err != nil {
@@ -156,10 +210,17 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
// Build response (similar to AWS EC2)
keyPairID := utils.GenerateResourceID("key")
- tags := utils.MapToEC2Tags(utils.ExtractTags(input.TagSpecifications, "key-pair"))
+ tagMap := utils.ExtractTags(input.TagSpecifications, "key-pair")
+ tags := utils.MapToEC2Tags(tagMap)
+ // AWS ends RSA material at its END line, while OpenSSH-format Ed25519
+ // material keeps the trailing newline.
+ keyMaterial := string(privateKeyData)
+ if keyType == "rsa" {
+ keyMaterial = strings.TrimSuffix(keyMaterial, "\n")
+ }
output := &ec2.CreateKeyPairOutput{
KeyFingerprint: aws.String(fingerprint),
- KeyMaterial: aws.String(string(privateKeyData)),
+ KeyMaterial: aws.String(keyMaterial),
KeyName: aws.String(keyName),
KeyPairId: aws.String(keyPairID),
Tags: tags,
@@ -185,6 +246,8 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
return nil, errors.New(awserrors.ErrorServerInternal)
}
+ s.projectRecordTags(ctx, accountID, keyPairID, tagMap)
+
slog.InfoContext(ctx, "Key pair created successfully", "keyName", keyName, "fingerprint", fingerprint, "keyPairId", keyPairID)
return output, nil
@@ -353,6 +416,17 @@ func (s *KeyServiceImpl) storeKeyPairMetadata(ctx context.Context, accountID, ke
return nil
}
+// errDuplicateKeyPair is AWS's error for a name already in use, on create and
+// import alike.
+func errDuplicateKeyPair() error {
+ return awserrors.Errorf(awserrors.ErrorInvalidKeyPairDuplicate, "The keypair already exists")
+}
+
+// errInvalidKeyFormat is AWS's error for import material it cannot use.
+func errInvalidKeyFormat() error {
+ return awserrors.Errorf(awserrors.ErrorInvalidKeyFormat, "Key is not in valid OpenSSH public key format")
+}
+
// getKeyNameFromKeyPairId retrieves the key name by directly reading the metadata file for a given keyPairId.
func (s *KeyServiceImpl) getKeyNameFromKeyPairId(ctx context.Context, accountID, keyPairID string) (string, error) {
metadataPath := fmt.Sprintf("keys/%s/%s.json", accountID, keyPairID)
@@ -511,6 +585,20 @@ func (s *KeyServiceImpl) GetPublicKeyMaterial(accountID, keyName string) (string
return material, nil
}
+// describedPublicKey returns the stored key as AWS reports it: one OpenSSH
+// line whose comment is the key pair's name, whatever comment was imported.
+func (s *KeyServiceImpl) describedPublicKey(accountID, keyName string) (string, error) {
+ material, err := s.GetPublicKeyMaterial(accountID, keyName)
+ if err != nil {
+ return "", err
+ }
+ publicKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(material))
+ if err != nil {
+ return "", fmt.Errorf("parse public key %s: %w", keyName, err)
+ }
+ return fmt.Sprintf("%s %s\n", bytes.TrimSpace(ssh.MarshalAuthorizedKey(publicKey)), keyName), nil
+}
+
// DeleteKeyPair removes a key pair (both public key and metadata from S3).
func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKeyPairInput, accountID string) (*ec2.DeleteKeyPairOutput, error) {
if input == nil {
@@ -538,7 +626,7 @@ func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKey
// AWS DeleteKeyPair is idempotent — return success for non-existent keys
if err.Error() == awserrors.ErrorInvalidKeyPairNotFound {
slog.DebugContext(ctx, "DeleteKeyPair: key pair not found, returning success (idempotent)", "keyPairId", keyPairID)
- return &ec2.DeleteKeyPairOutput{}, nil
+ return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true)}, nil
}
slog.ErrorContext(ctx, "Failed to get keyName from keyPairId", "keyPairId", keyPairID, "err", err)
return nil, err
@@ -558,7 +646,7 @@ func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKey
// AWS DeleteKeyPair is idempotent — return success for non-existent keys
if err.Error() == awserrors.ErrorInvalidKeyPairNotFound {
slog.DebugContext(ctx, "DeleteKeyPair: key pair not found, returning success (idempotent)", "keyName", keyName)
- return &ec2.DeleteKeyPairOutput{}, nil
+ return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true)}, nil
}
slog.ErrorContext(ctx, "Failed to find keyPairId from keyName", "keyName", keyName, "err", err)
return nil, err
@@ -591,9 +679,12 @@ func (s *KeyServiceImpl) DeleteKeyPair(ctx context.Context, input *ec2.DeleteKey
return nil, errors.New(awserrors.ErrorServerInternal)
}
+ s.clearRecordTags(ctx, accountID, keyPairID)
+
slog.InfoContext(ctx, "Key pair deleted successfully", "keyName", keyName, "keyPairId", keyPairID)
- return &ec2.DeleteKeyPairOutput{}, nil
+ // AWS names the key only when one was actually deleted.
+ return &ec2.DeleteKeyPairOutput{Return: aws.Bool(true), KeyPairId: aws.String(keyPairID)}, nil
}
// DescribeKeyPairs lists available key pairs by reading metadata files from S3
@@ -687,28 +778,18 @@ func (s *KeyServiceImpl) DescribeKeyPairs(ctx context.Context, input *ec2.Descri
// Filter by KeyName if specified
if len(input.KeyNames) > 0 {
- found := false
- for _, filterName := range input.KeyNames {
- if filterName != nil && metadata.KeyName != nil && *filterName == *metadata.KeyName {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(input.KeyNames, func(filterName *string) bool {
+ return filterName != nil && metadata.KeyName != nil && *filterName == *metadata.KeyName
+ }) {
continue
}
}
// Filter by KeyPairId if specified
if len(input.KeyPairIds) > 0 {
- found := false
- for _, filterID := range input.KeyPairIds {
- if filterID != nil && metadata.KeyPairId != nil && *filterID == *metadata.KeyPairId {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(input.KeyPairIds, func(filterID *string) bool {
+ return filterID != nil && metadata.KeyPairId != nil && *filterID == *metadata.KeyPairId
+ }) {
continue
}
}
@@ -735,6 +816,15 @@ func (s *KeyServiceImpl) DescribeKeyPairs(ctx context.Context, input *ec2.Descri
continue
}
+ if aws.BoolValue(input.IncludePublicKey) && metadata.KeyName != nil {
+ publicKey, err := s.describedPublicKey(accountID, *metadata.KeyName)
+ if err != nil {
+ slog.ErrorContext(ctx, "DescribeKeyPairs: failed to read public key", "keyName", *metadata.KeyName, "err", err)
+ return nil, errors.New(awserrors.ErrorServerInternal)
+ }
+ keyPairInfo.PublicKey = aws.String(publicKey)
+ }
+
keyPairs = append(keyPairs, keyPairInfo)
}
@@ -753,12 +843,12 @@ func (s *KeyServiceImpl) DescribeKeyPairs(ctx context.Context, input *ec2.Descri
}
for _, name := range input.KeyNames {
if name != nil && !foundNames[*name] {
- return nil, errors.New(awserrors.ErrorInvalidKeyPairNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidKeyPairNotFound, "The key pair '%s' does not exist", *name)
}
}
for _, id := range input.KeyPairIds {
if id != nil && !foundIDs[*id] {
- return nil, errors.New(awserrors.ErrorInvalidKeyPairNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidKeyPairNotFound, "The keyPairId '%s' does not exist", *id)
}
}
}
@@ -834,7 +924,7 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
if err == nil {
// Object exists - return duplicate error
slog.ErrorContext(ctx, "Key pair already exists", "keyName", keyName)
- return nil, errors.New(awserrors.ErrorInvalidKeyPairDuplicate)
+ return nil, errDuplicateKeyPair()
}
// The material is stored verbatim and later served to instances as their
@@ -852,7 +942,7 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
publicKeyData := bytes.TrimSpace(input.PublicKeyMaterial)
if bytes.ContainsAny(publicKeyData, "\r\n") {
slog.ErrorContext(ctx, "Public key material is not a single key", "keyName", keyName)
- return nil, errors.New(awserrors.ErrorInvalidKeyFormat)
+ return nil, errInvalidKeyFormat()
}
// Parse the authorized-key line ("ssh-rsa AAAAB... comment"), which also
@@ -860,7 +950,7 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
publicKey, _, options, _, err := ssh.ParseAuthorizedKey(publicKeyData)
if err != nil {
slog.ErrorContext(ctx, "Invalid public key format", "keyName", keyName, "err", err)
- return nil, errors.New(awserrors.ErrorInvalidKeyFormat)
+ return nil, errInvalidKeyFormat()
}
// An option prefix ("command=...", "from=...") is not covered by the
@@ -868,13 +958,13 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
// launched with this key pair. Refuse to import access the API cannot report.
if len(options) > 0 {
slog.ErrorContext(ctx, "Public key material carries authorized_keys options", "keyName", keyName, "options", options)
- return nil, errors.New(awserrors.ErrorInvalidKeyFormat)
+ return nil, errInvalidKeyFormat()
}
keyType, err := keyPairType(publicKey)
if err != nil {
slog.ErrorContext(ctx, "Unsupported key type", "algorithm", publicKey.Type(), "keyName", keyName, "err", err)
- return nil, errors.New(awserrors.ErrorInvalidKeyFormat)
+ return nil, errInvalidKeyFormat()
}
fingerprint, err := importedKeyFingerprint(publicKey)
@@ -898,7 +988,8 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
keyPairID := utils.GenerateResourceID("key")
// Build response output
- tags := utils.MapToEC2Tags(utils.ExtractTags(input.TagSpecifications, "key-pair"))
+ tagMap := utils.ExtractTags(input.TagSpecifications, "key-pair")
+ tags := utils.MapToEC2Tags(tagMap)
output := &ec2.ImportKeyPairOutput{
KeyFingerprint: aws.String(fingerprint),
KeyName: aws.String(keyName),
@@ -926,6 +1017,8 @@ func (s *KeyServiceImpl) ImportKeyPair(ctx context.Context, input *ec2.ImportKey
return nil, errors.New(awserrors.ErrorServerInternal)
}
+ s.projectRecordTags(ctx, accountID, keyPairID, tagMap)
+
slog.InfoContext(ctx, "Key pair imported successfully", "keyName", keyName, "fingerprint", fingerprint, "keyPairId", keyPairID, "keyType", keyType)
return output, nil
diff --git a/spinifex/handlers/ec2/key/service_impl_test.go b/spinifex/handlers/ec2/key/service_impl_test.go
index b4108a1c7..d35091f30 100644
--- a/spinifex/handlers/ec2/key/service_impl_test.go
+++ b/spinifex/handlers/ec2/key/service_impl_test.go
@@ -7,6 +7,7 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/json"
+ "errors"
"io"
"strings"
"testing"
@@ -178,6 +179,7 @@ func TestCreateKeyPair_ED25519(t *testing.T) {
// ED25519 has no PEM representation, so it stays in the OpenSSH container.
assert.Contains(t, *out.KeyMaterial, "BEGIN OPENSSH PRIVATE KEY")
+ assert.True(t, strings.HasSuffix(*out.KeyMaterial, "-----END OPENSSH PRIVATE KEY-----\n"))
}
func TestCreateKeyPair_RSA(t *testing.T) {
@@ -195,6 +197,7 @@ func TestCreateKeyPair_RSA(t *testing.T) {
// PKCS#1 PEM, as AWS returns it: get-password-data --priv-launch-key cannot
// read the OpenSSH container ssh-keygen writes by default.
assert.Contains(t, *out.KeyMaterial, "BEGIN RSA PRIVATE KEY")
+ assert.True(t, strings.HasSuffix(*out.KeyMaterial, "-----END RSA PRIVATE KEY-----"), "AWS ends RSA material without a newline")
// The key is generated per-run, so the digest is not knowable in advance.
// Recompute it from the private key the caller was handed: that is what pins
@@ -252,7 +255,7 @@ func TestCreateKeyPair_Duplicate(t *testing.T) {
}, testAccountID)
require.Error(t, err)
assert.Nil(t, out)
- assert.Equal(t, awserrors.ErrorInvalidKeyPairDuplicate, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidKeyPairDuplicate, "The keypair already exists")
}
func TestCreateKeyPair_InvalidKeyType(t *testing.T) {
@@ -269,6 +272,36 @@ func TestCreateKeyPair_InvalidKeyType(t *testing.T) {
assert.Equal(t, "1 validation error detected: Value 'dsa' at 'keyType' failed to satisfy constraint: Member must satisfy enum value set: [rsa, ed25519]", msg)
}
+func TestCreateKeyPair_KeyFormat(t *testing.T) {
+ svc, store := newTestKeyService()
+
+ for _, tc := range []struct {
+ format, msg string
+ }{
+ {"ppk", "The ppk key format is not supported; use pem."},
+ {"der", "1 validation error detected: Value 'der' at 'keyFormat' failed to satisfy constraint: Member must satisfy enum value set: [pem, ppk]"},
+ } {
+ out, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{
+ KeyName: aws.String("fmt-" + tc.format),
+ KeyFormat: aws.String(tc.format),
+ }, testAccountID)
+ assert.Nil(t, out)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+ assert.Equal(t, tc.msg, msg)
+ }
+ assert.Zero(t, store.Count(), "a rejected format must not store a key")
+
+ out, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{
+ KeyName: aws.String("fmt-pem"),
+ KeyFormat: aws.String("pem"),
+ KeyType: aws.String("rsa"),
+ }, testAccountID)
+ require.NoError(t, err)
+ assert.True(t, strings.HasPrefix(*out.KeyMaterial, "-----BEGIN RSA PRIVATE KEY-----"))
+}
+
// ============================================================
// ImportKeyPair Tests
// ============================================================
@@ -344,7 +377,7 @@ func TestImportKeyPair_Duplicate(t *testing.T) {
}, testAccountID)
require.Error(t, err)
assert.Nil(t, out)
- assert.Equal(t, awserrors.ErrorInvalidKeyPairDuplicate, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidKeyPairDuplicate, "The keypair already exists")
}
func TestImportKeyPair_InvalidKeyName(t *testing.T) {
@@ -433,7 +466,7 @@ func TestImportKeyPairInvalidKeyFormat(t *testing.T) {
PublicKeyMaterial: []byte(tt.publicKey),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, tt.expectedErrMsg, err.Error())
+ assertAWSError(t, err, tt.expectedErrMsg, "Key is not in valid OpenSSH public key format")
// Rejection must precede the upload, or the guest is served material
// the API refused.
@@ -457,7 +490,8 @@ func TestDeleteKeyPair_ByKeyName(t *testing.T) {
KeyName: aws.String("to-delete-by-name"),
}, testAccountID)
require.NoError(t, err)
- assert.NotNil(t, result)
+ assert.True(t, aws.BoolValue(result.Return))
+ assert.Equal(t, aws.StringValue(imported.KeyPairId), aws.StringValue(result.KeyPairId))
// Verify public key removed from S3
keyPath := "keys/" + testAccountID + "/to-delete-by-name"
@@ -485,7 +519,8 @@ func TestDeleteKeyPair_ByKeyPairId(t *testing.T) {
KeyPairId: imported.KeyPairId,
}, testAccountID)
require.NoError(t, err)
- assert.NotNil(t, result)
+ assert.True(t, aws.BoolValue(result.Return))
+ assert.Equal(t, aws.StringValue(imported.KeyPairId), aws.StringValue(result.KeyPairId))
// Verify public key removed from S3
keyPath := "keys/" + testAccountID + "/to-delete-by-id"
@@ -512,7 +547,8 @@ func TestDeleteKeyPairIdempotent(t *testing.T) {
KeyName: aws.String("no-such-key"),
}, testAccountID)
require.NoError(t, err)
- assert.NotNil(t, result)
+ assert.True(t, aws.BoolValue(result.Return))
+ assert.Nil(t, result.KeyPairId, "no key was deleted, so none is named")
})
t.Run("NonExistentKeyPairId", func(t *testing.T) {
@@ -520,7 +556,8 @@ func TestDeleteKeyPairIdempotent(t *testing.T) {
KeyPairId: aws.String("key-0123456789abcdef0"),
}, testAccountID)
require.NoError(t, err)
- assert.NotNil(t, result)
+ assert.True(t, aws.BoolValue(result.Return))
+ assert.Nil(t, result.KeyPairId, "no key was deleted, so none is named")
})
}
@@ -649,6 +686,73 @@ func TestDescribeKeyPairs_LegacyED25519Record(t *testing.T) {
testLegacyED25519Fingerprint)
}
+// AWS returns PublicKey only when asked, as one line ending in a newline whose
+// comment is the key pair's name, replacing any comment the caller imported.
+func TestDescribeKeyPairs_IncludePublicKey_Imported(t *testing.T) {
+ svc, _ := newTestKeyService()
+
+ _, err := svc.ImportKeyPair(context.Background(), &ec2.ImportKeyPairInput{
+ KeyName: aws.String("imported"),
+ PublicKeyMaterial: []byte(testED25519PubKey + " tf-user@julian-wattle"),
+ }, testAccountID)
+ require.NoError(t, err)
+
+ out, err := svc.DescribeKeyPairs(context.Background(), &ec2.DescribeKeyPairsInput{
+ IncludePublicKey: aws.Bool(true),
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, out.KeyPairs, 1)
+ assert.Equal(t, testED25519PubKey+" imported\n", aws.StringValue(out.KeyPairs[0].PublicKey))
+
+ for _, input := range []*ec2.DescribeKeyPairsInput{{}, {IncludePublicKey: aws.Bool(false)}} {
+ out, err := svc.DescribeKeyPairs(context.Background(), input, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, out.KeyPairs, 1)
+ assert.Nil(t, out.KeyPairs[0].PublicKey)
+ }
+}
+
+func TestDescribeKeyPairs_IncludePublicKey_Created(t *testing.T) {
+ svc, _ := newTestKeyService()
+
+ for _, keyType := range []string{"ed25519", "rsa"} {
+ created, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{
+ KeyName: aws.String(keyType + "-created"),
+ KeyType: aws.String(keyType),
+ }, testAccountID)
+ require.NoError(t, err)
+ signer, err := ssh.ParsePrivateKey([]byte(aws.StringValue(created.KeyMaterial)))
+ require.NoError(t, err)
+
+ out, err := svc.DescribeKeyPairs(context.Background(), &ec2.DescribeKeyPairsInput{
+ KeyNames: []*string{created.KeyName},
+ IncludePublicKey: aws.Bool(true),
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, out.KeyPairs, 1)
+
+ want := strings.TrimSuffix(string(ssh.MarshalAuthorizedKey(signer.PublicKey())), "\n") + " " + keyType + "-created\n"
+ assert.Equal(t, want, aws.StringValue(out.KeyPairs[0].PublicKey))
+ }
+}
+
+// A listed key whose public half cannot be read fails the call rather than
+// describing it without the PublicKey that was asked for.
+func TestDescribeKeyPairs_IncludePublicKey_MissingMaterial(t *testing.T) {
+ svc, store := newTestKeyService()
+ importTestKey(t, svc, "half-deleted")
+ _, err := store.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+ Bucket: aws.String(testBucket),
+ Key: aws.String("keys/" + testAccountID + "/half-deleted"),
+ })
+ require.NoError(t, err)
+
+ _, err = svc.DescribeKeyPairs(context.Background(), &ec2.DescribeKeyPairsInput{
+ IncludePublicKey: aws.Bool(true),
+ }, testAccountID)
+ require.EqualError(t, err, awserrors.ErrorServerInternal)
+}
+
func TestDescribeKeyPairs_FilterByKeyName(t *testing.T) {
svc, _ := newTestKeyService()
@@ -691,7 +795,7 @@ func TestDescribeKeyPairs_NotFound_ByKeyName(t *testing.T) {
KeyNames: []*string{aws.String("does-not-exist")},
}, testAccountID)
require.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidKeyPair.NotFound")
+ assertAWSError(t, err, awserrors.ErrorInvalidKeyPairNotFound, "The key pair 'does-not-exist' does not exist")
}
// TestDescribeKeyPairs_NotFound_ByKeyPairId is the KeyPairIds counterpart of
@@ -705,7 +809,7 @@ func TestDescribeKeyPairs_NotFound_ByKeyPairId(t *testing.T) {
KeyPairIds: []*string{aws.String("key-doesnotexist")},
}, testAccountID)
require.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidKeyPair.NotFound")
+ assertAWSError(t, err, awserrors.ErrorInvalidKeyPairNotFound, "The keyPairId 'key-doesnotexist' does not exist")
}
// TestDescribeKeyPairs_NotFound_PartialMatch asserts that naming one existing
@@ -1101,3 +1205,50 @@ func TestDescribeKeyPairs_AWSFilterWildcard(t *testing.T) {
require.NoError(t, err)
assert.Len(t, out.KeyPairs, 2)
}
+
+type failingCentralTagStore struct{ puts, deletes int }
+
+func (f *failingCentralTagStore) PutResourceTags(context.Context, string, string, map[string]string) error {
+ f.puts++
+ return errors.New("tag store down")
+}
+
+func (f *failingCentralTagStore) DeleteAllTags(context.Context, string, string) error {
+ f.deletes++
+ return errors.New("tag store down")
+}
+
+// The key pair exists once its records are written, so a central tag store
+// failure is logged and never fails create, import or delete.
+func TestKeyPair_CentralTagStoreFailureDoesNotFailOperation(t *testing.T) {
+ svc, _ := newTestKeyService()
+ central := &failingCentralTagStore{}
+ svc.SetCentralTagStore(central)
+ spec := []*ec2.TagSpecification{{
+ ResourceType: aws.String("key-pair"),
+ Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("k")}},
+ }}
+
+ _, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{
+ KeyName: aws.String("created"), TagSpecifications: spec,
+ }, testAccountID)
+ require.NoError(t, err)
+ _, err = svc.ImportKeyPair(context.Background(), &ec2.ImportKeyPairInput{
+ KeyName: aws.String("imported"), PublicKeyMaterial: []byte(testED25519PubKey), TagSpecifications: spec,
+ }, testAccountID)
+ require.NoError(t, err)
+ _, err = svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("created")}, testAccountID)
+ require.NoError(t, err)
+
+ assert.Equal(t, 2, central.puts)
+ assert.Equal(t, 1, central.deletes)
+}
+
+// assertAWSError checks the code a client receives and the message with it.
+func assertAWSError(t *testing.T, err error, wantCode, wantMessage string) {
+ t.Helper()
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "unresolvable error: %v", err)
+ assert.Equal(t, wantCode, code)
+ assert.Equal(t, wantMessage, msg)
+}
diff --git a/spinifex/handlers/ec2/launchtemplate/expand_test.go b/spinifex/handlers/ec2/launchtemplate/expand_test.go
index 431ecafa0..3574b4fc6 100644
--- a/spinifex/handlers/ec2/launchtemplate/expand_test.go
+++ b/spinifex/handlers/ec2/launchtemplate/expand_test.go
@@ -84,7 +84,7 @@ func TestExpandRunInstances_MissingVersion(t *testing.T) {
}
err := ExpandRunInstances(context.Background(), svc, input, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, err.Error())
+ assertCode(t, err, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
}
func TestExpandRunInstances_IdNameConflict(t *testing.T) {
@@ -115,5 +115,14 @@ func TestExpandRunInstances_UnknownTemplate(t *testing.T) {
}
err := ExpandRunInstances(context.Background(), svc, input, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, err.Error())
+ assertCode(t, err, awserrors.ErrorInvalidLaunchTemplateIdNotFound)
+}
+
+// assertCode checks the code a client receives; the message is the
+// DescribeLaunchTemplateVersions one the expansion passes through.
+func assertCode(t *testing.T, err error, want string) {
+ t.Helper()
+ code, ok := awserrors.ResolveErrorCode(err)
+ require.True(t, ok, "unresolvable error: %v", err)
+ assert.Equal(t, want, code)
}
diff --git a/spinifex/handlers/ec2/launchtemplate/service_impl.go b/spinifex/handlers/ec2/launchtemplate/service_impl.go
index a450e300c..d29dddc0f 100644
--- a/spinifex/handlers/ec2/launchtemplate/service_impl.go
+++ b/spinifex/handlers/ec2/launchtemplate/service_impl.go
@@ -79,11 +79,6 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
// --- key helpers ---
-// headerKey returns the header key: account.lt-.
-func headerKey(accountID, ltID string) string {
- return utils.AccountKey(accountID, ltID)
-}
-
// nameKey returns the name-index key: account.name..
// Launch template names allow characters such as parentheses that are not valid
// in NATS KV keys, so the name must not be used in its raw form here.
@@ -105,7 +100,7 @@ func versionPrefix(accountID, ltID string) string {
// getHeaderByID reads a header and its KV entry (for CAS) by launch template id.
func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
- entry, err := s.kv.Get(ctx, headerKey(accountID, ltID))
+ entry, err := s.kv.Get(ctx, utils.AccountKey(accountID, ltID))
if err != nil {
return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
}
@@ -143,9 +138,19 @@ func (s *LaunchTemplateServiceImpl) resolveHeader(ctx context.Context, accountID
if err := validateTemplateID(id); err != nil {
return nil, nil, err
}
- return s.getHeaderByID(ctx, accountID, id)
+ h, entry, err := s.getHeaderByID(ctx, accountID, id)
+ if err != nil && err.Error() == awserrors.ErrorInvalidLaunchTemplateIdNotFound {
+ return nil, nil, awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateIdNotFound,
+ "The specified launch template, with template ID %s, does not exist.", id)
+ }
+ return h, entry, err
case nm != "":
- return s.getHeaderByName(ctx, accountID, nm)
+ h, entry, err := s.getHeaderByName(ctx, accountID, nm)
+ if err != nil && err.Error() == awserrors.ErrorInvalidLaunchTemplateNameNotFoundException {
+ return nil, nil, awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException,
+ "The specified launch template, with template name %s, does not exist.", nm)
+ }
+ return h, entry, err
default:
return nil, nil, errors.New(awserrors.ErrorMissingParameter)
}
@@ -308,6 +313,11 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in
}, nil
}
+// errNameInUse is AWS's error for a launch template name already taken.
+func errNameInUse() error {
+ return awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException, "Launch template name already in use.")
+}
+
// claimName atomically reserves a template name via kv.Create, with
// repair-on-write: if the name already exists but its header is gone (a crash
// orphan), reclaim it via a revision-guarded CAS update. Only a name whose header
@@ -325,15 +335,15 @@ func (s *LaunchTemplateServiceImpl) claimName(ctx context.Context, accountID, na
// header means the name is taken; any other read error (transient fault, or a
// concurrent in-flight create whose header is not yet written) fails closed so
// the name is never stolen from a possibly-live template.
- _, herr := s.kv.Get(ctx, headerKey(accountID, string(entry.Value())))
+ _, herr := s.kv.Get(ctx, utils.AccountKey(accountID, string(entry.Value())))
switch {
case herr == nil:
- return errors.New(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException)
+ return errNameInUse()
case !errors.Is(herr, jetstream.ErrKeyNotFound):
return errors.New(awserrors.ErrorServerInternal)
}
if _, err := s.kv.Update(ctx, key, []byte(ltID), entry.Revision()); err != nil {
- return errors.New(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException)
+ return errNameInUse()
}
return nil
}
@@ -343,7 +353,7 @@ func (s *LaunchTemplateServiceImpl) putHeader(ctx context.Context, accountID str
if err != nil {
return errors.New(awserrors.ErrorServerInternal)
}
- if _, err := s.kv.Put(ctx, headerKey(accountID, h.LaunchTemplateId), data); err != nil {
+ if _, err := s.kv.Put(ctx, utils.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
return errors.New(awserrors.ErrorServerInternal)
}
return nil
@@ -381,11 +391,11 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont
if src := aws.StringValue(input.SourceVersion); src != "" {
n, err := s.resolveVersionNumber(ctx, accountID, header, src)
if err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, "Launch template version does not exist.")
}
base, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n)
if err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, "Launch template version does not exist.")
}
data = mergeResponseData(base.Data, override)
}
@@ -440,21 +450,22 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in
}
if sel := aws.StringValue(input.DefaultVersion); sel != "" {
+ notFound := fmt.Sprintf("The launch template version %s is not found for the specified launch template.", sel)
n, err := s.resolveVersionNumber(ctx, accountID, header, sel)
if err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, notFound)
}
// Verify the target body exists immediately before the header CAS so a
// concurrent delete degrades to VersionNotFound, never a dangling default.
if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, notFound)
}
header.DefaultVersionNumber = n
data, err := json.Marshal(header)
if err != nil {
return nil, errors.New(awserrors.ErrorServerInternal)
}
- if _, err := s.kv.Update(ctx, headerKey(accountID, header.LaunchTemplateId), data, entry.Revision()); err != nil {
+ if _, err := s.kv.Update(ctx, utils.AccountKey(accountID, header.LaunchTemplateId), data, entry.Revision()); err != nil {
return nil, errors.New(awserrors.ErrorServerInternal)
}
}
@@ -483,7 +494,7 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, in
// Delete the header first: the template immediately vanishes from every
// describe. Version bodies and the name index are best-effort cleanup.
- if err := s.kv.Delete(ctx, headerKey(accountID, header.LaunchTemplateId)); err != nil {
+ if err := s.kv.Delete(ctx, utils.AccountKey(accountID, header.LaunchTemplateId)); err != nil {
return nil, errors.New(awserrors.ErrorServerInternal)
}
if err := s.kv.Delete(ctx, nameKey(accountID, header.LaunchTemplateName)); err != nil {
@@ -514,6 +525,13 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Con
return nil, errors.New(awserrors.ErrorMissingParameter)
}
+ // Naming the default version fails the whole request, so nothing is deleted.
+ for _, v := range input.Versions {
+ if n, perr := strconv.ParseInt(aws.StringValue(v), 10, 64); perr == nil && n == header.DefaultVersionNumber {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The default version cannot be deleted. Either specify another version as default or delete the launch template.")
+ }
+ }
+
out := &ec2.DeleteLaunchTemplateVersionsOutput{}
for _, v := range input.Versions {
@@ -524,14 +542,9 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Con
deleteErrorItem(header, 0, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, "invalid version number"))
continue
}
- if n == header.DefaultVersionNumber {
- out.UnsuccessfullyDeletedLaunchTemplateVersions = append(out.UnsuccessfullyDeletedLaunchTemplateVersions,
- deleteErrorItem(header, n, awserrors.ErrorInvalidParameterValue, "cannot delete the default version of a launch template"))
- continue
- }
if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil {
out.UnsuccessfullyDeletedLaunchTemplateVersions = append(out.UnsuccessfullyDeletedLaunchTemplateVersions,
- deleteErrorItem(header, n, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, "version does not exist"))
+ deleteErrorItem(header, n, launchTemplateVersionDoesNotExist, "The launch template version does not exist."))
continue
}
if err := s.kv.Delete(ctx, versionKey(accountID, header.LaunchTemplateId, n)); err != nil {
@@ -551,6 +564,10 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Con
return out, nil
}
+// launchTemplateVersionDoesNotExist is the per-version code AWS reports in a
+// DeleteLaunchTemplateVersions result; it is never a request-level error.
+const launchTemplateVersionDoesNotExist = "launchTemplateVersionDoesNotExist"
+
func deleteErrorItem(h *LaunchTemplateHeader, n int64, code, msg string) *ec2.DeleteLaunchTemplateVersionsResponseErrorItem {
item := &ec2.DeleteLaunchTemplateVersionsResponseErrorItem{
LaunchTemplateId: aws.String(h.LaunchTemplateId),
@@ -629,14 +646,15 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplates(ctx context.Context,
templates = append(templates, headerToEC2(&h, latestByID[h.LaunchTemplateId]))
}
+ const missing = "At least one of the launch templates specified in the request does not exist."
for name := range nameSet {
if !foundNames[name] {
- return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException, missing)
}
}
for id := range idSet {
if !foundIDs[id] {
- return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateIdNotFound, missing)
}
}
@@ -732,12 +750,14 @@ func (s *LaunchTemplateServiceImpl) selectVersionNumbers(ctx context.Context, ac
seen := make(map[int64]bool)
var out []int64
for _, v := range input.Versions {
+ notFound := fmt.Sprintf("Could not find the specified version %s for the launch template with ID %s.",
+ aws.StringValue(v), header.LaunchTemplateId)
n, err := s.resolveVersionNumber(ctx, accountID, header, aws.StringValue(v))
if err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, notFound)
}
if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil {
- return nil, err
+ return nil, withVersionNotFoundMessage(err, notFound)
}
if !seen[n] {
seen[n] = true
@@ -858,6 +878,15 @@ func versionRecToEC2(h *LaunchTemplateHeader, rec *LaunchTemplateVersionRec, isD
// --- misc helpers ---
+// withVersionNotFoundMessage attaches AWS's message, which differs by action,
+// to a VersionNotFound error and passes any other error through.
+func withVersionNotFoundMessage(err error, message string) error {
+ if err.Error() != awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound {
+ return err
+ }
+ return awserrors.Errorf(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, "%s", message)
+}
+
// isHeaderKey reports whether k is a template header key (account.lt-) as
// opposed to a name index, version body (account.lt-.v), or _version key.
func isHeaderKey(k, prefix string) bool {
diff --git a/spinifex/handlers/ec2/launchtemplate/service_impl_test.go b/spinifex/handlers/ec2/launchtemplate/service_impl_test.go
index adc203c3c..30e7ede3b 100644
--- a/spinifex/handlers/ec2/launchtemplate/service_impl_test.go
+++ b/spinifex/handlers/ec2/launchtemplate/service_impl_test.go
@@ -7,8 +7,9 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
- "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -84,7 +85,7 @@ func TestCreateLaunchTemplate_DuplicateName(t *testing.T) {
LaunchTemplateData: &ec2.RequestLaunchTemplateData{ImageId: aws.String("ami-1")},
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException, "Launch template name already in use.")
}
func TestLaunchTemplateNameWithKVUnsafeCharacters(t *testing.T) {
@@ -115,7 +116,7 @@ func TestCreateLaunchTemplate_OrphanNameReclaim(t *testing.T) {
lt := createTemplate(t, svc, "orphan", "t3.micro")
// Simulate a crash after the name claim but with the header lost.
- require.NoError(t, svc.kv.Delete(t.Context(), headerKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
+ require.NoError(t, svc.kv.Delete(t.Context(), utils.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
// The name index still points at the now-orphaned id; create must reclaim it.
out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{
@@ -158,7 +159,8 @@ func TestDescribeLaunchTemplates_UnknownName(t *testing.T) {
LaunchTemplateNames: []*string{aws.String("missing")},
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateNameNotFoundException, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateNameNotFoundException,
+ "At least one of the launch templates specified in the request does not exist.")
}
func TestDescribeLaunchTemplates_TagFilter(t *testing.T) {
@@ -331,7 +333,39 @@ func TestDescribeLaunchTemplateVersions_MissingVersion(t *testing.T) {
Versions: []*string{aws.String("99")},
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound,
+ "Could not find the specified version 99 for the launch template with ID "+aws.StringValue(lt.LaunchTemplateId)+".")
+}
+
+func TestLaunchTemplate_NotFoundMessagesNameTheTemplate(t *testing.T) {
+ svc := setupTestService(t)
+ lt := createTemplate(t, svc, "web", "t3.micro")
+
+ _, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{
+ LaunchTemplateName: aws.String("missing"),
+ }, testAccountID)
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateNameNotFoundException,
+ "The specified launch template, with template name missing, does not exist.")
+
+ _, err = svc.DescribeLaunchTemplateVersions(context.Background(), &ec2.DescribeLaunchTemplateVersionsInput{
+ LaunchTemplateId: aws.String("lt-0123456789abcdef0"),
+ }, testAccountID)
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateIdNotFound,
+ "The specified launch template, with template ID lt-0123456789abcdef0, does not exist.")
+
+ _, err = svc.ModifyLaunchTemplate(context.Background(), &ec2.ModifyLaunchTemplateInput{
+ LaunchTemplateId: lt.LaunchTemplateId,
+ DefaultVersion: aws.String("42"),
+ }, testAccountID)
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound,
+ "The launch template version 42 is not found for the specified launch template.")
+
+ _, err = svc.CreateLaunchTemplateVersion(context.Background(), &ec2.CreateLaunchTemplateVersionInput{
+ LaunchTemplateId: lt.LaunchTemplateId,
+ SourceVersion: aws.String("42"),
+ LaunchTemplateData: &ec2.RequestLaunchTemplateData{InstanceType: aws.String("t3.small")},
+ }, testAccountID)
+ assertAWSError(t, err, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, "Launch template version does not exist.")
}
func TestDescribeLaunchTemplateVersions_LatestAfterTailDelete(t *testing.T) {
@@ -388,7 +422,8 @@ func TestModifyLaunchTemplate_DefaultToMissingVersion(t *testing.T) {
DefaultVersion: aws.String("42"),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, err.Error())
+ code, _ := awserrors.ResolveErrorCode(err)
+ assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, code)
}
// --- DeleteLaunchTemplateVersions ---
@@ -396,15 +431,23 @@ func TestModifyLaunchTemplate_DefaultToMissingVersion(t *testing.T) {
func TestDeleteLaunchTemplateVersions_RejectDefault(t *testing.T) {
svc := setupTestService(t)
lt := createTemplate(t, svc, "web", "t3.micro") // default = v1
- out, err := svc.DeleteLaunchTemplateVersions(context.Background(), &ec2.DeleteLaunchTemplateVersionsInput{
- LaunchTemplateId: lt.LaunchTemplateId,
- Versions: []*string{aws.String("1")},
+ id := aws.StringValue(lt.LaunchTemplateId)
+ addVersion(t, svc, id, "a", "") // v2
+
+ // The default alongside a deletable version still fails the whole request.
+ _, err := svc.DeleteLaunchTemplateVersions(context.Background(), &ec2.DeleteLaunchTemplateVersionsInput{
+ LaunchTemplateId: aws.String(id),
+ Versions: []*string{aws.String("2"), aws.String("1")},
}, testAccountID)
+ require.Error(t, err)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+ assert.Equal(t, "The default version cannot be deleted. Either specify another version as default or delete the launch template.", msg)
+
+ nums, err := svc.listVersionNumbers(t.Context(), testAccountID, id)
require.NoError(t, err)
- assert.Empty(t, out.SuccessfullyDeletedLaunchTemplateVersions)
- require.Len(t, out.UnsuccessfullyDeletedLaunchTemplateVersions, 1)
- assert.Equal(t, awserrors.ErrorInvalidParameterValue,
- aws.StringValue(out.UnsuccessfullyDeletedLaunchTemplateVersions[0].ResponseError.Code))
+ assert.ElementsMatch(t, []int64{1, 2}, nums)
}
func TestDeleteLaunchTemplateVersions_SuccessAndMissing(t *testing.T) {
@@ -421,7 +464,10 @@ func TestDeleteLaunchTemplateVersions_SuccessAndMissing(t *testing.T) {
require.Len(t, out.SuccessfullyDeletedLaunchTemplateVersions, 1)
assert.Equal(t, int64(2), aws.Int64Value(out.SuccessfullyDeletedLaunchTemplateVersions[0].VersionNumber))
require.Len(t, out.UnsuccessfullyDeletedLaunchTemplateVersions, 1)
- assert.Equal(t, int64(9), aws.Int64Value(out.UnsuccessfullyDeletedLaunchTemplateVersions[0].VersionNumber))
+ missing := out.UnsuccessfullyDeletedLaunchTemplateVersions[0]
+ assert.Equal(t, int64(9), aws.Int64Value(missing.VersionNumber))
+ assert.Equal(t, "launchTemplateVersionDoesNotExist", aws.StringValue(missing.ResponseError.Code))
+ assert.Equal(t, "The launch template version does not exist.", aws.StringValue(missing.ResponseError.Message))
}
// --- DeleteLaunchTemplate ---
@@ -442,7 +488,8 @@ func TestDeleteLaunchTemplate_RemovesEverything(t *testing.T) {
LaunchTemplateIds: []*string{aws.String(id)},
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, err.Error())
+ code, _ := awserrors.ResolveErrorCode(err)
+ assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, code)
// Version bodies and name index gone.
nums, err := svc.listVersionNumbers(t.Context(), testAccountID, id)
@@ -461,7 +508,8 @@ func TestDeleteLaunchTemplate_Unknown(t *testing.T) {
LaunchTemplateId: aws.String("lt-doesnotexist000"),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, err.Error())
+ code, _ := awserrors.ResolveErrorCode(err)
+ assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, code)
}
func TestResolveHeader_IdNameConflict(t *testing.T) {
@@ -492,3 +540,13 @@ func TestDescribeLaunchTemplates_MalformedId(t *testing.T) {
require.Error(t, err)
assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdMalformed, err.Error())
}
+
+// assertAWSError checks the code a client receives and the message with it.
+func assertAWSError(t *testing.T, err error, wantCode, wantMessage string) {
+ t.Helper()
+ require.Error(t, err)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "unresolvable error: %v", err)
+ assert.Equal(t, wantCode, code)
+ assert.Equal(t, wantMessage, msg)
+}
diff --git a/spinifex/handlers/ec2/natgw/service_impl.go b/spinifex/handlers/ec2/natgw/service_impl.go
index 535921f8b..52201451b 100644
--- a/spinifex/handlers/ec2/natgw/service_impl.go
+++ b/spinifex/handlers/ec2/natgw/service_impl.go
@@ -84,23 +84,16 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
}, nil
}
-// getOrCreateDeletedBucket creates the expiring bucket without changing an
-// existing bucket's replica or placement configuration.
+// getOrCreateDeletedBucket creates the expiring bucket that records a deletion
+// for as long as a client may still ask about it.
func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
// Terraform polls DescribeNatGateways after delete and expects state=deleted.
- kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: KVBucketDeletedNatGateways,
+ return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+ Name: KVBucketDeletedNatGateways,
Description: "Deleted NAT Gateways (auto-expire after 1 hour)",
History: 1,
TTL: time.Hour,
})
- if err == nil {
- return kv, nil
- }
- if !errors.Is(err, jetstream.ErrBucketExists) {
- return nil, err
- }
- return js.KeyValue(ctx, KVBucketDeletedNatGateways)
}
// natGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway topics.
diff --git a/spinifex/handlers/ec2/placementgroup/service_impl.go b/spinifex/handlers/ec2/placementgroup/service_impl.go
index 8ad56fc9f..246ea3f18 100644
--- a/spinifex/handlers/ec2/placementgroup/service_impl.go
+++ b/spinifex/handlers/ec2/placementgroup/service_impl.go
@@ -12,6 +12,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/config"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
@@ -49,6 +50,7 @@ type PlacementGroupServiceImpl struct {
config *config.Config
natsConn *nats.Conn
kv jetstream.KeyValue
+ region string
}
// NewPlacementGroupServiceImplWithNATS creates a placement group service with NATS JetStream.
@@ -66,12 +68,18 @@ func NewPlacementGroupServiceImplWithNATS(ctx context.Context, cfg *config.Confi
return nil, fmt.Errorf("migrate %s: %w", KVBucketPlacementGroups, err)
}
+ region := config.DefaultAWSRegion
+ if cfg != nil && cfg.Region != "" {
+ region = cfg.Region
+ }
+
slog.Info("Placement group service initialized with JetStream KV", "bucket", KVBucketPlacementGroups)
return &PlacementGroupServiceImpl{
config: cfg,
natsConn: natsConn,
kv: kv,
+ region: region,
}, nil
}
@@ -123,7 +131,7 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in
// Atomic create-if-not-exists to prevent TOCTOU race on duplicate names
if _, err := s.kv.Create(ctx, key, data); err != nil {
// Create fails if key already exists
- return nil, errors.New(awserrors.ErrorInvalidPlacementGroupDuplicate)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupDuplicate, "The placement group '%s' already exists.", groupName)
}
slog.InfoContext(ctx, "CreatePlacementGroup completed", "groupId", groupID, "groupName", groupName, "strategy", strategy, "accountID", accountID)
@@ -144,7 +152,8 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
entry, err := s.kv.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
+ // Lower case, unlike DescribePlacementGroups' message for the same code.
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
}
var record PlacementGroupRecord
@@ -252,9 +261,20 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
found[*g.GroupName] = true
}
}
- for name := range nameSet {
- if !found[name] {
- return nil, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
+ for _, name := range input.GroupNames {
+ if name != nil && !found[*name] {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The Placement Group '%s' is unknown.", *name)
+ }
+ }
+ }
+ if len(idSet) > 0 {
+ found := make(map[string]bool)
+ for _, g := range groups {
+ found[aws.StringValue(g.GroupId)] = true
+ }
+ for _, id := range input.GroupIds {
+ if id != nil && !found[*id] {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The Placement Group '%s' is unknown.", *id)
}
}
}
@@ -269,7 +289,7 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
// pgMatchesFilters checks whether a placement group record matches all parsed awsfilters.
func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
switch name {
@@ -293,14 +313,6 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
if !awsfilters.MatchesAny(values, record.GroupName) {
return false
}
- case "tag-key":
- if !pgMatchesAnyTag(record.Tags, values, func(k, _ string) string { return k }) {
- return false
- }
- case "tag-value":
- if !pgMatchesAnyTag(record.Tags, values, func(_, v string) string { return v }) {
- return false
- }
default:
return false
}
@@ -308,17 +320,6 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
return awsfilters.MatchesTags(filters, record.Tags)
}
-// pgMatchesAnyTag reports whether any tag's selected field (key or value)
-// matches any of the filter values.
-func pgMatchesAnyTag(tags map[string]string, values []string, field func(k, v string) string) bool {
- for k, v := range tags {
- if awsfilters.MatchesAny(values, field(k, v)) {
- return true
- }
- }
- return false
-}
-
const maxCASRetries = 5
// errPGAbsent and errPGContended mark the two kvutil outcomes that map to
@@ -568,6 +569,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context
// recordToEC2 converts an internal record to the AWS SDK PlacementGroup type.
func (s *PlacementGroupServiceImpl) recordToEC2(record *PlacementGroupRecord) *ec2.PlacementGroup {
pg := &ec2.PlacementGroup{
+ GroupArn: aws.String(arn.FormatEC2(arn.EC2PlacementGroup, s.region, record.AccountID, record.GroupName)),
GroupId: aws.String(record.GroupId),
GroupName: aws.String(record.GroupName),
Strategy: aws.String(record.Strategy),
diff --git a/spinifex/handlers/ec2/placementgroup/service_impl_test.go b/spinifex/handlers/ec2/placementgroup/service_impl_test.go
index e4c85a522..8aff8e8ba 100644
--- a/spinifex/handlers/ec2/placementgroup/service_impl_test.go
+++ b/spinifex/handlers/ec2/placementgroup/service_impl_test.go
@@ -7,9 +7,10 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
- "github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -83,6 +84,34 @@ func TestCreatePlacementGroup_Cluster(t *testing.T) {
assert.Nil(t, pg.SpreadLevel)
}
+// AWS builds GroupArn from the group name, not its ID, on create and describe.
+func TestPlacementGroup_GroupArnFromName(t *testing.T) {
+ _, nc, _ := testutil.StartTestJetStream(t)
+ svc, err := NewPlacementGroupServiceImplWithNATS(t.Context(), &config.Config{Region: "ap-southeast-2"}, nc)
+ require.NoError(t, err)
+
+ for _, strategy := range []string{"cluster", "spread"} {
+ name := "arn-" + strategy
+ want := "arn:aws:ec2:ap-southeast-2:123456789012:placement-group/" + name
+
+ pg := createTestGroup(t, svc, name, strategy)
+ assert.Equal(t, want, aws.StringValue(pg.GroupArn))
+
+ out, err := svc.DescribePlacementGroups(context.Background(), &ec2.DescribePlacementGroupsInput{
+ GroupNames: []*string{aws.String(name)},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, out.PlacementGroups, 1)
+ assert.Equal(t, want, aws.StringValue(out.PlacementGroups[0].GroupArn))
+ }
+}
+
+func TestPlacementGroup_GroupArnDefaultRegion(t *testing.T) {
+ svc := setupTestService(t)
+ pg := createTestGroup(t, svc, "no-region", "spread")
+ assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:placement-group/no-region", aws.StringValue(pg.GroupArn))
+}
+
func TestCreatePlacementGroup_DuplicateName(t *testing.T) {
svc := setupTestService(t)
createTestGroup(t, svc, "dup-group", "spread")
@@ -92,7 +121,7 @@ func TestCreatePlacementGroup_DuplicateName(t *testing.T) {
Strategy: aws.String("spread"),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidPlacementGroupDuplicate, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidPlacementGroupDuplicate, "The placement group 'dup-group' already exists.")
}
func TestCreatePlacementGroup_PartitionRejected(t *testing.T) {
@@ -190,13 +219,31 @@ func TestDeletePlacementGroup_Success(t *testing.T) {
assert.Empty(t, out.PlacementGroups)
}
+// AWS answers the ID of a deleted group with InvalidPlacementGroup.Unknown
+// naming that ID, not an empty list.
+func TestDescribePlacementGroups_DeletedGroupId(t *testing.T) {
+ svc := setupTestService(t)
+ kept := createTestGroup(t, svc, "kept", "spread")
+ deleted := createTestGroup(t, svc, "deleted", "spread")
+ _, err := svc.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{GroupName: deleted.GroupName}, testAccountID)
+ require.NoError(t, err)
+
+ _, err = svc.DescribePlacementGroups(context.Background(), &ec2.DescribePlacementGroupsInput{
+ GroupIds: []*string{kept.GroupId, deleted.GroupId},
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidPlacementGroupUnknown, code)
+ assert.Equal(t, "The Placement Group '"+*deleted.GroupId+"' is unknown.", msg)
+}
+
func TestDeletePlacementGroup_NotFound(t *testing.T) {
svc := setupTestService(t)
_, err := svc.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{
GroupName: aws.String("nonexistent"),
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidPlacementGroupUnknown, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group 'nonexistent' is unknown.")
}
func TestDeletePlacementGroup_InUse(t *testing.T) {
@@ -333,7 +380,7 @@ func TestDescribePlacementGroups_NameNotFound(t *testing.T) {
GroupNames: []*string{aws.String("ghost")},
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidPlacementGroupUnknown, err.Error())
+ assertAWSError(t, err, awserrors.ErrorInvalidPlacementGroupUnknown, "The Placement Group 'ghost' is unknown.")
}
func TestDescribePlacementGroups_AccountScoped(t *testing.T) {
@@ -859,3 +906,12 @@ func TestRemoveInstance_ReadFailureIsNotSuccess(t *testing.T) {
require.Error(t, err)
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServerInternal), "got %v", err)
}
+
+// assertAWSError checks the code a client receives and the message with it.
+func assertAWSError(t *testing.T, err error, wantCode, wantMessage string) {
+ t.Helper()
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "unresolvable error: %v", err)
+ assert.Equal(t, wantCode, code)
+ assert.Equal(t, wantMessage, msg)
+}
diff --git a/spinifex/handlers/ec2/routetable/service_impl.go b/spinifex/handlers/ec2/routetable/service_impl.go
index fa6d0fc53..f5a4d023d 100644
--- a/spinifex/handlers/ec2/routetable/service_impl.go
+++ b/spinifex/handlers/ec2/routetable/service_impl.go
@@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"log/slog"
+ "net/netip"
+ "slices"
"strings"
"time"
@@ -19,6 +21,7 @@ import (
handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw"
handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+ "github.com/mulgadc/spinifex/spinifex/paging"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -91,7 +94,7 @@ func (s *RouteTableServiceImpl) getRouteTable(ctx context.Context, accountID, rt
entry, err := s.rtbKV.Get(ctx, utils.AccountKey(accountID, rtbID))
if err != nil {
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidRouteTableIDNotFound)
+ return nil, rtbNotFoundError(rtbID)
}
slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
@@ -145,7 +148,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
case err == nil:
return nil
case errors.Is(err, errRTBAbsent):
- return errors.New(awserrors.ErrorInvalidRouteTableIDNotFound)
+ return rtbNotFoundError(rtbID)
case errors.Is(err, errRTBContended):
// Contended rather than broken.
slog.ErrorContext(ctx, "Route table CAS retries exhausted under contention",
@@ -168,7 +171,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
func (s *RouteTableServiceImpl) getVPCCidr(ctx context.Context, accountID, vpcID string) (string, error) {
entry, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID))
if err != nil {
- return "", errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return "", awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
}
var vpcRecord handlers_ec2_vpc.VPCRecord
if err := json.Unmarshal(entry.Value(), &vpcRecord); err != nil {
@@ -550,7 +553,8 @@ func (s *RouteTableServiceImpl) CreateRouteTable(ctx context.Context, input *ec2
}
return &ec2.CreateRouteTableOutput{
- RouteTable: recordToEC2(record),
+ RouteTable: recordToEC2(record),
+ ClientToken: input.ClientToken,
}, nil
}
@@ -569,13 +573,13 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
}
if record.IsMain {
- return nil, errors.New(awserrors.ErrorDependencyViolation)
+ return nil, awserrors.HasDependencies("routeTable", rtbID)
}
// Check for non-main associations (subnets still using this table)
for _, assoc := range record.Associations {
if !assoc.Main && assoc.SubnetId != "" {
- return nil, errors.New(awserrors.ErrorDependencyViolation)
+ return nil, awserrors.HasDependencies("routeTable", rtbID)
}
}
@@ -587,6 +591,13 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
return &ec2.DeleteRouteTableOutput{}, nil
}
+var describeRouteTablesPaging = paging.EC2{
+ MaxResults: 100,
+ TooLarge: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 100.",
+ TooSmall: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter RouteTableIds cannot be used with the parameter MaxResults",
+}
+
var describeRouteTablesValidFilters = map[string]bool{
"vpc-id": true,
"route-table-id": true,
@@ -599,10 +610,17 @@ var describeRouteTablesValidFilters = map[string]bool{
"route.state": true,
"route.origin": true,
"owner-id": true,
+ "tag-key": true,
+ "tag-value": true,
}
// DescribeRouteTables lists route tables, optionally filtered.
func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *ec2.DescribeRouteTablesInput, accountID string) (*ec2.DescribeRouteTablesOutput, error) {
+ // Validated but not paged: AWS's paging of DescribeRouteTables is unobserved.
+ if _, err := describeRouteTablesPaging.Parse(input.MaxResults, input.NextToken, len(input.RouteTableIds)); err != nil {
+ return nil, err
+ }
+
rtbIDs := make(map[string]bool)
for _, id := range input.RouteTableIds {
if id != nil {
@@ -658,9 +676,9 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
}
// Return error if specific IDs were requested but not found
- for id := range rtbIDs {
- if !foundIDs[id] {
- return nil, errors.New(awserrors.ErrorInvalidRouteTableIDNotFound)
+ for _, id := range input.RouteTableIds {
+ if id != nil && !foundIDs[*id] {
+ return nil, rtbNotFoundError(*id)
}
}
@@ -670,6 +688,44 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
}
// CreateRoute adds a route to a route table.
+// gatewayNotFoundError is AWS's answer to a route naming a gateway that does
+// not exist, whatever the gateway's type.
+// checkRouteOutsideVPC rejects a gateway route whose destination is, or lies
+// inside, one of the VPC's CIDR blocks (the table's local routes). AWS allows
+// only an interface or instance there, which Spinifex does not route to.
+func checkRouteOutsideVPC(record *RouteTableRecord, destCidr string) error {
+ dest, parseErr := netip.ParsePrefix(destCidr)
+ if parseErr != nil {
+ return nil //nolint:nilerr // an unparseable destination is not a VPC-range conflict
+ }
+ for _, r := range record.Routes {
+ if r.GatewayId != "local" {
+ continue
+ }
+ vpc, err := netip.ParsePrefix(r.DestinationCidrBlock)
+ if err != nil {
+ continue
+ }
+ if dest.Bits() >= vpc.Bits() && vpc.Contains(dest.Addr()) {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "The destination CIDR block %s is equal to or more specific than one of this VPC's CIDR blocks. This route can target only an interface or an instance.", destCidr)
+ }
+ }
+ return nil
+}
+
+func rtbNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidRouteTableIDNotFound, "routeTable", id)
+}
+
+func associationNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidAssociationIDNotFound, "association", id)
+}
+
+func gatewayNotFoundError(id string) error {
+ return awserrors.Errorf(awserrors.ErrorInvalidGatewayIDNotFound, "The gateway ID '%s' does not exist", id)
+}
+
func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.CreateRouteInput, accountID string) (*ec2.CreateRouteOutput, error) {
if input.RouteTableId == nil || *input.RouteTableId == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -686,14 +742,10 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
return nil, err
}
- // Check for duplicate destination
- for _, r := range record.Routes {
- if r.DestinationCidrBlock == destCidr {
- return nil, errors.New(awserrors.ErrorRouteAlreadyExists)
- }
- }
-
+ // AWS validates the target, then the destination, then looks for an
+ // existing route, so events are held back until all three pass.
var route RouteRecord
+ var publish func()
switch {
case input.GatewayId != nil && *input.GatewayId != "":
@@ -701,7 +753,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
// Verify IGW exists and is attached to the same VPC
igwEntry, err := s.igwKV.Get(ctx, utils.AccountKey(accountID, igwID))
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
+ return nil, gatewayNotFoundError(igwID)
}
var igwRecord handlers_ec2_igw.IGWRecord
if err := json.Unmarshal(igwEntry.Value(), &igwRecord); err != nil {
@@ -722,7 +774,9 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
// Publish vpc.add-igw-route events for each subnet associated with this
// route table so the network subscriber installs per-subnet egress policies.
- s.publishIGWRouteEvents(ctx, accountID, "vpc.add-igw-route", record, igwRecord.VpcId, igwID, destCidr)
+ publish = func() {
+ s.publishIGWRouteEvents(ctx, accountID, "vpc.add-igw-route", record, igwRecord.VpcId, igwID, destCidr)
+ }
case input.NatGatewayId != nil && *input.NatGatewayId != "":
natgwID := *input.NatGatewayId
@@ -750,12 +804,32 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
}
// Publish vpc.add-nat-gateway events for each subnet associated with this route table
- s.publishNatGatewayEvents(ctx, accountID, record, natgwRecord.VpcId, natgwID, natgwRecord.PublicIp, destCidr)
+ publish = func() {
+ s.publishNatGatewayEvents(ctx, accountID, record, natgwRecord.VpcId, natgwID, natgwRecord.PublicIp, destCidr)
+ }
default:
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
+ if err := checkRouteOutsideVPC(record, destCidr); err != nil {
+ return nil, err
+ }
+
+ // A repeat of an existing route (same destination, same target) is a
+ // successful no-op; a different target is a conflict.
+ for _, r := range record.Routes {
+ if r.DestinationCidrBlock != destCidr {
+ continue
+ }
+ if r.GatewayId == route.GatewayId && r.NatGatewayId == route.NatGatewayId {
+ return &ec2.CreateRouteOutput{Return: aws.Bool(true)}, nil
+ }
+ return nil, awserrors.Errorf(awserrors.ErrorRouteAlreadyExists, "The route identified by %s already exists.", destCidr)
+ }
+
+ publish()
+
record.Routes = append(record.Routes, route)
if err := s.putRouteTable(ctx, accountID, record); err != nil {
@@ -800,7 +874,7 @@ func (s *RouteTableServiceImpl) DeleteRoute(ctx context.Context, input *ec2.Dele
}
}
if idx < 0 {
- return nil, errors.New(awserrors.ErrorInvalidRouteNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidRouteNotFound, "no route with destination-cidr-block %s in route table %s", destCidr, rtbID)
}
departing := record.Routes[idx]
@@ -853,6 +927,23 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep
return nil, err
}
+ // V1: only GatewayId target supported
+ if input.GatewayId == nil || *input.GatewayId == "" {
+ return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+ }
+
+ igwID := *input.GatewayId
+
+ // AWS reports a missing gateway ahead of a missing route.
+ igwEntry, err := s.igwKV.Get(ctx, utils.AccountKey(accountID, igwID))
+ if err != nil {
+ return nil, gatewayNotFoundError(igwID)
+ }
+ var igwRecord handlers_ec2_igw.IGWRecord
+ if err := json.Unmarshal(igwEntry.Value(), &igwRecord); err != nil {
+ return nil, errors.New(awserrors.ErrorServerInternal)
+ }
+
idx := -1
for i, r := range record.Routes {
if r.DestinationCidrBlock == destCidr {
@@ -861,29 +952,13 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep
}
}
if idx < 0 {
- return nil, errors.New(awserrors.ErrorInvalidRouteNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "There is no route defined for '%s' in the route table. Use CreateRoute instead.", destCidr)
}
if record.Routes[idx].GatewayId == "local" {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
-
- // V1: only GatewayId target supported
- if input.GatewayId == nil || *input.GatewayId == "" {
- return nil, errors.New(awserrors.ErrorInvalidParameterValue)
- }
-
- igwID := *input.GatewayId
-
- // Verify IGW exists and is attached to same VPC
- igwEntry, err := s.igwKV.Get(ctx, utils.AccountKey(accountID, igwID))
- if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound)
- }
- var igwRecord handlers_ec2_igw.IGWRecord
- if err := json.Unmarshal(igwEntry.Value(), &igwRecord); err != nil {
- return nil, errors.New(awserrors.ErrorServerInternal)
- }
if igwRecord.VpcId != record.VpcId {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
@@ -1006,7 +1081,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu
keys, err := s.rtbKV.Keys(ctx)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
- return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound)
+ return nil, associationNotFoundError(assocID)
}
slog.ErrorContext(ctx, "Failed to list route table keys", "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
@@ -1071,7 +1146,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu
}
}
- return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound)
+ return nil, associationNotFoundError(assocID)
}
// ReplaceRouteTableAssociation atomically moves a subnet from one route table to another.
@@ -1097,7 +1172,7 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context
keys, err := s.rtbKV.Keys(ctx)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
- return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound)
+ return nil, associationNotFoundError(assocID)
}
slog.ErrorContext(ctx, "Failed to list route table keys", "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
@@ -1189,13 +1264,13 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context
}
}
- return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound)
+ return nil, associationNotFoundError(assocID)
}
// rtbMatchesFilters checks if a route table record matches all parsed awsfilters.
func rtbMatchesFilters(record *RouteTableRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
switch name {
@@ -1208,92 +1283,53 @@ func rtbMatchesFilters(record *RouteTableRecord, filters map[string][]string) bo
return false
}
case "association.main":
- hasMain := false
- for _, a := range record.Associations {
- if a.Main {
- hasMain = true
- break
- }
- }
+ hasMain := slices.ContainsFunc(record.Associations, func(a AssociationRecord) bool {
+ return a.Main
+ })
wantMain := awsfilters.MatchesAny(values, "true")
if wantMain != hasMain {
return false
}
case "association.route-table-association-id":
- found := false
- for _, a := range record.Associations {
- if awsfilters.MatchesAny(values, a.AssociationId) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Associations, func(a AssociationRecord) bool {
+ return awsfilters.MatchesAny(values, a.AssociationId)
+ }) {
return false
}
case "association.subnet-id":
- found := false
- for _, a := range record.Associations {
- if awsfilters.MatchesAny(values, a.SubnetId) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Associations, func(a AssociationRecord) bool {
+ return awsfilters.MatchesAny(values, a.SubnetId)
+ }) {
return false
}
case "route.destination-cidr-block":
- found := false
- for _, r := range record.Routes {
- if awsfilters.MatchesAny(values, r.DestinationCidrBlock) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Routes, func(r RouteRecord) bool {
+ return awsfilters.MatchesAny(values, r.DestinationCidrBlock)
+ }) {
return false
}
case "route.gateway-id":
- found := false
- for _, r := range record.Routes {
- if awsfilters.MatchesAny(values, r.GatewayId) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Routes, func(r RouteRecord) bool {
+ return awsfilters.MatchesAny(values, r.GatewayId)
+ }) {
return false
}
case "route.nat-gateway-id":
- found := false
- for _, r := range record.Routes {
- if awsfilters.MatchesAny(values, r.NatGatewayId) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Routes, func(r RouteRecord) bool {
+ return awsfilters.MatchesAny(values, r.NatGatewayId)
+ }) {
return false
}
case "route.state":
- found := false
- for _, r := range record.Routes {
- if awsfilters.MatchesAny(values, r.State) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Routes, func(r RouteRecord) bool {
+ return awsfilters.MatchesAny(values, r.State)
+ }) {
return false
}
case "route.origin":
- found := false
- for _, r := range record.Routes {
- if awsfilters.MatchesAny(values, r.Origin) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.Routes, func(r RouteRecord) bool {
+ return awsfilters.MatchesAny(values, r.Origin)
+ }) {
return false
}
case "owner-id":
diff --git a/spinifex/handlers/ec2/routetable/service_impl_test.go b/spinifex/handlers/ec2/routetable/service_impl_test.go
index 7dfd29cb2..2de2bfdf0 100644
--- a/spinifex/handlers/ec2/routetable/service_impl_test.go
+++ b/spinifex/handlers/ec2/routetable/service_impl_test.go
@@ -103,9 +103,11 @@ func TestCreateRouteTable(t *testing.T) {
t.Parallel()
svc := setupTestService(t)
out, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{
- VpcId: aws.String("vpc-test1"),
+ VpcId: aws.String("vpc-test1"),
+ ClientToken: aws.String("token-1"),
}, testAccountID)
require.NoError(t, err)
+ assert.Equal(t, "token-1", aws.StringValue(out.ClientToken), "AWS echoes the request's ClientToken")
rtb := out.RouteTable
assert.NotEmpty(t, *rtb.RouteTableId)
@@ -155,13 +157,42 @@ func TestCreateRouteTable_PersistsTagsForTagFilterDiscovery(t *testing.T) {
assert.Equal(t, "cp-private-rt", tags["spinifex:eks-role"])
}
+func TestDescribeRouteTables_FilterByTagKeyAndTagValue(t *testing.T) {
+ t.Parallel()
+ svc := setupTestService(t)
+ out, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{
+ VpcId: aws.String("vpc-test1"),
+ TagSpecifications: []*ec2.TagSpecification{{
+ ResourceType: aws.String(ec2.ResourceTypeRouteTable),
+ Tags: []*ec2.Tag{{Key: aws.String("Env"), Value: aws.String("prod")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestRtb(t, svc) // untagged
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Env"}, {"tag-value", "prod"}} {
+ desc, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: aws.StringSlice([]string{tc.value})}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.RouteTables, 1, tc.name)
+ assert.Equal(t, *out.RouteTable.RouteTableId, *desc.RouteTables[0].RouteTableId, tc.name)
+
+ desc, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: aws.StringSlice([]string{"zz-awsdiff-none"})}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.RouteTables, tc.name)
+ }
+}
+
func TestCreateRouteTable_VpcNotFound(t *testing.T) {
t.Parallel()
svc := setupTestService(t)
_, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{
VpcId: aws.String("vpc-nonexistent"),
}, testAccountID)
- assert.EqualError(t, err, awserrors.ErrorInvalidVpcIDNotFound)
+ requireAWSError(t, err, awserrors.ErrorInvalidVpcIDNotFound, "The vpc ID 'vpc-nonexistent' does not exist")
}
func TestDeleteRouteTable(t *testing.T) {
@@ -176,7 +207,7 @@ func TestDeleteRouteTable(t *testing.T) {
// Should be gone
_, err = svc.getRouteTable(t.Context(), testAccountID, rtbID)
- assert.EqualError(t, err, awserrors.ErrorInvalidRouteTableIDNotFound)
+ requireAWSError(t, err, awserrors.ErrorInvalidRouteTableIDNotFound, "The routeTable ID '"+rtbID+"' does not exist")
}
func TestDeleteRouteTable_Main(t *testing.T) {
@@ -188,7 +219,7 @@ func TestDeleteRouteTable_Main(t *testing.T) {
_, err = svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{
RouteTableId: aws.String(record.RouteTableId),
}, testAccountID)
- assert.EqualError(t, err, awserrors.ErrorDependencyViolation)
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The routeTable '"+record.RouteTableId+"' has dependencies and cannot be deleted.")
}
func TestDeleteRouteTable_WithAssociations(t *testing.T) {
@@ -207,7 +238,7 @@ func TestDeleteRouteTable_WithAssociations(t *testing.T) {
_, err = svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{
RouteTableId: aws.String(rtbID),
}, testAccountID)
- assert.EqualError(t, err, awserrors.ErrorDependencyViolation)
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The routeTable '"+rtbID+"' has dependencies and cannot be deleted.")
}
func TestDescribeRouteTables(t *testing.T) {
@@ -381,13 +412,47 @@ func TestCreateRoute_DuplicateDestination(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
- // Duplicate should fail
- _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
+ // Same destination and target: AWS returns success without a second route
+ out, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
RouteTableId: aws.String(rtbID),
DestinationCidrBlock: aws.String("0.0.0.0/0"),
GatewayId: aws.String("igw-test1"),
}, testAccountID)
- assert.EqualError(t, err, awserrors.ErrorRouteAlreadyExists)
+ require.NoError(t, err)
+ assert.True(t, aws.BoolValue(out.Return))
+
+ // Same destination, different target: still a conflict
+ _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String("0.0.0.0/0"),
+ NatGatewayId: aws.String("nat-test1"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorRouteAlreadyExists, "The route identified by 0.0.0.0/0 already exists.")
+
+ // AWS checks the target before the destination: "local" is no gateway
+ _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String("10.0.0.0/16"),
+ GatewayId: aws.String("local"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidGatewayIDNotFound, "The gateway ID 'local' does not exist")
+
+ // A gateway route to the VPC's CIDR, or inside it, is refused before the
+ // existing local route is considered
+ for _, dest := range []string{"10.0.0.0/16", "10.0.5.0/24"} {
+ _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String(dest),
+ GatewayId: aws.String("igw-test1"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+ "The destination CIDR block "+dest+" is equal to or more specific than one of this VPC's CIDR blocks. This route can target only an interface or an instance.")
+ }
+
+ record, err := svc.getRouteTable(t.Context(), testAccountID, rtbID)
+ require.NoError(t, err)
+ require.Len(t, record.Routes, 2)
+ assert.Equal(t, "igw-test1", record.Routes[1].GatewayId)
}
func TestDeleteRoute(t *testing.T) {
@@ -437,7 +502,7 @@ func TestDeleteRoute_NotFound(t *testing.T) {
RouteTableId: aws.String(rtbID),
DestinationCidrBlock: aws.String("192.168.0.0/16"),
}, testAccountID)
- assert.EqualError(t, err, awserrors.ErrorInvalidRouteNotFound)
+ requireAWSError(t, err, awserrors.ErrorInvalidRouteNotFound, "no route with destination-cidr-block 192.168.0.0/16 in route table "+rtbID)
}
func TestReplaceRoute(t *testing.T) {
@@ -461,6 +526,49 @@ func TestReplaceRoute(t *testing.T) {
require.NoError(t, err)
}
+func requireAWSError(t *testing.T, err error, code, message string) {
+ t.Helper()
+ got, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, code, got)
+ assert.Equal(t, message, msg)
+}
+
+func TestCreateRoute_MissingGateway(t *testing.T) {
+ t.Parallel()
+ svc := setupTestService(t)
+ rtbID := createTestRtb(t, svc)
+
+ _, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String("0.0.0.0/0"),
+ GatewayId: aws.String("igw-0000000000000dead"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidGatewayIDNotFound, "The gateway ID 'igw-0000000000000dead' does not exist")
+}
+
+func TestReplaceRoute_MissingRouteOrGateway(t *testing.T) {
+ t.Parallel()
+ svc := setupTestService(t)
+ rtbID := createTestRtb(t, svc)
+
+ _, err := svc.ReplaceRoute(t.Context(), &ec2.ReplaceRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String("198.51.100.0/24"),
+ GatewayId: aws.String("igw-test1"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+ "There is no route defined for '198.51.100.0/24' in the route table. Use CreateRoute instead.")
+
+ // AWS reports the missing gateway ahead of the missing route.
+ _, err = svc.ReplaceRoute(t.Context(), &ec2.ReplaceRouteInput{
+ RouteTableId: aws.String(rtbID),
+ DestinationCidrBlock: aws.String("198.51.100.0/24"),
+ GatewayId: aws.String("igw-0000000000000dead"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidGatewayIDNotFound, "The gateway ID 'igw-0000000000000dead' does not exist")
+}
+
func TestAssociateRouteTable(t *testing.T) {
t.Parallel()
svc := setupTestService(t)
@@ -1466,3 +1574,22 @@ func TestPublishGateDecisionsForVPC_EmptyVPCNoOp(t *testing.T) {
assertNoEvents(t, svc, gate, "expected no gate event")
assertNoEvents(t, svc, ungate, "expected no ungate event")
}
+
+func TestDescribeRouteTables_PagingValidation(t *testing.T) {
+ t.Parallel()
+ svc := setupTestService(t)
+
+ _, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{MaxResults: aws.Int64(101)}, testAccountID)
+ code, message, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+ assert.Equal(t, "Value ( 101 ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 100.", message)
+
+ _, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{
+ MaxResults: aws.Int64(5), RouteTableIds: []*string{aws.String("rtb-0123456789abcdef0")},
+ }, testAccountID)
+ code, message, ok = awserrors.ResolveErrorDetail(err)
+ require.True(t, ok)
+ assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+ assert.Equal(t, "The parameter RouteTableIds cannot be used with the parameter MaxResults", message)
+}
diff --git a/spinifex/handlers/ec2/spotinstance/service_impl.go b/spinifex/handlers/ec2/spotinstance/service_impl.go
index 1570cc5c6..6bc2ae698 100644
--- a/spinifex/handlers/ec2/spotinstance/service_impl.go
+++ b/spinifex/handlers/ec2/spotinstance/service_impl.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "slices"
"strings"
"time"
@@ -71,24 +72,12 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
// getOrCreateTerminalBucket opens the terminal bucket without changing its
// existing replica or placement configuration, creating it only when absent.
func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
- kv, err := js.KeyValue(ctx, KVBucketSpotRequestsTerminal)
- if err == nil {
- return kv, nil
- }
- if !errors.Is(err, jetstream.ErrBucketNotFound) {
- return nil, err
- }
-
- kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: KVBucketSpotRequestsTerminal,
+ return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+ Name: KVBucketSpotRequestsTerminal,
Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)",
History: 1,
TTL: spotTerminalTTL,
})
- if errors.Is(err, jetstream.ErrBucketExists) {
- return js.KeyValue(ctx, KVBucketSpotRequestsTerminal)
- }
- return kv, err
}
// PutSpotInstanceRequests stores each request in the active bucket.
@@ -372,12 +361,9 @@ func sirMatchesFilters(req *ec2.SpotInstanceRequest, filters map[string][]string
}
func sirMatchesTagKey(tags []*ec2.Tag, values []string) bool {
- for _, t := range tags {
- if t.Key != nil && awsfilters.MatchesAny(values, *t.Key) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(tags, func(t *ec2.Tag) bool {
+ return t.Key != nil && awsfilters.MatchesAny(values, *t.Key)
+ })
}
// launchSpec returns the request's launch specification, or an empty one when
diff --git a/spinifex/handlers/ec2/tags/key_pair_tags_test.go b/spinifex/handlers/ec2/tags/key_pair_tags_test.go
new file mode 100644
index 000000000..b2f6d05fd
--- /dev/null
+++ b/spinifex/handlers/ec2/tags/key_pair_tags_test.go
@@ -0,0 +1,86 @@
+package handlers_ec2_tags_test
+
+import (
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/config"
+ handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+ handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
+ "github.com/mulgadc/spinifex/spinifex/objectstore"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const (
+ testAccountID = "111111111111"
+ testED25519PubKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"
+)
+
+func newTagsService(t *testing.T) *handlers_ec2_tags.TagsServiceImpl {
+ t.Helper()
+ _, nc, _ := testutil.StartTestJetStream(t)
+ kv, err := handlers_ec2_tags.GetOrCreateTagsBucket(t.Context(), testutil.NewJetStream(t, nc))
+ require.NoError(t, err)
+ cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+ return handlers_ec2_tags.NewTagsServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), kv)
+}
+
+func keyPairTagSpec(tags map[string]string) []*ec2.TagSpecification {
+ spec := &ec2.TagSpecification{ResourceType: aws.String("key-pair")}
+ for k, v := range tags {
+ spec.Tags = append(spec.Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
+ }
+ return []*ec2.TagSpecification{spec}
+}
+
+func describeKeyPairTags(t *testing.T, svc *handlers_ec2_tags.TagsServiceImpl) map[string]map[string]string {
+ t.Helper()
+ out, err := svc.DescribeTags(t.Context(), &ec2.DescribeTagsInput{
+ Filters: []*ec2.Filter{{Name: aws.String("resource-type"), Values: []*string{aws.String("key-pair")}}},
+ }, testAccountID)
+ require.NoError(t, err)
+ got := map[string]map[string]string{}
+ for _, td := range out.Tags {
+ if got[*td.ResourceId] == nil {
+ got[*td.ResourceId] = map[string]string{}
+ }
+ got[*td.ResourceId][*td.Key] = *td.Value
+ }
+ return got
+}
+
+// AWS lists the TagSpecifications tags of a created or imported key pair in
+// DescribeTags under resource type key-pair, and drops them when it is deleted.
+func TestKeyPairCreationTags_VisibleToDescribeTags(t *testing.T) {
+ tagsSvc := newTagsService(t)
+ keySvc := handlers_ec2_key.NewKeyServiceImplWithStore(objectstore.NewMemoryObjectStore(), "test-bucket")
+ keySvc.SetCentralTagStore(tagsSvc)
+
+ created, err := keySvc.CreateKeyPair(t.Context(), &ec2.CreateKeyPairInput{
+ KeyName: aws.String("created"),
+ TagSpecifications: keyPairTagSpec(map[string]string{"Name": "created", "awsdiff": "1"}),
+ }, testAccountID)
+ require.NoError(t, err)
+
+ imported, err := keySvc.ImportKeyPair(t.Context(), &ec2.ImportKeyPairInput{
+ KeyName: aws.String("imported"),
+ PublicKeyMaterial: []byte(testED25519PubKey),
+ TagSpecifications: keyPairTagSpec(map[string]string{"Name": "imported"}),
+ }, testAccountID)
+ require.NoError(t, err)
+
+ assert.Equal(t, map[string]map[string]string{
+ *created.KeyPairId: {"Name": "created", "awsdiff": "1"},
+ *imported.KeyPairId: {"Name": "imported"},
+ }, describeKeyPairTags(t, tagsSvc))
+
+ _, err = keySvc.DeleteKeyPair(t.Context(), &ec2.DeleteKeyPairInput{KeyName: aws.String("created")}, testAccountID)
+ require.NoError(t, err)
+
+ assert.Equal(t, map[string]map[string]string{
+ *imported.KeyPairId: {"Name": "imported"},
+ }, describeKeyPairTags(t, tagsSvc))
+}
diff --git a/spinifex/handlers/ec2/tags/service_impl.go b/spinifex/handlers/ec2/tags/service_impl.go
index e11d8b820..a82ca4026 100644
--- a/spinifex/handlers/ec2/tags/service_impl.go
+++ b/spinifex/handlers/ec2/tags/service_impl.go
@@ -14,13 +14,14 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/s3"
+ "github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
- "github.com/mulgadc/spinifex/spinifex/config"
awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
- handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
- handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
+ handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+ handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/objectstore"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go/jetstream"
@@ -35,6 +36,9 @@ var _ handlers_ec2_instance.InstanceTagWriter = (*TagsServiceImpl)(nil)
// Ensure TagsServiceImpl can both project and clear vpc/subnet/sg/eni record tags.
var _ handlers_ec2_vpc.CentralTagStore = (*TagsServiceImpl)(nil)
+// Ensure TagsServiceImpl can both project and clear key-pair record tags.
+var _ handlers_ec2_key.CentralTagStore = (*TagsServiceImpl)(nil)
+
// TagsServiceImpl implements TagsService over a JetStream KV bucket, one entry
// per resource, scoped by account in the key.
//
@@ -176,13 +180,6 @@ func (s *TagsServiceImpl) getResourceTags(ctx context.Context, accountID, resour
return s.legacyTags(ctx, accountID, resourceID)
}
-// PutResourceTags overwrites the stored tag set for a resource. Used to
-// project an instance record's tags (the source of truth) into the central
-// store so describe-tags agrees with describe-instances.
-func (s *TagsServiceImpl) PutResourceTags(ctx context.Context, accountID, resourceID string, tags map[string]string) error {
- return s.putTags(ctx, accountID, resourceID, tags)
-}
-
// DeleteAllTags removes the stored tags for a resource, so describe-tags stops
// reporting a resource that is gone. Instance terminate uses it while the
// terminated record keeps its own tags until TTL; the vpc delete paths use it
@@ -285,6 +282,8 @@ var describeTagsValidFilters = map[string]bool{
"resource-type": true,
"key": true,
"value": true,
+ "tag-key": true,
+ "tag-value": true,
}
// DescribeTags returns tags matching the specified awsfilters.
@@ -333,6 +332,14 @@ func (s *TagsServiceImpl) DescribeTags(ctx context.Context, input *ec2.DescribeT
if !awsfilters.MatchesAny(filters["value"], value) {
continue
}
+ // On DescribeTags, AWS applies tag-key and tag-value to each tag row,
+ // exactly as it applies key and value.
+ if !awsfilters.MatchesAny(filters["tag-key"], key) || !awsfilters.MatchesAny(filters["tag-value"], value) {
+ continue
+ }
+ if !rowMatchesTagFilters(filters, key, value) {
+ continue
+ }
tags = append(tags, &ec2.TagDescription{
ResourceId: aws.String(resourceID),
@@ -350,6 +357,22 @@ func (s *TagsServiceImpl) DescribeTags(ctx context.Context, input *ec2.DescribeT
}, nil
}
+// rowMatchesTagFilters applies each tag: filter to one tag row, as
+// tag-key and tag-value are applied here: the row must be that key, with a
+// value the filter names.
+func rowMatchesTagFilters(filters map[string][]string, key, value string) bool {
+ for name, values := range filters {
+ tagKey, ok := strings.CutPrefix(name, "tag:")
+ if !ok {
+ continue
+ }
+ if key != tagKey || !awsfilters.MatchesAny(values, value) {
+ return false
+ }
+ }
+ return true
+}
+
// DeleteTags removes tags from the specified resources.
func (s *TagsServiceImpl) DeleteTags(ctx context.Context, input *ec2.DeleteTagsInput, accountID string) (*ec2.DeleteTagsOutput, error) {
if input == nil {
diff --git a/spinifex/handlers/ec2/tags/service_impl_test.go b/spinifex/handlers/ec2/tags/service_impl_test.go
index ec8346c93..ca4145181 100644
--- a/spinifex/handlers/ec2/tags/service_impl_test.go
+++ b/spinifex/handlers/ec2/tags/service_impl_test.go
@@ -287,6 +287,76 @@ func TestDescribeTags_FilterByValue(t *testing.T) {
assert.Equal(t, "production", *result.Tags[0].Value)
}
+// TestDescribeTags_FilterByTagKeyAndTagValue checks that tag-key and tag-value
+// select tag rows, as key and value do, rather than whole resources.
+func TestDescribeTags_FilterByTagKeyAndTagValue(t *testing.T) {
+ svc, _ := setupTestTagsService(t)
+
+ _, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+ Resources: []*string{aws.String("vpc-test1")},
+ Tags: []*ec2.Tag{
+ {Key: aws.String("Name"), Value: aws.String("web")},
+ {Key: aws.String("awsdiff"), Value: aws.String("yes")},
+ },
+ }, testAccountID)
+ require.NoError(t, err)
+
+ describe := func(name, value string) []*ec2.TagDescription {
+ t.Helper()
+ result, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(name), Values: []*string{aws.String(value)}}},
+ }, testAccountID)
+ require.NoError(t, err)
+ return result.Tags
+ }
+
+ byKey := describe("tag-key", "awsdiff")
+ require.Len(t, byKey, 1)
+ assert.Equal(t, "awsdiff", *byKey[0].Key)
+
+ byValue := describe("tag-value", "web")
+ require.Len(t, byValue, 1)
+ assert.Equal(t, "Name", *byValue[0].Key)
+
+ assert.Empty(t, describe("tag-key", "zz-awsdiff-none"))
+ assert.Empty(t, describe("tag-value", "zz-awsdiff-none"))
+}
+
+// TestDescribeTags_FilterByTagKeyName checks that a tag: filter selects
+// the matching tag row, and only rows of resources that carry it.
+func TestDescribeTags_FilterByTagKeyName(t *testing.T) {
+ svc, _ := setupTestTagsService(t)
+
+ for id, run := range map[string]string{"subnet-test1": "run-1", "subnet-test2": "run-2"} {
+ _, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+ Resources: []*string{aws.String(id)},
+ Tags: []*ec2.Tag{
+ {Key: aws.String("env"), Value: aws.String("prod")},
+ {Key: aws.String("awsdiff"), Value: aws.String(run)},
+ },
+ }, testAccountID)
+ require.NoError(t, err)
+ }
+
+ result, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{
+ Filters: []*ec2.Filter{
+ {Name: aws.String("resource-type"), Values: []*string{aws.String("subnet")}},
+ {Name: aws.String("tag:awsdiff"), Values: []*string{aws.String("run-1")}},
+ },
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, result.Tags, 1)
+ assert.Equal(t, "subnet-test1", aws.StringValue(result.Tags[0].ResourceId))
+ assert.Equal(t, "awsdiff", aws.StringValue(result.Tags[0].Key))
+ assert.Equal(t, "run-1", aws.StringValue(result.Tags[0].Value))
+
+ result, err = svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{
+ Filters: []*ec2.Filter{{Name: aws.String("tag:awsdiff"), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err)
+ assert.Empty(t, result.Tags)
+}
+
// TestDescribeTags_Empty tests listing tags when none exist.
func TestDescribeTags_Empty(t *testing.T) {
svc, _ := setupTestTagsService(t)
diff --git a/spinifex/handlers/ec2/tags/store.go b/spinifex/handlers/ec2/tags/store.go
index 2bc411088..e75ce4899 100644
--- a/spinifex/handlers/ec2/tags/store.go
+++ b/spinifex/handlers/ec2/tags/store.go
@@ -96,10 +96,10 @@ func (s *TagsServiceImpl) mutateTags(ctx context.Context, accountID, resourceID
return err
}
-// putTags replaces a resource's whole tag set. It still goes through the CAS
-// loop so a concurrent merge is overwritten as a whole rather than interleaved
-// with it, which is what a caller projecting an authoritative record wants.
-func (s *TagsServiceImpl) putTags(ctx context.Context, accountID, resourceID string, tags map[string]string) error {
+// PutResourceTags overwrites the stored tag set for a resource, projecting an
+// authoritative record (an instance's tags) into the central store. It goes
+// through the CAS loop so a concurrent merge is overwritten whole, not interleaved.
+func (s *TagsServiceImpl) PutResourceTags(ctx context.Context, accountID, resourceID string, tags map[string]string) error {
data, err := encodeTags(tags)
if err != nil {
return err
diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go
index 5b453fd15..ed06daacd 100644
--- a/spinifex/handlers/ec2/volume/service_impl.go
+++ b/spinifex/handlers/ec2/volume/service_impl.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "slices"
"strings"
"sync"
"time"
@@ -415,12 +416,9 @@ func volumeAttachmentMatchesAny(attachments []*ec2.VolumeAttachment, fieldFn fun
if len(attachments) == 0 {
return false
}
- for _, a := range attachments {
- if awsfilters.MatchesAny(values, fieldFn(a)) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(attachments, func(a *ec2.VolumeAttachment) bool {
+ return awsfilters.MatchesAny(values, fieldFn(a))
+ })
}
// DescribeVolumeStatus returns the status of one or more EBS volumes
diff --git a/spinifex/handlers/ec2/vpc/GetSecurityGroupsForVpc_test.go b/spinifex/handlers/ec2/vpc/GetSecurityGroupsForVpc_test.go
index c26154141..b55610ff7 100644
--- a/spinifex/handlers/ec2/vpc/GetSecurityGroupsForVpc_test.go
+++ b/spinifex/handlers/ec2/vpc/GetSecurityGroupsForVpc_test.go
@@ -114,7 +114,7 @@ func TestGetSecurityGroupsForVpc_ExcludesOtherAccount(t *testing.T) {
VpcId: otherVpc.Vpc.VpcId,
}, testAccountID)
require.Error(t, err)
- assert.Equal(t, awserrors.ErrorInvalidVpcIDNotFound, err.Error())
+ assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidVpcIDNotFound), err.Error())
}
func TestGetSecurityGroupsForVpc_MissingVpcId(t *testing.T) {
@@ -136,7 +136,7 @@ func TestGetSecurityGroupsForVpc_UnknownVpcIsNotFound(t *testing.T) {
}, testAccountID)
require.Error(t, err, "an unknown VPC must not be reported as a VPC with no groups")
assert.Nil(t, out)
- assert.Equal(t, awserrors.ErrorInvalidVpcIDNotFound, err.Error())
+ assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidVpcIDNotFound), err.Error())
}
// A VPC whose only group is the default one still answers, so the not-found
@@ -161,7 +161,7 @@ func TestGetSecurityGroupsForVpc_PagesEveryGroupOnce(t *testing.T) {
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
expected := describedGroupIDsForVpc(t, svc, vpcID)
- for _, name := range []string{"sg-a", "sg-b", "sg-c", "sg-d", "sg-e", "sg-f", "sg-g"} {
+ for _, name := range []string{"grp-a", "grp-b", "grp-c", "grp-d", "grp-e", "grp-f", "grp-g"} {
expected = append(expected, createTestSG(t, svc, vpcID, name))
}
diff --git a/spinifex/handlers/ec2/vpc/default_vpc.go b/spinifex/handlers/ec2/vpc/default_vpc.go
index 78c4b0cdd..b8a0e89bf 100644
--- a/spinifex/handlers/ec2/vpc/default_vpc.go
+++ b/spinifex/handlers/ec2/vpc/default_vpc.go
@@ -270,7 +270,6 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
AZ: s.localAZ(),
EnableDnsSupport: true, // AWS default
EnableDnsHostnames: true, // AWS default for default VPC
- Tags: map[string]string{"Name": "default"},
CreatedAt: time.Now(),
})
if err != nil {
@@ -292,7 +291,6 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
State: "available",
IsDefault: true,
MapPublicIpOnLaunch: !s.disableDefaultPublicIP, // AWS default subnets auto-assign public IPs (unless external mode has none)
- Tags: map[string]string{"Name": "default"},
CreatedAt: time.Now(),
})
if err != nil {
diff --git a/spinifex/handlers/ec2/vpc/default_vpc_test.go b/spinifex/handlers/ec2/vpc/default_vpc_test.go
index ebf70d29b..781e95d80 100644
--- a/spinifex/handlers/ec2/vpc/default_vpc_test.go
+++ b/spinifex/handlers/ec2/vpc/default_vpc_test.go
@@ -270,3 +270,22 @@ func TestEnsureDefaultVPC_UsesBootstrapIDs(t *testing.T) {
assert.Equal(t, ids.IgwId, info.InternetGatewayId)
}
}
+
+// AWS leaves the default VPC and its default subnet untagged.
+func TestEnsureDefaultVPC_LeavesVPCUntagged(t *testing.T) {
+ t.Parallel()
+ svcs, _, _ := newNodeServices(t, 1)
+
+ info, err := svcs[0].EnsureDefaultVPC("000000000001", handlers_ec2_vpc.BootstrapIDs{})
+ require.NoError(t, err)
+
+ out, err := svcs[0].DescribeVpcs(t.Context(), &ec2.DescribeVpcsInput{VpcIds: []*string{aws.String(info.VpcId)}}, "000000000001")
+ require.NoError(t, err)
+ require.Len(t, out.Vpcs, 1)
+ assert.Empty(t, out.Vpcs[0].Tags)
+
+ subnets, err := svcs[0].DescribeSubnets(t.Context(), &ec2.DescribeSubnetsInput{SubnetIds: []*string{aws.String(info.SubnetId)}}, "000000000001")
+ require.NoError(t, err)
+ require.Len(t, subnets.Subnets, 1)
+ assert.Empty(t, subnets.Subnets[0].Tags)
+}
diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go
index e2817d9b9..8c3ccf7ee 100644
--- a/spinifex/handlers/ec2/vpc/eni.go
+++ b/spinifex/handlers/ec2/vpc/eni.go
@@ -7,6 +7,7 @@ import (
"fmt"
"log/slog"
"net/netip"
+ "slices"
"strings"
"time"
@@ -17,6 +18,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/network/topology"
+ "github.com/mulgadc/spinifex/spinifex/paging"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go/jetstream"
)
@@ -149,7 +151,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
if err != nil {
switch {
case errors.Is(err, ErrIPInUse):
- return nil, errors.New(awserrors.ErrorInvalidIPAddressInUse)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidIPAddressInUse, "The specified address is already in use.")
case errors.Is(err, ErrIPOutOfRange):
return nil, eniAddressError(*input.PrivateIpAddress, subnet.CidrBlock)
default:
@@ -166,7 +168,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
}
// Generate a deterministic MAC address
- macAddr := generateENIMac(eniId)
+ macAddr := utils.HashMAC(eniId)
description := ""
if input.Description != nil {
@@ -216,8 +218,10 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
return nil, err
}
+ groupNames := make(map[string]string)
+ s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
return &ec2.CreateNetworkInterfaceOutput{
- NetworkInterface: s.eniRecordToEC2(&record, accountID),
+ NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames),
}, nil
}
@@ -255,7 +259,7 @@ func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, acco
if force {
return &ec2.DeleteNetworkInterfaceOutput{}, nil
}
- return nil, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return nil, eniNotFoundError(eniId)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -350,7 +354,7 @@ func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniI
if force {
return false, nil
}
- return false, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return false, eniNotFoundError(eniID)
}
return false, errors.New(awserrors.ErrorServerInternal)
}
@@ -405,18 +409,48 @@ func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniI
return false, fmt.Errorf("DetachAndDeleteENI: exhausted %d CAS attempts for %s: %w", maxAttempts, eniID, lastErr)
}
-// ModifyNetworkInterfaceAttribute modifies ENI attributes (security groups,
-// description). SourceDestCheck=true is accepted as a no-op.
+// ValidateModifyNetworkInterfaceAttributeAttributes checks the attribute fields
+// of a ModifyNetworkInterfaceAttribute request, which AWS does before it looks
+// the interface up. The gateway and the handler share it so their order agrees.
+func ValidateModifyNetworkInterfaceAttributeAttributes(input *ec2.ModifyNetworkInterfaceAttributeInput) error {
+ var set []string
+ if input.SourceDestCheck != nil {
+ set = append(set, "sourceDestCheck")
+ }
+ if input.Description != nil {
+ set = append(set, "description")
+ }
+ if len(input.Groups) > 0 {
+ set = append(set, "securityGroups")
+ }
+ if len(set) == 0 {
+ return errors.New(awserrors.ErrorInvalidParameterValue)
+ }
+ if len(set) > 1 {
+ return multipleAttributesError(set)
+ }
+ // Disabling source/dest check is unsupported: OVN port security enforces it.
+ if input.SourceDestCheck != nil && input.SourceDestCheck.Value != nil && !*input.SourceDestCheck.Value {
+ return errors.New(awserrors.ErrorUnsupported)
+ }
+ return nil
+}
+
+// multipleAttributesError is AWS's answer to a Modify*Attribute request that
+// sets more than one attribute. AWS's name order is not stable; this one is.
+func multipleAttributesError(names []string) error {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterCombination,
+ "Fields for multiple attribute types specified: %s", strings.Join(names, ", "))
+}
+
+// ModifyNetworkInterfaceAttribute modifies one ENI attribute (security groups
+// or description). SourceDestCheck=true is accepted as a no-op.
func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, input *ec2.ModifyNetworkInterfaceAttributeInput, accountID string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) {
if input.NetworkInterfaceId == nil || *input.NetworkInterfaceId == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
- if len(input.Groups) == 0 && input.Description == nil && input.SourceDestCheck == nil {
- return nil, errors.New(awserrors.ErrorInvalidParameterValue)
- }
- // Disabling source/dest check is unsupported: OVN port security enforces it.
- if input.SourceDestCheck != nil && input.SourceDestCheck.Value != nil && !*input.SourceDestCheck.Value {
- return nil, errors.New(awserrors.ErrorUnsupported)
+ if err := ValidateModifyNetworkInterfaceAttributeAttributes(input); err != nil {
+ return nil, err
}
eniId := *input.NetworkInterfaceId
@@ -424,7 +458,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
entry, err := s.eniKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return nil, eniNotFoundError(eniId)
}
var record ENIRecord
@@ -474,6 +508,13 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
return &ec2.ModifyNetworkInterfaceAttributeOutput{}, nil
}
+var describeNetworkInterfacesPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+ TooSmall: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter NetworkInterfaceIds cannot be used with the parameter MaxResults",
+}
+
var describeNetworkInterfacesValidFilters = map[string]bool{
"network-interface-id": true,
"subnet-id": true,
@@ -487,10 +528,17 @@ var describeNetworkInterfacesValidFilters = map[string]bool{
"attachment.attachment-id": true,
"attachment.instance-id": true,
"attachment.status": true,
+ "tag-key": true,
+ "tag-value": true,
}
// DescribeNetworkInterfaces lists ENIs with optional awsfilters.
func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *ec2.DescribeNetworkInterfacesInput, accountID string) (*ec2.DescribeNetworkInterfacesOutput, error) {
+ pageReq, err := describeNetworkInterfacesPaging.Parse(input.MaxResults, input.NextToken, len(input.NetworkInterfaceIds))
+ if err != nil {
+ return nil, err
+ }
+
parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeNetworkInterfacesValidFilters)
if err != nil {
slog.WarnContext(ctx, "DescribeNetworkInterfaces: invalid filter", "err", err)
@@ -498,10 +546,14 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
}
enis := make([]*ec2.NetworkInterface, 0)
+ groupNames := make(map[string]string)
eniIDs := make(map[string]bool)
for _, id := range input.NetworkInterfaceIds {
if id != nil {
+ if err := eniIDMalformedError(*id); err != nil {
+ return nil, err
+ }
eniIDs[*id] = true
}
}
@@ -539,7 +591,8 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
continue
}
- enis = append(enis, s.eniRecordToEC2(&record, accountID))
+ s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
+ enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames))
}
// If specific ENI IDs were requested but not found, return error
@@ -550,24 +603,27 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
found[*eni.NetworkInterfaceId] = true
}
}
- for id := range eniIDs {
- if !found[id] {
- return nil, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ for _, id := range input.NetworkInterfaceIds {
+ if id != nil && !found[*id] {
+ return nil, eniNotFoundError(*id)
}
}
}
+ enis, nextToken := paging.EC2Page(enis, func(e *ec2.NetworkInterface) string { return *e.NetworkInterfaceId }, pageReq)
+
slog.InfoContext(ctx, "DescribeNetworkInterfaces completed", "count", len(enis), "accountID", accountID)
return &ec2.DescribeNetworkInterfacesOutput{
NetworkInterfaces: enis,
+ NextToken: nextToken,
}, nil
}
// eniMatchesFilters checks whether an ENI record matches all parsed awsfilters.
func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
switch name {
@@ -596,14 +652,9 @@ func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool {
return false
}
case "group-id":
- found := false
- for _, sgId := range record.SecurityGroupIds {
- if awsfilters.MatchesAny(values, sgId) {
- found = true
- break
- }
- }
- if !found {
+ if !slices.ContainsFunc(record.SecurityGroupIds, func(sgId string) bool {
+ return awsfilters.MatchesAny(values, sgId)
+ }) {
return false
}
case "mac-address":
@@ -647,7 +698,7 @@ func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instan
key := utils.AccountKey(accountID, eniId)
entry, err := s.eniKV.Get(ctx, key)
if err != nil {
- return "", errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return "", eniNotFoundError(eniId)
}
var record ENIRecord
@@ -689,7 +740,7 @@ func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string)
key := utils.AccountKey(accountID, eniId)
entry, err := s.eniKV.Get(ctx, key)
if err != nil {
- return errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return eniNotFoundError(eniId)
}
var record ENIRecord
@@ -725,7 +776,7 @@ func (s *VPCServiceImpl) getENIRecord(ctx context.Context, accountID, eniId stri
key := utils.AccountKey(accountID, eniId)
entry, err := s.eniKV.Get(ctx, key)
if err != nil {
- return ENIRecord{}, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return ENIRecord{}, eniNotFoundError(eniId)
}
var record ENIRecord
if err := json.Unmarshal(entry.Value(), &record); err != nil {
@@ -745,7 +796,7 @@ func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string,
key := utils.AccountKey(accountID, eniId)
entry, err := s.eniKV.Get(ctx, key)
if err != nil {
- return errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ return eniNotFoundError(eniId)
}
var record ENIRecord
if err := json.Unmarshal(entry.Value(), &record); err != nil {
@@ -958,8 +1009,32 @@ func (s *VPCServiceImpl) updateENIPublicIP(ctx context.Context, accountID, eniId
return nil
}
-// eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface.
-func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec2.NetworkInterface {
+// fillSGNames records in names the name of each group in ids it does not
+// already hold. A group that cannot be read is recorded as "", so it is listed
+// without a name and not looked up again.
+func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids []string, names map[string]string) {
+ for _, id := range ids {
+ if _, ok := names[id]; ok {
+ continue
+ }
+ names[id] = ""
+ entry, err := s.sgKV.Get(ctx, utils.AccountKey(accountID, id))
+ if err != nil {
+ slog.WarnContext(ctx, "fillSGNames: SG read failed", "groupId", id, "err", err)
+ continue
+ }
+ var rec SecurityGroupRecord
+ if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+ slog.WarnContext(ctx, "fillSGNames: SG unmarshal failed", "groupId", id, "err", err)
+ continue
+ }
+ names[id] = rec.GroupName
+ }
+}
+
+// eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface, naming
+// each security group from groupNames.
+func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string) *ec2.NetworkInterface {
// ENIs with spinifex:managed-by tag are system-managed (e.g. by ELBv2)
requesterManaged := record.Tags["spinifex:managed-by"] != ""
@@ -988,13 +1063,22 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec
if len(record.SecurityGroupIds) > 0 {
groups := make([]*ec2.GroupIdentifier, 0, len(record.SecurityGroupIds))
for _, sgId := range record.SecurityGroupIds {
- groups = append(groups, &ec2.GroupIdentifier{
- GroupId: aws.String(sgId),
- })
+ group := &ec2.GroupIdentifier{GroupId: aws.String(sgId)}
+ if name := groupNames[sgId]; name != "" {
+ group.GroupName = aws.String(name)
+ }
+ groups = append(groups, group)
}
eni.Groups = groups
}
+ if s.privateDNSName != nil && record.PrivateIpAddress != "" {
+ if name := s.privateDNSName(record.PrivateIpAddress); name != "" {
+ eni.PrivateDnsName = aws.String(name)
+ eni.PrivateIpAddresses[0].PrivateDnsName = aws.String(name)
+ }
+ }
+
if record.PublicIpAddress != "" {
eni.Association = &ec2.NetworkInterfaceAssociation{
PublicIp: aws.String(record.PublicIpAddress),
@@ -1016,11 +1100,6 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec
return eni
}
-// generateENIMac creates a locally-administered unicast MAC address from an ENI ID.
-func generateENIMac(eniId string) string {
- return utils.HashMAC(eniId)
-}
-
// portEventPayload is the wire shape for vpc.create-port / vpc.delete-port.
// Mirrors network/subscribers.PortEvent — duplicated here to avoid a
// subscribers → handlers import cycle.
@@ -1111,7 +1190,7 @@ func (s *VPCServiceImpl) validateSGAttachment(ctx context.Context, accountID str
for _, sgId := range sgIds {
sgEntry, err := s.sgKV.Get(ctx, utils.AccountKey(accountID, sgId))
if err != nil {
- return errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return sgNotFoundError(sgId)
}
var sg SecurityGroupRecord
if err := json.Unmarshal(sgEntry.Value(), &sg); err != nil {
diff --git a/spinifex/handlers/ec2/vpc/eni_abandoned_test.go b/spinifex/handlers/ec2/vpc/eni_abandoned_test.go
index daa897b33..7e38535dd 100644
--- a/spinifex/handlers/ec2/vpc/eni_abandoned_test.go
+++ b/spinifex/handlers/ec2/vpc/eni_abandoned_test.go
@@ -137,8 +137,7 @@ func TestAbandonedENIBlocksSecurityGroupDelete(t *testing.T) {
GroupId: aws.String(sgID),
}, testAccountID)
require.Error(t, err)
- assert.ErrorContains(t, err, awserrors.ErrorDependencyViolation)
- assert.ErrorContains(t, err, eniID, "the refusal must name the ENI that blocked it")
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "resource "+sgID+" has a dependent object")
orphans, err := svc.ListAbandonedInstanceENIs(context.Background(), 15*time.Minute)
require.NoError(t, err)
diff --git a/spinifex/handlers/ec2/vpc/eni_mac_test.go b/spinifex/handlers/ec2/vpc/eni_mac_test.go
deleted file mode 100644
index 3b93615e8..000000000
--- a/spinifex/handlers/ec2/vpc/eni_mac_test.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package handlers_ec2_vpc
-
-import (
- "net"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestGenerateENIMac_Deterministic(t *testing.T) {
- t.Parallel()
- mac1 := generateENIMac("eni-abc123")
- mac2 := generateENIMac("eni-abc123")
- assert.Equal(t, mac1, mac2, "same input must produce same MAC")
-}
-
-func TestGenerateENIMac_DifferentInputs(t *testing.T) {
- t.Parallel()
- mac1 := generateENIMac("eni-aaa")
- mac2 := generateENIMac("eni-bbb")
- assert.NotEqual(t, mac1, mac2, "different inputs should produce different MACs")
-}
-
-func TestGenerateENIMac_LocallyAdministered(t *testing.T) {
- t.Parallel()
- mac := generateENIMac("eni-test123")
- hw, err := net.ParseMAC(mac)
- require.NoError(t, err)
- // IEEE 802 reserved bits on first octet: bit0=0 unicast, bit1=1 LAA.
- assert.Equal(t, byte(0x02), hw[0]&0x03)
-}
-
-func TestGenerateENIMac_EmptyString(t *testing.T) {
- t.Parallel()
- mac := generateENIMac("")
- hw, err := net.ParseMAC(mac)
- require.NoError(t, err)
- assert.Equal(t, byte(0x02), hw[0]&0x03)
- // New impl hashes the prefix-and-id; empty input no longer collapses to
- // the degenerate 02:00:00:00:00:00 of the old 24-bit impl.
- assert.NotEqual(t, "02:00:00:00:00:00", hw.String())
-}
diff --git a/spinifex/handlers/ec2/vpc/eni_test.go b/spinifex/handlers/ec2/vpc/eni_test.go
index edcd1e0af..788ef2147 100644
--- a/spinifex/handlers/ec2/vpc/eni_test.go
+++ b/spinifex/handlers/ec2/vpc/eni_test.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "net"
"sync"
"testing"
"time"
@@ -50,6 +49,46 @@ func TestCreateNetworkInterface(t *testing.T) {
assert.NotEmpty(t, *eni.MacAddress)
}
+func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+ subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+ sgId := createTestSG(t, svc, vpcId, "web")
+
+ // Without a namer an interface carries no private name.
+ unnamed := createTestENI(t, svc, subnetId)
+ desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+ NetworkInterfaceIds: []*string{aws.String(unnamed)},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, desc.NetworkInterfaces, 1)
+ assert.Nil(t, desc.NetworkInterfaces[0].PrivateDnsName)
+
+ svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
+ out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
+ SubnetId: aws.String(subnetId),
+ Groups: []*string{aws.String(sgId)},
+ }, testAccountID)
+ require.NoError(t, err)
+
+ desc, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+ NetworkInterfaceIds: []*string{out.NetworkInterface.NetworkInterfaceId},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, desc.NetworkInterfaces, 1)
+
+ for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc.NetworkInterfaces[0]} {
+ want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
+ assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName), name)
+ require.Len(t, eni.PrivateIpAddresses, 1, name)
+ assert.Equal(t, want, aws.StringValue(eni.PrivateIpAddresses[0].PrivateDnsName), name)
+ require.Len(t, eni.Groups, 1, name)
+ assert.Equal(t, sgId, aws.StringValue(eni.Groups[0].GroupId), name)
+ assert.Equal(t, "web", aws.StringValue(eni.Groups[0].GroupName), name)
+ }
+}
+
func TestCreateNetworkInterface_SequentialIPs(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -178,7 +217,7 @@ func TestCreateNetworkInterface_RequestedPrivateIPRaceHasOneWinner(t *testing.T)
switch {
case err == nil:
successes++
- case err.Error() == awserrors.ErrorInvalidIPAddressInUse:
+ case awserrors.IsErrorCode(err, awserrors.ErrorInvalidIPAddressInUse):
conflicts++
default:
t.Fatalf("unexpected error: %v", err)
@@ -413,9 +452,30 @@ func TestDescribeNetworkInterfaces_NotFound(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
_, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
- NetworkInterfaceIds: []*string{aws.String("eni-nonexistent")},
+ NetworkInterfaceIds: []*string{aws.String("eni-0000000000000dead")},
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidNetworkInterfaceID.NotFound")
+ requireAWSError(t, err, awserrors.ErrorInvalidNetworkInterfaceIDNotFound, "The networkInterface ID 'eni-0000000000000dead' does not exist")
+}
+
+func TestDescribeNetworkInterfaces_MalformedID(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ describe := func(ids ...string) error {
+ _, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+ NetworkInterfaceIds: aws.StringSlice(ids),
+ }, testAccountID)
+ return err
+ }
+
+ for _, id := range []string{"eni-xyz", "eni-", "eni-ABC"} {
+ requireAWSError(t, describe(id), awserrors.ErrorInvalidNetworkInterfaceIdMalformed, fmt.Sprintf("Invalid id: %q", id))
+ }
+ requireAWSError(t, describe("foo"), awserrors.ErrorInvalidNetworkInterfaceIdMalformed, `Invalid id: "foo" (expecting "eni-...")`)
+ // Unlike a group ID, any length of lowercase hex is merely unknown.
+ for _, id := range []string{"eni-12345", "eni-000000000000000000001"} {
+ requireAWSCode(t, describe(id), awserrors.ErrorInvalidNetworkInterfaceIDNotFound)
+ }
+ requireAWSCode(t, describe("eni-0aaaaaaa", "eni-xyz"), awserrors.ErrorInvalidNetworkInterfaceIdMalformed)
}
func TestAttachENI(t *testing.T) {
@@ -629,20 +689,6 @@ func TestDetachAndDeleteENI_AbsentForceFalse_ReturnsNotFound(t *testing.T) {
assert.ErrorContains(t, err, "InvalidNetworkInterfaceID.NotFound")
}
-func TestGenerateENIMac(t *testing.T) {
- t.Parallel()
- mac := generateENIMac("eni-test123")
- hw, err := net.ParseMAC(mac)
- require.NoError(t, err)
- assert.Equal(t, byte(0x02), hw[0]&0x03)
-
- // Same input produces same MAC
- assert.Equal(t, mac, generateENIMac("eni-test123"))
-
- // Different input produces different MAC
- assert.NotEqual(t, mac, generateENIMac("eni-test456"))
-}
-
// --- Filter tests ---
func TestDescribeNetworkInterfaces_FilterByVpcId(t *testing.T) {
@@ -830,8 +876,8 @@ func TestModifyNetworkInterfaceAttribute_SecurityGroups(t *testing.T) {
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
eniId := createTestENI(t, svc, subnetId)
- sg1 := createTestSG(t, svc, vpcId, "sg-one")
- sg2 := createTestSG(t, svc, vpcId, "sg-two")
+ sg1 := createTestSG(t, svc, vpcId, "grp-one")
+ sg2 := createTestSG(t, svc, vpcId, "grp-two")
_, err := svc.ModifyNetworkInterfaceAttribute(context.Background(), &ec2.ModifyNetworkInterfaceAttributeInput{
NetworkInterfaceId: aws.String(eniId),
@@ -855,8 +901,8 @@ func TestModifyNetworkInterfaceAttribute_PublishesUpdatePortSGs(t *testing.T) {
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
eniId := createTestENI(t, svc, subnetId)
- sg1 := createTestSG(t, svc, vpcId, "sg-mod-1")
- sg2 := createTestSG(t, svc, vpcId, "sg-mod-2")
+ sg1 := createTestSG(t, svc, vpcId, "grp-mod-1")
+ sg2 := createTestSG(t, svc, vpcId, "grp-mod-2")
eventCh := make(chan *nats.Msg, 1)
sub, err := nc.Subscribe("vpc.update-port-sgs", func(msg *nats.Msg) {
@@ -913,7 +959,7 @@ func TestCreateNetworkInterface_PublishesEventCarriesSGs(t *testing.T) {
svc, nc := setupTestVPCServiceWithNC(t)
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
- sgA := createTestSG(t, svc, vpcId, "sg-evt-A")
+ sgA := createTestSG(t, svc, vpcId, "grp-evt-A")
eventCh := make(chan *nats.Msg, 1)
sub, err := nc.Subscribe("vpc.create-port", func(msg *nats.Msg) {
@@ -1062,8 +1108,8 @@ func TestCreateNetworkInterface_WithSecurityGroups(t *testing.T) {
svc := setupTestVPCService(t)
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
- sgA := createTestSG(t, svc, vpcId, "sg-aaa")
- sgB := createTestSG(t, svc, vpcId, "sg-bbb")
+ sgA := createTestSG(t, svc, vpcId, "grp-aaa")
+ sgB := createTestSG(t, svc, vpcId, "grp-bbb")
out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetId),
@@ -1222,8 +1268,8 @@ func TestDescribeNetworkInterfaces_FilterByGroupId(t *testing.T) {
svc := setupTestVPCService(t)
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
- sgA := createTestSG(t, svc, vpcId, "sg-aaa")
- sgB := createTestSG(t, svc, vpcId, "sg-bbb")
+ sgA := createTestSG(t, svc, vpcId, "grp-aaa")
+ sgB := createTestSG(t, svc, vpcId, "grp-bbb")
// Create ENI with security groups
out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
@@ -1465,7 +1511,7 @@ func TestValidateSGAttachment_TooMany(t *testing.T) {
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
sgs := make([]string, 6)
for i := range sgs {
- sgs[i] = createTestSG(t, svc, vpcID, fmt.Sprintf("sg-%d", i))
+ sgs[i] = createTestSG(t, svc, vpcID, fmt.Sprintf("grp-%d", i))
}
err := svc.validateSGAttachment(t.Context(), testAccountID, sgs, vpcID)
@@ -1508,7 +1554,7 @@ func TestModifyNetworkInterfaceAttribute_VpcdError_Propagated(t *testing.T) {
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
eniId := createTestENI(t, svc, subnetId)
- sg1 := createTestSG(t, svc, vpcId, "sg-mod-fail-1")
+ sg1 := createTestSG(t, svc, vpcId, "grp-mod-fail-1")
// Swap the stub's vpc.update-port-sgs reply to an error in-place so
// there's exactly one responder (no race with a layered subscriber).
@@ -1552,3 +1598,22 @@ func TestUpdateENIPublicIP_NotFound(t *testing.T) {
require.Error(t, err)
assert.Contains(t, err.Error(), "eni-missing")
}
+
+// AWS checks the attribute combination before it looks the interface up, and
+// its order of names is unstable, so only the names are asserted.
+func TestModifyNetworkInterfaceAttribute_MultipleAttributes(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+
+ _, err := svc.ModifyNetworkInterfaceAttribute(context.Background(), &ec2.ModifyNetworkInterfaceAttributeInput{
+ NetworkInterfaceId: aws.String("eni-nonexistent"),
+ Groups: []*string{aws.String("sg-111")},
+ Description: &ec2.AttributeValue{Value: aws.String("desc")},
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+ assert.Contains(t, msg, "Fields for multiple attribute types specified: ")
+ assert.Contains(t, msg, "description")
+ assert.Contains(t, msg, "securityGroups")
+}
diff --git a/spinifex/handlers/ec2/vpc/ids.go b/spinifex/handlers/ec2/vpc/ids.go
new file mode 100644
index 000000000..5bb61db3d
--- /dev/null
+++ b/spinifex/handlers/ec2/vpc/ids.go
@@ -0,0 +1,68 @@
+package handlers_ec2_vpc
+
+import (
+ "strings"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// isLowerHex reports whether s is non-empty and holds only lowercase hex digits.
+func isLowerHex(s string) bool {
+ if s == "" {
+ return false
+ }
+ for _, c := range s {
+ if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
+ return false
+ }
+ }
+ return true
+}
+
+// malformedIDError is AWS's Invalid id answer for an ID that is not of the
+// form -, or whose suffix fails suffixOK.
+func malformedIDError(code, prefix, id string, suffixOK func(string) bool) error {
+ suffix, ok := strings.CutPrefix(id, prefix+"-")
+ if !ok {
+ return awserrors.Errorf(code, "Invalid id: %q (expecting %q)", id, prefix+"-...")
+ }
+ if !suffixOK(suffix) {
+ return awserrors.Errorf(code, "Invalid id: %q", id)
+ }
+ return nil
+}
+
+// sgIDMalformedError returns AWS's error for a group ID it calls malformed, or
+// nil. AWS answers NotFound for up to 8 hex digits or exactly 17, the length
+// of every Spinifex ID; other lengths are Malformed.
+func sgIDMalformedError(id string) error {
+ return malformedIDError(awserrors.ErrorInvalidGroupIdMalformed, "sg", id, func(s string) bool {
+ return isLowerHex(s) && (len(s) <= 8 || len(s) == 17)
+ })
+}
+
+// eniIDMalformedError returns AWS's error for an interface ID it calls
+// malformed, or nil. AWS answers NotFound for lowercase hex of any length.
+func eniIDMalformedError(id string) error {
+ return malformedIDError(awserrors.ErrorInvalidNetworkInterfaceIdMalformed, "eni", id, isLowerHex)
+}
+
+func vpcNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", id)
+}
+
+func eniNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidNetworkInterfaceIDNotFound, "networkInterface", id)
+}
+
+// sgNotFoundError is AWS's security group not-found answer, which unlike the
+// other resource types says no "ID".
+func sgNotFoundError(id string) error {
+ return awserrors.Errorf(awserrors.ErrorInvalidGroupNotFound, "The security group '%s' does not exist", id)
+}
+
+// subnetRangeError answers a subnet CIDR too large, too small or outside
+// its VPC, which AWS does not tell apart.
+func subnetRangeError(cidr string) error {
+ return awserrors.Errorf(awserrors.ErrorInvalidSubnetRange, "The CIDR '%s' is invalid.", cidr)
+}
diff --git a/spinifex/handlers/ec2/vpc/paging_test.go b/spinifex/handlers/ec2/vpc/paging_test.go
new file mode 100644
index 000000000..dd39505db
--- /dev/null
+++ b/spinifex/handlers/ec2/vpc/paging_test.go
@@ -0,0 +1,148 @@
+//test:in-package — reuses the in-package VPC service fixtures (setupTestVPCService, createTestVPC)
+
+package handlers_ec2_vpc
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDescribeSubnets_PagesThroughEverySubnetOnce(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ want := map[string]bool{}
+ for i := range 7 {
+ want[createTestSubnet(t, svc, vpcID, fmt.Sprintf("10.0.%d.0/24", i))] = true
+ }
+
+ got := map[string]bool{}
+ var token *string
+ pages := 0
+ for {
+ out, err := svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
+ MaxResults: aws.Int64(5),
+ NextToken: token,
+ }, testAccountID)
+ require.NoError(t, err)
+ require.LessOrEqual(t, len(out.Subnets), 5)
+ for _, s := range out.Subnets {
+ assert.False(t, got[*s.SubnetId], "subnet %s returned twice", *s.SubnetId)
+ got[*s.SubnetId] = true
+ }
+ pages++
+ if out.NextToken == nil {
+ break
+ }
+ token = out.NextToken
+ }
+ assert.Equal(t, want, got)
+ assert.Equal(t, 2, pages)
+}
+
+func TestDescribeSecurityGroups_PagesAfterFiltering(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ for i := range 6 {
+ createTestSG(t, svc, vpcID, fmt.Sprintf("paged-%d", i))
+ }
+ createTestSG(t, svc, createTestVPC(t, svc, "10.1.0.0/16"), "other-vpc")
+
+ filter := []*ec2.Filter{{Name: aws.String("group-name"), Values: []*string{aws.String("paged-*")}}}
+ first, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ Filters: filter, MaxResults: aws.Int64(5),
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, first.SecurityGroups, 5)
+ require.NotNil(t, first.NextToken)
+
+ second, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ Filters: filter, MaxResults: aws.Int64(5), NextToken: first.NextToken,
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, second.SecurityGroups, 1)
+ assert.Nil(t, second.NextToken)
+}
+
+func TestDescribeVpcs_ValidatesPagingButReturnsEverything(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ for i := range 6 {
+ createTestVPC(t, svc, fmt.Sprintf("10.%d.0.0/16", i))
+ }
+
+ out, err := svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{MaxResults: aws.Int64(5)}, testAccountID)
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, len(out.Vpcs), 6)
+ assert.Nil(t, out.NextToken)
+}
+
+// TestNetworkDescribes_PagingErrors pins each operation to the code and message
+// AWS returns, which differ in wording and casing between operations.
+func TestNetworkDescribes_PagingErrors(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ ctx := context.Background()
+ ids := []*string{aws.String("x-0123456789abcdef0")}
+
+ cases := []struct {
+ name string
+ call func() error
+ code string
+ message string
+ }{
+ {"DescribeVpcs too large", func() error {
+ _, err := svc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{MaxResults: aws.Int64(2000)}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterValue, "Value ( 2000 ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000."},
+ {"DescribeVpcs ids", func() error {
+ _, err := svc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{MaxResults: aws.Int64(5), VpcIds: ids}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterCombination, "The parameter VpcIds cannot be used with the parameter MaxResults"},
+ {"DescribeSubnets too small", func() error {
+ _, err := svc.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{MaxResults: aws.Int64(4)}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterValue, "Value ( 4 ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5."},
+ {"DescribeSubnets bad token", func() error {
+ _, err := svc.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{NextToken: aws.String("garbage")}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterValue, "Value ( garbage ) for parameter NextToken is invalid. The token is invalid."},
+ {"DescribeSecurityGroups too large", func() error {
+ _, err := svc.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{MaxResults: aws.Int64(1001)}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterValue, "Value ( 1001 ) for parameter maxResults is invalid. Expecting a value smaller than 1000."},
+ {"DescribeSecurityGroups ids", func() error {
+ _, err := svc.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{MaxResults: aws.Int64(5), GroupIds: ids}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterCombination, "The parameter securityGroupIdSet cannot be used with the parameter maxResults"},
+ {"DescribeSecurityGroupRules bad token", func() error {
+ _, err := svc.DescribeSecurityGroupRules(ctx, &ec2.DescribeSecurityGroupRulesInput{NextToken: aws.String("garbage")}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidPaginationToken, "Next token 'garbage' is invalid"},
+ {"DescribeSecurityGroupRules ids", func() error {
+ _, err := svc.DescribeSecurityGroupRules(ctx, &ec2.DescribeSecurityGroupRulesInput{MaxResults: aws.Int64(5), SecurityGroupRuleIds: ids}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterCombination, "The parameter 'securityGroupRuleIds' may not be used in combination with 'maxResults'."},
+ {"DescribeNetworkInterfaces ids", func() error {
+ _, err := svc.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{MaxResults: aws.Int64(5), NetworkInterfaceIds: ids}, testAccountID)
+ return err
+ }, awserrors.ErrorInvalidParameterCombination, "The parameter NetworkInterfaceIds cannot be used with the parameter MaxResults"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ code, message, ok := awserrors.ResolveErrorDetail(tc.call())
+ require.True(t, ok)
+ assert.Equal(t, tc.code, code)
+ assert.Equal(t, tc.message, message)
+ })
+ }
+}
diff --git a/spinifex/handlers/ec2/vpc/security_group.go b/spinifex/handlers/ec2/vpc/security_group.go
index c23147e3d..4fffb7d1c 100644
--- a/spinifex/handlers/ec2/vpc/security_group.go
+++ b/spinifex/handlers/ec2/vpc/security_group.go
@@ -19,6 +19,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ "github.com/mulgadc/spinifex/spinifex/paging"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go/jetstream"
)
@@ -60,6 +61,16 @@ func normalizeIPProtocol(proto string) (string, error) {
return "", fmt.Errorf("invalid IpProtocol %q: supported values are tcp, udp, icmp, -1 (or 6, 17, 1)", proto)
}
+// validateSGRulePorts rejects a tcp or udp rule whose FromPort exceeds its ToPort.
+// Only create paths call it, so a rule stored before the check can still be
+// revoked or re-described by its ports.
+func validateSGRulePorts(proto string, fromPort, toPort int64) error {
+ if (proto == "tcp" || proto == "udp") && fromPort > toPort {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Invalid TCP/UDP port range(%d:%d)", fromPort, toPort)
+ }
+ return nil
+}
+
// validateSGRule rejects CidrIp values that are non-canonical or IPv6, CidrIpv6
// values that are non-canonical or IPv4, IpProtocol values the ACL builder
// cannot express, and SourceSG values not matching the sg-ID format. At least
@@ -75,7 +86,7 @@ func validateSGRule(r SGRule) error {
if r.CidrIp != "" {
_, ipnet, err := net.ParseCIDR(r.CidrIp)
if err != nil {
- return fmt.Errorf("invalid CidrIp %q: %w", r.CidrIp, err)
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "CIDR block %s is malformed", r.CidrIp)
}
if ipnet.IP.To4() == nil {
return fmt.Errorf("invalid CidrIp %q: IPv6 belongs in CidrIpv6", r.CidrIp)
@@ -87,7 +98,7 @@ func validateSGRule(r SGRule) error {
if r.CidrIpv6 != "" {
_, ipnet, err := net.ParseCIDR(r.CidrIpv6)
if err != nil {
- return fmt.Errorf("invalid CidrIpv6 %q: %w", r.CidrIpv6, err)
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "CIDR block %s is malformed", r.CidrIpv6)
}
if ipnet.IP.To4() != nil {
return fmt.Errorf("invalid CidrIpv6 %q: IPv4 belongs in CidrIp", r.CidrIpv6)
@@ -174,15 +185,18 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
vpcId := *input.VpcId
groupName := *input.GroupName
- // "default" is reserved for the per-VPC default SG that CreateVpc
- // provisions internally. Matches AWS behavior.
- if groupName == defaultSecurityGroupName {
- return nil, errors.New(awserrors.ErrorInvalidGroupReserved)
- }
-
if err := s.requireVPCExists(ctx, accountID, vpcId); err != nil {
return nil, err
}
+ // "default" is reserved for the per-VPC default SG that CreateVpc
+ // provisions internally. AWS checks the VPC first.
+ if groupName == defaultSecurityGroupName {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Cannot use reserved security group name: %s", groupName)
+ }
+ if strings.HasPrefix(groupName, "sg-") {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "Value (%s) for parameter GroupName is invalid. Group names may not be in the format sg-*.", groupName)
+ }
// Check for duplicate group name in the same VPC and enforce the per-VPC
// SG quota in the same bucket walk.
@@ -219,7 +233,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
}
sgsInVPC++
if existing.GroupName == groupName {
- return nil, errors.New(awserrors.ErrorInvalidGroupDuplicate)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidGroupDuplicate, "The security group '%s' already exists for VPC '%s'", groupName, vpcId)
}
}
if sgsInVPC >= maxSGsPerVPC {
@@ -273,6 +287,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
return &ec2.CreateSecurityGroupOutput{
GroupId: aws.String(groupId),
+ Tags: utils.MapToEC2Tags(record.Tags),
}, nil
}
@@ -291,7 +306,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del
// success. Destroy orchestration tolerates it via awserrors.IsNotFound;
// a transient read error stays a server error.
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -302,7 +317,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del
}
if record.IsDefault {
- return nil, errors.New(awserrors.ErrorCannotDelete)
+ return nil, awserrors.Errorf(awserrors.ErrorCannotDelete, "the specified group: %q name: %q cannot be deleted by a user", groupId, record.GroupName)
}
if err := s.checkSGDependencies(ctx, accountID, groupId); err != nil {
@@ -337,14 +352,14 @@ func (s *VPCServiceImpl) validateSGRuleReferences(ctx context.Context, accountID
}
entry, err := s.sgKV.Get(ctx, utils.AccountKey(accountID, r.SourceSG))
if err != nil {
- return errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return sgNotFoundError(r.SourceSG)
}
var rec SecurityGroupRecord
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
return errors.New(awserrors.ErrorServerInternal)
}
if rec.VpcId != ownerVpcId {
- return errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return sgNotFoundError(r.SourceSG)
}
}
return nil
@@ -381,8 +396,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
// afterwards otherwise means cross-referencing the whole ENI table.
slog.WarnContext(ctx, "checkSGDependencies: SG still attached to ENI",
"groupId", groupId, "eniId", eni.NetworkInterfaceId, "accountID", accountID)
- return awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the security group has a dependent network interface %s that must be detached first", eni.NetworkInterfaceId)
+ return sgDependentObjectError(groupId)
}
}
@@ -421,16 +435,49 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
return nil
}
-// sgReferencedByError logs and returns the DependencyViolation naming the SG
-// whose rule still references groupId, so the blocking id survives even when
-// the message does not reach the client.
+// sgReferencedByError logs the SG whose rule still references groupId, since
+// AWS's message does not name it, and returns the DependencyViolation.
func sgReferencedByError(ctx context.Context, accountID, groupId, referencingGroupId, direction string) error {
slog.WarnContext(ctx, "checkSGDependencies: SG referenced by another SG's rule",
"groupId", groupId, "referencingGroupId", referencingGroupId,
"direction", direction, "accountID", accountID)
- return awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the security group is referenced by a %s rule on security group %s that must be revoked first",
- direction, referencingGroupId)
+ return sgDependentObjectError(groupId)
+}
+
+// sgDependentObjectError is AWS's answer to deleting a group that an
+// interface or another group's rule still uses; it does not say which.
+func sgDependentObjectError(groupId string) error {
+ return awserrors.Errorf(awserrors.ErrorDependencyViolation, "resource %s has a dependent object", groupId)
+}
+
+// sgRuleDuplicateError is AWS's answer to adding a rule the group already
+// has, which describes the rule by its peer, protocol and ports.
+func sgRuleDuplicateError(rule SGRule) error {
+ peer := rule.CidrIp
+ if peer == "" {
+ peer = rule.CidrIpv6
+ }
+ if peer == "" {
+ peer = rule.SourceSG
+ }
+ var desc string
+ switch rule.IpProtocol {
+ case allProtocols:
+ desc = fmt.Sprintf("peer: %s, ALL, ALLOW", peer)
+ case "icmp":
+ desc = fmt.Sprintf("peer: %s, ICMP, type: %s, code: %s, ALLOW", peer, icmpField(rule.FromPort), icmpField(rule.ToPort))
+ default:
+ desc = fmt.Sprintf("peer: %s, %s, from port: %d, to port: %d, ALLOW", peer, strings.ToUpper(rule.IpProtocol), rule.FromPort, rule.ToPort)
+ }
+ return awserrors.Errorf(awserrors.ErrorInvalidPermissionDuplicate, "the specified rule %q already exists", desc)
+}
+
+// icmpField renders an ICMP type or code as AWS does, with -1 as ALL.
+func icmpField(v int64) string {
+ if v == -1 {
+ return "ALL"
+ }
+ return strconv.FormatInt(v, 10)
}
// describeSecurityGroupsValidFilters defines the set of filter names accepted by DescribeSecurityGroups.
@@ -440,12 +487,27 @@ var describeSecurityGroupsValidFilters = map[string]bool{
"vpc-id": true,
"description": true,
"ip-permission.cidr": true,
+ "tag-key": true,
+ "tag-value": true,
+}
+
+// AWS accepts 5 and 1000 despite the "smaller than" and "greater than" wording.
+var describeSecurityGroupsPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter maxResults is invalid. Expecting a value smaller than 1000.",
+ TooSmall: "Value ( %d ) for parameter maxResults is invalid. Expecting a value greater than 5.",
+ WithIDs: "The parameter securityGroupIdSet cannot be used with the parameter maxResults",
}
// DescribeSecurityGroups lists security groups with optional awsfilters.
func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error) {
groups := []*ec2.SecurityGroup{}
+ pageReq, err := describeSecurityGroupsPaging.Parse(input.MaxResults, input.NextToken, len(input.GroupIds))
+ if err != nil {
+ return nil, err
+ }
+
groupIDs := make(map[string]bool)
for _, id := range input.GroupIds {
if id != nil {
@@ -458,6 +520,14 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
slog.WarnContext(ctx, "DescribeSecurityGroups: invalid filter", "err", err)
return nil, err
}
+ for _, id := range input.GroupIds {
+ if id == nil {
+ continue
+ }
+ if err := sgIDMalformedError(*id); err != nil {
+ return nil, err
+ }
+ }
prefix := accountID + "."
keys, err := s.sgKV.Keys(ctx)
@@ -504,24 +574,27 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
found[*sg.GroupId] = true
}
}
- for id := range groupIDs {
- if !found[id] {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ for _, id := range input.GroupIds {
+ if id != nil && !found[*id] {
+ return nil, sgNotFoundError(*id)
}
}
}
+ groups, nextToken := paging.EC2Page(groups, func(g *ec2.SecurityGroup) string { return *g.GroupId }, pageReq)
+
slog.InfoContext(ctx, "DescribeSecurityGroups completed", "count", len(groups), "accountID", accountID)
return &ec2.DescribeSecurityGroupsOutput{
SecurityGroups: groups,
+ NextToken: nextToken,
}, nil
}
// sgMatchesFilters checks whether a SecurityGroupRecord satisfies all parsed awsfilters.
func sgMatchesFilters(record *SecurityGroupRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
@@ -556,12 +629,9 @@ func sgMatchesFilters(record *SecurityGroupRecord, filters map[string][]string)
// sgIngressCIDRMatchesAny checks if any ingress rule's CIDR matches any of the filter values.
func sgIngressCIDRMatchesAny(rules []SGRule, values []string) bool {
- for _, rule := range rules {
- if rule.CidrIp != "" && awsfilters.MatchesAny(values, rule.CidrIp) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(rules, func(rule SGRule) bool {
+ return rule.CidrIp != "" && awsfilters.MatchesAny(values, rule.CidrIp)
+ })
}
// getSecurityGroupsForVpcValidFilters defines the set of filter names accepted
@@ -733,27 +803,37 @@ var describeSecurityGroupRulesValidFilters = map[string]bool{
"group-id": true,
"security-group-rule-id": true,
"tag-key": true,
+ "tag-value": true,
+}
+
+var describeSecurityGroupRulesPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter maxResults is invalid. Expecting a value less than or equal to 1000.",
+ TooSmall: "Value ( %d ) for parameter maxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter 'securityGroupRuleIds' may not be used in combination with 'maxResults'.",
+ PaginationTokenError: true,
}
// DescribeSecurityGroupRules returns a flat list of SecurityGroupRule objects
// for the caller's account, optionally narrowed by SecurityGroupRuleIds or
-// awsfilters. MaxResults and NextToken are accepted but ignored.
+// filters, one page at a time.
func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *ec2.DescribeSecurityGroupRulesInput, accountID string) (*ec2.DescribeSecurityGroupRulesOutput, error) {
- requested := make(map[string]bool)
- if input != nil {
- for _, id := range input.SecurityGroupRuleIds {
- if id == nil || *id == "" {
- return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
- }
- requested[*id] = true
- }
+ if input == nil {
+ input = &ec2.DescribeSecurityGroupRulesInput{}
+ }
+ pageReq, err := describeSecurityGroupRulesPaging.Parse(input.MaxResults, input.NextToken, len(input.SecurityGroupRuleIds))
+ if err != nil {
+ return nil, err
}
- var filters []*ec2.Filter
- if input != nil {
- filters = input.Filters
+ requested := make(map[string]bool)
+ for _, id := range input.SecurityGroupRuleIds {
+ if id == nil || *id == "" {
+ return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed)
+ }
+ requested[*id] = true
}
- parsedFilters, err := awsfilters.ParseFilters(filters, describeSecurityGroupRulesValidFilters)
+ parsedFilters, err := awsfilters.ParseFilters(input.Filters, describeSecurityGroupRulesValidFilters)
if err != nil {
slog.WarnContext(ctx, "DescribeSecurityGroupRules: invalid filter", "err", err)
return nil, err
@@ -805,11 +885,18 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
}
}
- if len(requested) > 0 {
- for id := range requested {
- if !emitted[id] {
- return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdNotFound)
- }
+ for _, id := range input.SecurityGroupRuleIds {
+ if !emitted[*id] {
+ return nil, sgRuleNotFoundError(*id)
+ }
+ }
+
+ rules, nextToken := paging.EC2Page(rules, func(r *ec2.SecurityGroupRule) string { return *r.SecurityGroupRuleId }, pageReq)
+ // AWS lists an untagged rule with empty Tags here, but omits them from
+ // the Authorize* responses that share the rule shape.
+ for _, r := range rules {
+ if r.Tags == nil {
+ r.Tags = []*ec2.Tag{}
}
}
@@ -817,6 +904,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
return &ec2.DescribeSecurityGroupRulesOutput{
SecurityGroupRules: rules,
+ NextToken: nextToken,
}, nil
}
@@ -824,7 +912,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
// single rule.
func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
switch name {
@@ -836,10 +924,6 @@ func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[
if !awsfilters.MatchesAny(values, rule.RuleId) {
return false
}
- case "tag-key":
- if !sgRuleMatchesTagKey(rule.Tags, values) {
- return false
- }
default:
// Unreachable: ParseFilters rejects names not in the valid set.
// Logs an error if a future map entry lacks a matching case.
@@ -918,17 +1002,6 @@ func findSGRuleByID(rules []SGRule, ruleID string) *SGRule {
return nil
}
-// sgRuleMatchesTagKey reports whether the rule carries any of the named tag
-// keys, whatever their values.
-func sgRuleMatchesTagKey(tags map[string]string, keys []string) bool {
- for key := range tags {
- if awsfilters.MatchesAny(keys, key) {
- return true
- }
- }
- return false
-}
-
// applySGRuleTags stamps the security-group-rule TagSpecification onto every
// rule an authorize call creates. AWS tags the rules the call creates and no
// others, so this runs on the new rules alone.
@@ -944,8 +1017,8 @@ func applySGRuleTags(rules []SGRule, specs []*ec2.TagSpecification) []SGRule {
}
// sgRuleToSecurityGroupRule flattens a stored SGRule into the AWS API shape.
-// accountID supplies GroupOwnerId and ReferencedGroupInfo.UserId; VpcId is
-// derived from the parent record (same-VPC references are enforced on write).
+// accountID supplies GroupOwnerId and ReferencedGroupInfo.UserId. AWS names
+// no VpcId for a same-VPC reference, the only kind Spinifex allows.
func sgRuleToSecurityGroupRule(record *SecurityGroupRecord, rule SGRule, isEgress bool, accountID string) *ec2.SecurityGroupRule {
// An all-protocol rule has no ports to report. AWS answers -1 for both;
// reporting the stored zeroes offers a caller two ports it never sent.
@@ -976,7 +1049,6 @@ func sgRuleToSecurityGroupRule(record *SecurityGroupRecord, rule SGRule, isEgres
out.ReferencedGroupInfo = &ec2.ReferencedSecurityGroup{
GroupId: aws.String(rule.SourceSG),
UserId: aws.String(accountID),
- VpcId: aws.String(record.VpcId),
}
}
return out
@@ -993,7 +1065,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
var record SecurityGroupRecord
@@ -1008,6 +1080,9 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
newRules, err := ipPermissionsToSGRules(input.IpPermissions, sgParseAuthorize)
if err != nil {
slog.WarnContext(ctx, "AuthorizeSecurityGroupIngress: invalid rule", "groupId", groupId, "err", err)
+ if _, ok := awserrors.ResolveErrorCode(err); ok {
+ return nil, err
+ }
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "%s", err)
}
newRules = applySGRuleTags(newRules, input.TagSpecifications)
@@ -1020,7 +1095,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
}
for _, nr := range newRules {
if _, ok := existing[sgRuleKey(nr)]; ok {
- return nil, errors.New(awserrors.ErrorInvalidPermissionDuplicate)
+ return nil, sgRuleDuplicateError(nr)
}
}
if len(record.IngressRules)+len(newRules) > maxRulesPerSGSide {
@@ -1073,7 +1148,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
var record SecurityGroupRecord
@@ -1088,6 +1163,9 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
newRules, err := ipPermissionsToSGRules(input.IpPermissions, sgParseAuthorize)
if err != nil {
slog.WarnContext(ctx, "AuthorizeSecurityGroupEgress: invalid rule", "groupId", groupId, "err", err)
+ if _, ok := awserrors.ResolveErrorCode(err); ok {
+ return nil, err
+ }
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "%s", err)
}
newRules = applySGRuleTags(newRules, input.TagSpecifications)
@@ -1100,7 +1178,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
}
for _, nr := range newRules {
if _, ok := existing[sgRuleKey(nr)]; ok {
- return nil, errors.New(awserrors.ErrorInvalidPermissionDuplicate)
+ return nil, sgRuleDuplicateError(nr)
}
}
if len(record.EgressRules)+len(newRules) > maxRulesPerSGSide {
@@ -1153,7 +1231,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input *
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
var record SecurityGroupRecord
@@ -1220,7 +1298,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
var record SecurityGroupRecord
@@ -1341,7 +1419,7 @@ func (s *VPCServiceImpl) updateSGRuleDescriptions(ctx context.Context, accountID
key := utils.AccountKey(accountID, req.groupId)
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return sgNotFoundError(req.groupId)
}
var record SecurityGroupRecord
@@ -1417,7 +1495,7 @@ func applySGRuleDescriptions(rules []SGRule, descriptions []*ec2.SecurityGroupRu
// absent from this record and so is not-found, never a silent no-op.
i, ok := byID[id]
if !ok {
- return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdNotFound)
+ return nil, sgRuleNotFoundError(id)
}
if d.Description != nil {
out[i].Description = *d.Description
@@ -1534,7 +1612,7 @@ func (s *VPCServiceImpl) ModifySecurityGroupRules(ctx context.Context, input *ec
entry, err := s.sgKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidGroupNotFound)
+ return nil, sgNotFoundError(groupId)
}
var record SecurityGroupRecord
@@ -1677,7 +1755,7 @@ func checkSGRuleModificationDuplicates(record *SecurityGroupRecord, mods []sgRul
for i, r := range rules {
if !replaced[i] && newKeys[sgRuleKey(r)] {
- return errors.New(awserrors.ErrorInvalidPermissionDuplicate)
+ return sgRuleDuplicateError(r)
}
}
return nil
@@ -1708,6 +1786,9 @@ func sgRuleRequestToSGRule(req *ec2.SecurityGroupRuleRequest) (SGRule, error) {
"Invalid value for portRange. Must specify both from and to ports with TCP/UDP.")
}
r.FromPort, r.ToPort = *req.FromPort, *req.ToPort
+ if err := validateSGRulePorts(proto, r.FromPort, r.ToPort); err != nil {
+ return SGRule{}, err
+ }
case allProtocols:
// Stored as 0/0, the form authorize stores for the no-ports case, so the
// rule's sgRuleKey matches Terraform's and the default egress rule's.
@@ -1782,8 +1863,11 @@ func sgRuleIDLookupError(id string) error {
if SGRuleIDIsMalformed(id) {
return awserrors.Errorf(awserrors.ErrorInvalidSecurityGroupRuleIdMalformed, "Invalid id: %q", id)
}
- return awserrors.Errorf(awserrors.ErrorInvalidSecurityGroupRuleIdNotFound,
- "The security group rule ID '%s' does not exist", id)
+ return sgRuleNotFoundError(id)
+}
+
+func sgRuleNotFoundError(id string) error {
+ return awserrors.IDNotFound(awserrors.ErrorInvalidSecurityGroupRuleIdNotFound, "security group rule", id)
}
// sgRecordToEC2 converts a SecurityGroupRecord to an EC2 SecurityGroup.
@@ -1794,8 +1878,8 @@ func (s *VPCServiceImpl) sgRecordToEC2(record *SecurityGroupRecord, accountID st
Description: aws.String(record.Description),
VpcId: aws.String(record.VpcId),
OwnerId: aws.String(accountID),
- IpPermissions: sgRulesToIpPermissions(record.IngressRules),
- IpPermissionsEgress: sgRulesToIpPermissions(record.EgressRules),
+ IpPermissions: sgRulesToIpPermissions(record.IngressRules, accountID),
+ IpPermissionsEgress: sgRulesToIpPermissions(record.EgressRules, accountID),
}
sg.Tags = utils.MapToEC2Tags(record.Tags)
@@ -1838,6 +1922,11 @@ func ipPermissionsToSGRules(perms []*ec2.IpPermission, mode sgParseMode) ([]SGRu
if perm.ToPort != nil {
toPort = *perm.ToPort
}
+ if mode == sgParseAuthorize {
+ if err := validateSGRulePorts(proto, fromPort, toPort); err != nil {
+ return nil, err
+ }
+ }
appended := false
for _, ipRange := range perm.IpRanges {
@@ -1902,7 +1991,8 @@ func ipPermissionsToSGRules(perms []*ec2.IpPermission, mode sgParseMode) ([]SGRu
}
// sgRulesToIpPermissions converts SGRule slice to AWS IpPermission slice.
-func sgRulesToIpPermissions(rules []SGRule) []*ec2.IpPermission {
+// accountID owns every referenced group, since references stay in one VPC.
+func sgRulesToIpPermissions(rules []SGRule, accountID string) []*ec2.IpPermission {
// Group rules by protocol+port range
type permKey struct {
IpProtocol string
@@ -1915,10 +2005,12 @@ func sgRulesToIpPermissions(rules []SGRule) []*ec2.IpPermission {
key := permKey{IpProtocol: rule.IpProtocol, FromPort: rule.FromPort, ToPort: rule.ToPort}
perm, exists := grouped[key]
if !exists {
- perm = &ec2.IpPermission{
- IpProtocol: aws.String(rule.IpProtocol),
- FromPort: aws.Int64(rule.FromPort),
- ToPort: aws.Int64(rule.ToPort),
+ perm = &ec2.IpPermission{IpProtocol: aws.String(rule.IpProtocol)}
+ // AWS omits both ports on an all-protocol permission here, unlike the
+ // -1/-1 that DescribeSecurityGroupRules reports for the same rule.
+ if rule.IpProtocol != allProtocols {
+ perm.FromPort = aws.Int64(rule.FromPort)
+ perm.ToPort = aws.Int64(rule.ToPort)
}
grouped[key] = perm
}
@@ -1938,7 +2030,7 @@ func sgRulesToIpPermissions(rules []SGRule) []*ec2.IpPermission {
perm.Ipv6Ranges = append(perm.Ipv6Ranges, ipRange)
}
if rule.SourceSG != "" {
- pair := &ec2.UserIdGroupPair{GroupId: aws.String(rule.SourceSG)}
+ pair := &ec2.UserIdGroupPair{GroupId: aws.String(rule.SourceSG), UserId: aws.String(accountID)}
if rule.Description != "" {
pair.Description = aws.String(rule.Description)
}
@@ -2007,7 +2099,11 @@ func removeSGRules(existing, toRemove []SGRule) []SGRule {
}
// sgRuleKey returns a string key for deduplication/matching of SG rules.
+// An all-protocol rule has no ports, so its stored or requested ports are ignored.
func sgRuleKey(r SGRule) string {
+ if r.IpProtocol == allProtocols {
+ r.FromPort, r.ToPort = 0, 0
+ }
return fmt.Sprintf("%s:%d:%d:%s:%s:%s", r.IpProtocol, r.FromPort, r.ToPort, r.CidrIp, r.CidrIpv6, r.SourceSG)
}
@@ -2036,7 +2132,7 @@ func resolveRuleIDsToRemove(existing []SGRule, ruleIDs []*string) ([]SGRule, err
}
r, ok := byID[*idp]
if !ok {
- return nil, errors.New(awserrors.ErrorInvalidSecurityGroupRuleIdNotFound)
+ return nil, sgRuleNotFoundError(*idp)
}
out = append(out, r)
}
diff --git a/spinifex/handlers/ec2/vpc/security_group_modify_test.go b/spinifex/handlers/ec2/vpc/security_group_modify_test.go
index 5a3e353ab..c735a7988 100644
--- a/spinifex/handlers/ec2/vpc/security_group_modify_test.go
+++ b/spinifex/handlers/ec2/vpc/security_group_modify_test.go
@@ -7,6 +7,7 @@ package handlers_ec2_vpc
import (
"context"
"encoding/json"
+ "fmt"
"sync"
"testing"
@@ -27,6 +28,14 @@ func requireAWSCode(t *testing.T, err error, code string) {
assert.Equal(t, code, got, "error: %v", err)
}
+func requireAWSError(t *testing.T, err error, code, message string) {
+ t.Helper()
+ got, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, code, got)
+ assert.Equal(t, message, msg)
+}
+
func modifyRules(svc *VPCServiceImpl, sgID string, updates ...*ec2.SecurityGroupRuleUpdate) error {
_, err := svc.ModifySecurityGroupRules(context.Background(), &ec2.ModifySecurityGroupRulesInput{
GroupId: aws.String(sgID),
@@ -631,3 +640,85 @@ func TestModifySecurityGroupRules_ConcurrentAuthorize(t *testing.T) {
assert.Equal(t, modErr == nil, modified, "the modify succeeded iff its effect is stored")
assert.Equal(t, authErr == nil, authorized, "the authorize succeeded iff its effect is stored")
}
+
+func requireInvertedPortRange(t *testing.T, err error, from, to int64) {
+ t.Helper()
+ require.Error(t, err)
+ code, message, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %q carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+ assert.Equal(t, fmt.Sprintf("Invalid TCP/UDP port range(%d:%d)", from, to), message)
+}
+
+func TestAuthorizeSecurityGroupRules_RejectsInvertedPortRange(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ sgID := createTestSG(t, svc, vpcID, "inverted-ports")
+ before := storedSGRecord(t, svc, sgID)
+
+ perm := func(proto string) []*ec2.IpPermission {
+ return []*ec2.IpPermission{{
+ IpProtocol: aws.String(proto), FromPort: aws.Int64(100), ToPort: aws.Int64(50),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/24")}},
+ }}
+ }
+ _, err := svc.AuthorizeSecurityGroupIngress(context.Background(), &ec2.AuthorizeSecurityGroupIngressInput{
+ GroupId: aws.String(sgID), IpPermissions: perm("tcp"),
+ }, testAccountID)
+ requireInvertedPortRange(t, err, 100, 50)
+
+ _, err = svc.AuthorizeSecurityGroupEgress(context.Background(), &ec2.AuthorizeSecurityGroupEgressInput{
+ GroupId: aws.String(sgID), IpPermissions: perm("udp"),
+ }, testAccountID)
+ requireInvertedPortRange(t, err, 100, 50)
+
+ assert.Equal(t, before, storedSGRecord(t, svc, sgID), "a rejected authorize may store nothing")
+}
+
+func TestModifySecurityGroupRules_RejectsInvertedPortRange(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ sgID := createTestSG(t, svc, vpcID, "mod-inverted-ports")
+ ruleID := authorizeIngressTCP(t, svc, sgID, 80, "10.0.0.0/24")
+
+ req := tcpCIDRRequest(100, "10.0.0.0/24")
+ req.ToPort = aws.Int64(50)
+ requireInvertedPortRange(t, modifyRules(svc, sgID, ruleUpdate(ruleID, req)), 100, 50)
+ assert.Equal(t, int64(80), storedSGRule(t, svc, sgID, ruleID).FromPort)
+}
+
+// TestRevokeSecurityGroupIngress_InvertedPortRangeStillRevocable covers a rule
+// stored before the range check existed: revoking it by its ports must work.
+func TestRevokeSecurityGroupIngress_InvertedPortRangeStillRevocable(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ sgID := createTestSG(t, svc, vpcID, "stale-inverted-ports")
+
+ key := utils.AccountKey(testAccountID, sgID)
+ var rec SecurityGroupRecord
+ require.NoError(t, json.Unmarshal(storedSGRecord(t, svc, sgID), &rec))
+ rec.IngressRules = append(rec.IngressRules, SGRule{
+ RuleId: "sgr-0123456789abcdef0", IpProtocol: "tcp", FromPort: 100, ToPort: 50, CidrIp: "10.0.0.0/24",
+ })
+ data, err := json.Marshal(rec)
+ require.NoError(t, err)
+ _, err = svc.sgKV.Put(t.Context(), key, data)
+ require.NoError(t, err)
+
+ _, err = svc.RevokeSecurityGroupIngress(context.Background(), &ec2.RevokeSecurityGroupIngressInput{
+ GroupId: aws.String(sgID),
+ IpPermissions: []*ec2.IpPermission{{
+ IpProtocol: aws.String("tcp"), FromPort: aws.Int64(100), ToPort: aws.Int64(50),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/24")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+
+ require.NoError(t, json.Unmarshal(storedSGRecord(t, svc, sgID), &rec))
+ for _, r := range rec.IngressRules {
+ assert.NotEqual(t, "sgr-0123456789abcdef0", r.RuleId)
+ }
+}
diff --git a/spinifex/handlers/ec2/vpc/security_group_test.go b/spinifex/handlers/ec2/vpc/security_group_test.go
index 86a8d8dcb..2b6ce671e 100644
--- a/spinifex/handlers/ec2/vpc/security_group_test.go
+++ b/spinifex/handlers/ec2/vpc/security_group_test.go
@@ -3,6 +3,7 @@ package handlers_ec2_vpc
import (
"context"
"encoding/json"
+ "fmt"
"strings"
"testing"
"time"
@@ -44,6 +45,7 @@ func TestCreateSecurityGroup_Success(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
assert.NotEmpty(t, *out.GroupId)
+ assert.Nil(t, out.Tags, "AWS omits Tags from an untagged CreateSecurityGroup")
}
func TestCreateSecurityGroup_MissingGroupName(t *testing.T) {
@@ -93,8 +95,7 @@ func TestCreateSecurityGroup_DuplicateName(t *testing.T) {
GroupName: aws.String("dup-sg"),
VpcId: aws.String(vpcID),
}, testAccountID)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidGroup.Duplicate")
+ requireAWSError(t, err, awserrors.ErrorInvalidGroupDuplicate, "The security group 'dup-sg' already exists for VPC '"+vpcID+"'")
}
// --- DeleteSecurityGroup ---
@@ -132,8 +133,8 @@ func TestDescribeSecurityGroups_All(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
- createTestSG(t, svc, vpcID, "sg-a")
- createTestSG(t, svc, vpcID, "sg-b")
+ createTestSG(t, svc, vpcID, "grp-a")
+ createTestSG(t, svc, vpcID, "grp-b")
desc, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{}, testAccountID)
require.NoError(t, err)
@@ -178,10 +179,33 @@ func TestDescribeSecurityGroups_NotFound(t *testing.T) {
svc := setupTestVPCService(t)
_, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
- GroupIds: []*string{aws.String("sg-nonexistent")},
+ GroupIds: []*string{aws.String("sg-0000000000000dead")},
}, testAccountID)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidGroup.NotFound")
+ requireAWSError(t, err, awserrors.ErrorInvalidGroupNotFound, "The security group 'sg-0000000000000dead' does not exist")
+}
+
+// The shapes below are the answers AWS gave for each ID; a 17-digit ID is
+// always treated as unknown here, though AWS decodes some of those as Malformed.
+func TestDescribeSecurityGroups_MalformedID(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ describe := func(ids ...string) error {
+ _, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ GroupIds: aws.StringSlice(ids),
+ }, testAccountID)
+ return err
+ }
+
+ for _, id := range []string{"sg-xyz", "sg-", "sg-0AAAAAAAAAAAAAAA1", "sg-000000001", "sg-0000000000000001", "sg-000000000000000001"} {
+ requireAWSError(t, describe(id), awserrors.ErrorInvalidGroupIdMalformed, fmt.Sprintf("Invalid id: %q", id))
+ }
+ requireAWSError(t, describe("foo"), awserrors.ErrorInvalidGroupIdMalformed, `Invalid id: "foo" (expecting "sg-...")`)
+ for _, id := range []string{"sg-a", "sg-0aaaaaaa", "sg-01234567890abcdef"} {
+ requireAWSCode(t, describe(id), awserrors.ErrorInvalidGroupNotFound)
+ }
+
+ // A malformed ID wins over an unknown one, wherever it sits.
+ requireAWSCode(t, describe("sg-0aaaaaaa", "sg-xyz"), awserrors.ErrorInvalidGroupIdMalformed)
}
// --- AuthorizeSecurityGroupIngress ---
@@ -405,6 +429,34 @@ func TestRevokeSecurityGroupEgress_Success(t *testing.T) {
require.NoError(t, err)
}
+// TestRevokeSecurityGroupEgress_AllProtocolIgnoresPorts: DescribeSecurityGroups
+// reports no ports on an all-protocol rule, so a caller revoking it may send
+// -1/-1 (as the UI does) and must still match the stored rule.
+func TestRevokeSecurityGroupEgress_AllProtocolIgnoresPorts(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+ sgID := createTestSG(t, svc, vpcID, "revoke-egress-ports-sg")
+
+ _, err := svc.RevokeSecurityGroupEgress(context.Background(), &ec2.RevokeSecurityGroupEgressInput{
+ GroupId: aws.String(sgID),
+ IpPermissions: []*ec2.IpPermission{{
+ IpProtocol: aws.String("-1"),
+ FromPort: aws.Int64(-1),
+ ToPort: aws.Int64(-1),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+
+ desc, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ GroupIds: []*string{aws.String(sgID)},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, desc.SecurityGroups, 1)
+ assert.Empty(t, desc.SecurityGroups[0].IpPermissionsEgress)
+}
+
func TestRevokeSecurityGroupEgress_NotFound(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -583,29 +635,25 @@ func ingressRuleIDReferencing(t *testing.T, svc *VPCServiceImpl, sgID, srcSG str
return ""
}
-// TestDeleteSecurityGroup_DependencyViolationNamesENI pins that the refusal
-// identifies the attached ENI. A bare code forces the blocking id to be
-// recovered by cross-referencing the whole ENI table after the fact.
-func TestDeleteSecurityGroup_DependencyViolationNamesENI(t *testing.T) {
+// TestDeleteSecurityGroup_DependencyViolationForENI pins AWS's refusal for a
+// group still attached to an interface, which names the group, not the ENI.
+func TestDeleteSecurityGroup_DependencyViolationForENI(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
sgID := createTestSG(t, svc, vpcID, "attached-sg")
- eniID := createAutoENI(t, svc, subnetID, "i-holder", []string{sgID}, time.Hour)
+ createAutoENI(t, svc, subnetID, "i-holder", []string{sgID}, time.Hour)
_, err := svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{
GroupId: aws.String(sgID),
}, testAccountID)
- require.Error(t, err)
- code, _ := awserrors.ResolveErrorCode(err)
- assert.Equal(t, awserrors.ErrorDependencyViolation, code)
- assert.ErrorContains(t, err, eniID, "the refusal must name the blocking ENI")
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "resource "+sgID+" has a dependent object")
}
-// TestDeleteSecurityGroup_DependencyViolationNamesReferencingSG pins the same
-// for the SourceSG case: the group holding the rule is named, not just the code.
-func TestDeleteSecurityGroup_DependencyViolationNamesReferencingSG(t *testing.T) {
+// TestDeleteSecurityGroup_DependencyViolationForReferencingSG pins the same
+// answer for a group another group's rule references.
+func TestDeleteSecurityGroup_DependencyViolationForReferencingSG(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -617,10 +665,7 @@ func TestDeleteSecurityGroup_DependencyViolationNamesReferencingSG(t *testing.T)
_, err := svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{
GroupId: aws.String(targetSG),
}, testAccountID)
- require.Error(t, err)
- code, _ := awserrors.ResolveErrorCode(err)
- assert.Equal(t, awserrors.ErrorDependencyViolation, code)
- assert.ErrorContains(t, err, holderSG, "the refusal must name the referencing security group")
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "resource "+targetSG+" has a dependent object")
}
// TestRevokeSecurityGroupIngress_ByRuleID_BreaksMutualReferenceTeardown
@@ -879,7 +924,7 @@ func TestSGRulesToIpPermissions_IPv6RoundTrip(t *testing.T) {
perms := sgRulesToIpPermissions([]SGRule{
{IpProtocol: "-1", CidrIp: "0.0.0.0/0"},
{IpProtocol: "-1", CidrIpv6: "::/0"},
- })
+ }, testAccountID)
require.Len(t, perms, 1)
require.Len(t, perms[0].IpRanges, 1)
require.Len(t, perms[0].Ipv6Ranges, 1)
@@ -894,7 +939,7 @@ func TestSGRulesToIpPermissions_Conversion(t *testing.T) {
{IpProtocol: "-1", SourceSG: "sg-other"},
}
- perms := sgRulesToIpPermissions(rules)
+ perms := sgRulesToIpPermissions(rules, testAccountID)
require.Len(t, perms, 2)
// Order is non-deterministic (map-based), find each by protocol
@@ -913,8 +958,11 @@ func TestSGRulesToIpPermissions_Conversion(t *testing.T) {
assert.Len(t, tcpPerm.IpRanges, 1)
require.NotNil(t, allPerm, "should have all-traffic permission")
+ assert.Nil(t, allPerm.FromPort, "AWS omits ports on an all-protocol permission")
+ assert.Nil(t, allPerm.ToPort)
assert.Len(t, allPerm.UserIdGroupPairs, 1)
assert.Equal(t, "sg-other", *allPerm.UserIdGroupPairs[0].GroupId)
+ assert.Equal(t, testAccountID, aws.StringValue(allPerm.UserIdGroupPairs[0].UserId))
}
func TestSGRuleKey(t *testing.T) {
@@ -1272,8 +1320,8 @@ func TestDescribeSecurityGroups_FilterByVpcId(t *testing.T) {
svc := setupTestVPCService(t)
vpc1 := createTestVPC(t, svc, "10.0.0.0/16")
vpc2 := createTestVPC(t, svc, "172.16.0.0/16")
- createTestSG(t, svc, vpc1, "sg-in-vpc1")
- createTestSG(t, svc, vpc2, "sg-in-vpc2")
+ createTestSG(t, svc, vpc1, "grp-in-vpc1")
+ createTestSG(t, svc, vpc2, "grp-in-vpc2")
out, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
Filters: []*ec2.Filter{
@@ -1325,13 +1373,13 @@ func TestDescribeSecurityGroups_FilterMultipleValues_OR(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
- createTestSG(t, svc, vpcID, "sg-alpha")
- createTestSG(t, svc, vpcID, "sg-beta")
- createTestSG(t, svc, vpcID, "sg-gamma")
+ createTestSG(t, svc, vpcID, "grp-alpha")
+ createTestSG(t, svc, vpcID, "grp-beta")
+ createTestSG(t, svc, vpcID, "grp-gamma")
out, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
Filters: []*ec2.Filter{
- {Name: aws.String("group-name"), Values: []*string{aws.String("sg-alpha"), aws.String("sg-gamma")}},
+ {Name: aws.String("group-name"), Values: []*string{aws.String("grp-alpha"), aws.String("grp-gamma")}},
},
}, testAccountID)
require.NoError(t, err)
@@ -1403,6 +1451,9 @@ func TestDescribeSecurityGroups_FilterByTag(t *testing.T) {
},
}, testAccountID)
require.NoError(t, err)
+ require.Len(t, out.Tags, 1, "CreateSecurityGroup echoes the requested tags")
+ assert.Equal(t, "Env", aws.StringValue(out.Tags[0].Key))
+ assert.Equal(t, "prod", aws.StringValue(out.Tags[0].Value))
createTestSG(t, svc, vpcID, "untagged-sg")
@@ -1416,6 +1467,39 @@ func TestDescribeSecurityGroups_FilterByTag(t *testing.T) {
assert.Equal(t, *out.GroupId, *desc.SecurityGroups[0].GroupId)
}
+func TestDescribeSecurityGroups_FilterByTagKeyAndTagValue(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+
+ out, err := svc.CreateSecurityGroup(context.Background(), &ec2.CreateSecurityGroupInput{
+ GroupName: aws.String("tagged-sg"),
+ Description: aws.String("tagged"),
+ VpcId: aws.String(vpcID),
+ TagSpecifications: []*ec2.TagSpecification{{
+ ResourceType: aws.String("security-group"),
+ Tags: []*ec2.Tag{{Key: aws.String("Env"), Value: aws.String("prod")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestSG(t, svc, vpcID, "untagged-sg")
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Env"}, {"tag-value", "prod"}} {
+ desc, err := svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.SecurityGroups, 1, tc.name)
+ assert.Equal(t, *out.GroupId, *desc.SecurityGroups[0].GroupId, tc.name)
+
+ desc, err = svc.DescribeSecurityGroups(context.Background(), &ec2.DescribeSecurityGroupsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.SecurityGroups, tc.name)
+ }
+}
+
func TestDescribeSecurityGroups_FilterNoResults(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -1488,8 +1572,31 @@ func TestCreateSecurityGroup_RejectsReservedDefaultName(t *testing.T) {
Description: aws.String("user-supplied"),
VpcId: aws.String(vpcID),
}, testAccountID)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidGroup.Reserved")
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "Cannot use reserved security group name: default")
+}
+
+func TestCreateSecurityGroup_RejectsSgPrefixedName(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+
+ _, err := svc.CreateSecurityGroup(context.Background(), &ec2.CreateSecurityGroupInput{
+ GroupName: aws.String("sg-foo"),
+ Description: aws.String("d"),
+ VpcId: aws.String(vpcID),
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+ assert.Equal(t, "Value (sg-foo) for parameter GroupName is invalid. Group names may not be in the format sg-*.", msg)
+
+ // AWS reports a missing VPC ahead of the name.
+ _, err = svc.CreateSecurityGroup(context.Background(), &ec2.CreateSecurityGroupInput{
+ GroupName: aws.String("sg-foo"),
+ Description: aws.String("d"),
+ VpcId: aws.String("vpc-nonexistent"),
+ }, testAccountID)
+ assert.ErrorContains(t, err, awserrors.ErrorInvalidVpcIDNotFound)
}
func TestDeleteSecurityGroup_RejectsDefault(t *testing.T) {
@@ -1501,8 +1608,12 @@ func TestDeleteSecurityGroup_RejectsDefault(t *testing.T) {
_, err := svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{
GroupId: aws.String(sgID),
}, testAccountID)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "CannotDelete")
+ requireAWSError(t, err, awserrors.ErrorCannotDelete, `the specified group: "`+sgID+`" name: "default" cannot be deleted by a user`)
+
+ _, err = svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{
+ GroupId: aws.String("sg-0000000000000dead"),
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidGroupNotFound, "The security group 'sg-0000000000000dead' does not exist")
}
func TestDeleteSecurityGroup_RejectsAttachedToENI(t *testing.T) {
@@ -1553,14 +1664,10 @@ func TestDeleteVpc_BlocksOnNonDefaultSG(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
- sgID := createTestSG(t, svc, vpcID, "user-sg")
+ createTestSG(t, svc, vpcID, "user-sg")
_, err := svc.DeleteVpc(context.Background(), &ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}, testAccountID)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "DependencyViolation")
- // The message must name the blocking security group so the caller knows
- // what to delete first, not just that something is blocking.
- assert.Contains(t, err.Error(), sgID)
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The vpc '"+vpcID+"' has dependencies and cannot be deleted.")
}
func TestDeleteVpc_CascadesDefaultSG(t *testing.T) {
@@ -2200,7 +2307,7 @@ func TestDescribeSecurityGroupRules_ReferencedGroupInfo(t *testing.T) {
dstSG := createTestSG(t, svc, vpcID, "dest")
proto := "tcp"
- _, err := svc.AuthorizeSecurityGroupIngress(context.Background(), &ec2.AuthorizeSecurityGroupIngressInput{
+ authOut, err := svc.AuthorizeSecurityGroupIngress(context.Background(), &ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String(dstSG),
IpPermissions: []*ec2.IpPermission{{
IpProtocol: &proto,
@@ -2210,6 +2317,8 @@ func TestDescribeSecurityGroupRules_ReferencedGroupInfo(t *testing.T) {
}},
}, testAccountID)
require.NoError(t, err)
+ require.Len(t, authOut.SecurityGroupRules, 1)
+ assert.Nil(t, authOut.SecurityGroupRules[0].Tags, "Authorize* omits Tags on an untagged rule")
out, err := svc.DescribeSecurityGroupRules(context.Background(), &ec2.DescribeSecurityGroupRulesInput{
Filters: []*ec2.Filter{{Name: aws.String("group-id"), Values: []*string{aws.String(dstSG)}}},
@@ -2221,11 +2330,12 @@ func TestDescribeSecurityGroupRules_ReferencedGroupInfo(t *testing.T) {
if r.ReferencedGroupInfo != nil {
require.NotNil(t, r.ReferencedGroupInfo.GroupId)
require.NotNil(t, r.ReferencedGroupInfo.UserId)
- require.NotNil(t, r.ReferencedGroupInfo.VpcId)
assert.Equal(t, srcSG, *r.ReferencedGroupInfo.GroupId)
assert.Equal(t, testAccountID, *r.ReferencedGroupInfo.UserId)
- assert.Equal(t, vpcID, *r.ReferencedGroupInfo.VpcId)
+ assert.Nil(t, r.ReferencedGroupInfo.VpcId, "AWS names no VpcId for a same-VPC reference")
assert.Nil(t, r.CidrIpv4)
+ assert.NotNil(t, r.Tags, "DescribeSecurityGroupRules lists an untagged rule with empty Tags")
+ assert.Empty(t, r.Tags)
found = true
}
}
@@ -2285,8 +2395,43 @@ func TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate(t *testing.T) {
GroupId: aws.String(sgID),
IpPermissions: []*ec2.IpPermission{perm},
}, testAccountID)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "InvalidPermission.Duplicate")
+ requireAWSError(t, err, awserrors.ErrorInvalidPermissionDuplicate,
+ `the specified rule "peer: 10.0.0.0/24, TCP, from port: 22, to port: 22, ALLOW" already exists`)
+
+ // The default egress rule, re-added, is described without ports
+ _, err = svc.AuthorizeSecurityGroupEgress(context.Background(), &ec2.AuthorizeSecurityGroupEgressInput{
+ GroupId: aws.String(sgID),
+ IpPermissions: []*ec2.IpPermission{{
+ IpProtocol: aws.String("-1"),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
+ }},
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidPermissionDuplicate, `the specified rule "peer: 0.0.0.0/0, ALL, ALLOW" already exists`)
+
+ icmp := &ec2.IpPermission{
+ IpProtocol: aws.String("icmp"), FromPort: aws.Int64(8), ToPort: aws.Int64(-1),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/24")}},
+ }
+ for range 2 {
+ _, err = svc.AuthorizeSecurityGroupIngress(context.Background(), &ec2.AuthorizeSecurityGroupIngressInput{
+ GroupId: aws.String(sgID), IpPermissions: []*ec2.IpPermission{icmp},
+ }, testAccountID)
+ }
+ requireAWSError(t, err, awserrors.ErrorInvalidPermissionDuplicate, `the specified rule "peer: 10.0.0.0/24, ICMP, type: 8, code: ALL, ALLOW" already exists`)
+
+ _, err = svc.AuthorizeSecurityGroupIngress(context.Background(), &ec2.AuthorizeSecurityGroupIngressInput{
+ GroupId: aws.String(sgID),
+ IpPermissions: []*ec2.IpPermission{{
+ IpProtocol: &proto, FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+ IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/33")}},
+ }},
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "CIDR block 10.0.0.0/33 is malformed")
+
+ _, err = svc.DescribeSecurityGroupRules(context.Background(), &ec2.DescribeSecurityGroupRulesInput{
+ SecurityGroupRuleIds: []*string{aws.String("sgr-0000000000000dead")},
+ }, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidSecurityGroupRuleIdNotFound, "The security group rule ID 'sgr-0000000000000dead' does not exist")
}
func TestDescribeSecurityGroupRules_InvalidFilter(t *testing.T) {
@@ -2441,6 +2586,13 @@ func TestDescribeSecurityGroupRules_FiltersByTag(t *testing.T) {
require.NoError(t, err)
require.Len(t, byKey.SecurityGroupRules, 1)
assert.Equal(t, taggedID, aws.StringValue(byKey.SecurityGroupRules[0].SecurityGroupRuleId))
+
+ byTagValue, err := svc.DescribeSecurityGroupRules(context.Background(), &ec2.DescribeSecurityGroupRulesInput{
+ Filters: []*ec2.Filter{{Name: aws.String("tag-value"), Values: []*string{aws.String("alb")}}},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, byTagValue.SecurityGroupRules, 1)
+ assert.Equal(t, taggedID, aws.StringValue(byTagValue.SecurityGroupRules[0].SecurityGroupRuleId))
}
func TestDescribeSecurityGroupRules_AllProtocolReportsMinusOnePorts(t *testing.T) {
diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go
index 36246a24b..05f2bb0db 100644
--- a/spinifex/handlers/ec2/vpc/service_impl.go
+++ b/spinifex/handlers/ec2/vpc/service_impl.go
@@ -14,10 +14,12 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/config"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+ "github.com/mulgadc/spinifex/spinifex/paging"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -96,6 +98,10 @@ type VPCServiceImpl struct {
// A nil value leaves both unchanged.
centralTags CentralTagStore
+ // Optional: injected after construction to name interfaces' private
+ // addresses. Nil leaves them unnamed.
+ privateDNSName func(privateIP string) string
+
// disableDefaultPublicIP seeds default subnets with MapPublicIpOnLaunch=false
// (non-pool external modes have no public IPs to assign). Zero value keeps
// the AWS-faithful default of true.
@@ -115,6 +121,13 @@ func (s *VPCServiceImpl) SetExternalIPAM(ipam *ExternalIPAM, eipKV jetstream.Key
s.eipKV = eipKV
}
+// SetPrivateDNSNamer injects how a private address is named, so an interface
+// reports the same PrivateDnsName as the instance using it. Nil, or a namer
+// that answers "", leaves interfaces unnamed.
+func (s *VPCServiceImpl) SetPrivateDNSNamer(namer func(privateIP string) string) {
+ s.privateDNSName = namer
+}
+
// SetCentralTagStore injects the central tag store so create paths project
// their record tags into it and delete paths clear them. Nil leaves both
// unchanged.
@@ -132,6 +145,15 @@ func (s *VPCServiceImpl) localAZ() string {
return s.config.AZ
}
+// region returns the configured AWS region, falling back to the default when
+// no config or region is wired.
+func (s *VPCServiceImpl) region() string {
+ if s.config == nil || s.config.Region == "" {
+ return config.DefaultAWSRegion
+ }
+ return s.config.Region
+}
+
// NewVPCServiceImplWithNATS creates a VPC service with NATS JetStream for persistence.
func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*VPCServiceImpl, error) {
js, err := jetstream.New(natsConn)
@@ -244,22 +266,27 @@ func (s *VPCServiceImpl) nextVNI(ctx context.Context) (int64, error) {
return allocated, nil
}
+// invalidCIDRBlockError is AWS's answer to a CidrBlock that does not parse.
+func invalidCIDRBlockError(cidr string) error {
+ return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "Value (%s) for parameter cidrBlock is invalid. This is not a valid CIDR block.", cidr)
+}
+
// CreateVpc creates a new VPC.
func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInput, accountID string) (*ec2.CreateVpcOutput, error) {
if input.CidrBlock == nil || *input.CidrBlock == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
- // Validate CIDR block
_, ipNet, err := net.ParseCIDR(*input.CidrBlock)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidVpcRange)
+ return nil, invalidCIDRBlockError(*input.CidrBlock)
}
// AWS allows /16 to /28 for VPC CIDR blocks
ones, _ := ipNet.Mask.Size()
if ones < 16 || ones > 28 {
- return nil, errors.New(awserrors.ErrorInvalidVpcRange)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidVpcRange, "The CIDR '%s' is invalid.", *input.CidrBlock)
}
// Allocate VNI for overlay network
@@ -312,16 +339,18 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
return nil, errors.New(awserrors.ErrorServerInternal)
}
- return &ec2.CreateVpcOutput{
- Vpc: s.vpcRecordToEC2(&record, accountID),
- }, nil
+ // AWS renders the empty IPv6 association list on CreateVpc only;
+ // DescribeVpcs omits it.
+ vpc := s.vpcRecordToEC2(&record, accountID)
+ vpc.Ipv6CidrBlockAssociationSet = []*ec2.VpcIpv6CidrBlockAssociation{}
+ return &ec2.CreateVpcOutput{Vpc: vpc}, nil
}
// requireVPCExists returns InvalidVpcID.NotFound if the VPC doesn't exist for
// this account.
func (s *VPCServiceImpl) requireVPCExists(ctx context.Context, accountID, vpcId string) error {
if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcId)); err != nil {
- return errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return vpcNotFoundError(vpcId)
}
return nil
}
@@ -340,7 +369,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
// tolerates it on destroy); destroy orchestration tolerates it too.
// A transient read error stays a server error.
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, vpcNotFoundError(vpcID)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -376,8 +405,8 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
return nil, errors.New(awserrors.ErrorServerInternal)
}
if subnet.VpcId == vpcID {
- return nil, awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the VPC has a dependent subnet %s that must be deleted first", subnet.SubnetId)
+ slog.WarnContext(ctx, "DeleteVpc: VPC has a dependent subnet", "vpcId", vpcID, "subnetId", subnet.SubnetId)
+ return nil, awserrors.HasDependencies("vpc", vpcID)
}
}
@@ -413,8 +442,8 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
continue
}
if !sg.IsDefault {
- return nil, awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the VPC has a dependent security group %s that must be deleted first", sg.GroupId)
+ slog.WarnContext(ctx, "DeleteVpc: VPC has a dependent security group", "vpcId", vpcID, "groupId", sg.GroupId)
+ return nil, awserrors.HasDependencies("vpc", vpcID)
}
defaultSGId = sg.GroupId
}
@@ -454,8 +483,8 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
continue
}
if !rtb.IsMain {
- return nil, awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the VPC has a dependent route table %s that must be deleted first", rtb.RouteTableId)
+ slog.WarnContext(ctx, "DeleteVpc: VPC has a dependent route table", "vpcId", vpcID, "routeTableId", rtb.RouteTableId)
+ return nil, awserrors.HasDependencies("vpc", vpcID)
}
}
@@ -548,8 +577,8 @@ func (s *VPCServiceImpl) rejectAttachedIGW(ctx context.Context, accountID, vpcID
return errors.New(awserrors.ErrorServerInternal)
}
if igw.VpcId == vpcID {
- return awserrors.Errorf(awserrors.ErrorDependencyViolation,
- "the VPC has a dependent internet gateway %s that must be detached first", igw.InternetGatewayId)
+ slog.WarnContext(ctx, "DeleteVpc: VPC has an attached internet gateway", "vpcId", vpcID, "internetGatewayId", igw.InternetGatewayId)
+ return awserrors.HasDependencies("vpc", vpcID)
}
}
@@ -569,6 +598,8 @@ var describeVpcsValidFilters = map[string]bool{
"isDefault": true,
"owner-id": true,
"cidr-block-association.cidr-block": true,
+ "tag-key": true,
+ "tag-value": true,
}
// SupportsDescribeVpcsFilter reports whether DescribeVpcs accepts a filter name.
@@ -583,6 +614,11 @@ func SupportsDescribeVpcsFilter(name string) bool {
func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVpcsInput, accountID string) (*ec2.DescribeVpcsOutput, error) {
var vpcs []*ec2.Vpc
+ // Validated but not paged: AWS's paging of DescribeVpcs is unobserved.
+ if _, err := describeVpcsPaging.Parse(input.MaxResults, input.NextToken, len(input.VpcIds)); err != nil {
+ return nil, err
+ }
+
vpcIDs := make(map[string]bool)
for _, id := range input.VpcIds {
if id != nil {
@@ -641,9 +677,9 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp
found[*vpc.VpcId] = true
}
}
- for id := range vpcIDs {
- if !found[id] {
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ for _, id := range input.VpcIds {
+ if id != nil && !found[*id] {
+ return nil, vpcNotFoundError(*id)
}
}
}
@@ -666,26 +702,27 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
vpcID := *input.VpcId
+ // AWS rejects an unparseable CIDR before it looks the VPC up.
+ _, subnetNet, err := net.ParseCIDR(*input.CidrBlock)
+ if err != nil {
+ return nil, invalidCIDRBlockError(*input.CidrBlock)
+ }
+
// Verify VPC exists and belongs to this account
vpcEntry, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID))
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, vpcNotFoundError(vpcID)
}
var vpcRecord VPCRecord
if err := json.Unmarshal(vpcEntry.Value(), &vpcRecord); err != nil {
return nil, errors.New(awserrors.ErrorServerInternal)
}
- // Validate subnet CIDR
- _, subnetNet, err := net.ParseCIDR(*input.CidrBlock)
- if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidSubnetRange)
- }
// AWS allows /16 to /28 for subnet CIDR blocks
ones, _ := subnetNet.Mask.Size()
if ones < 16 || ones > 28 {
- return nil, errors.New(awserrors.ErrorInvalidSubnetRange)
+ return nil, subnetRangeError(*input.CidrBlock)
}
// Verify subnet CIDR is within VPC CIDR
@@ -695,7 +732,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
}
if !vpcNet.Contains(subnetNet.IP) {
- return nil, errors.New(awserrors.ErrorInvalidSubnetRange)
+ return nil, subnetRangeError(*input.CidrBlock)
}
// Check for CIDR conflicts with existing subnets in this VPC (same account)
@@ -728,7 +765,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
continue
}
if existingNet.Contains(subnetNet.IP) || subnetNet.Contains(existingNet.IP) {
- return nil, errors.New(awserrors.ErrorInvalidSubnetConflict)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidSubnetConflict, "The CIDR '%s' conflicts with another subnet", *input.CidrBlock)
}
}
@@ -793,7 +830,7 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn
// destroy); destroy orchestration tolerates it too. A transient read
// error stays a server error.
if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidSubnetIDNotFound, "The subnet ID '%s' does not exist", subnetID)
}
return nil, errors.New(awserrors.ErrorServerInternal)
}
@@ -858,7 +895,7 @@ func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, su
continue
}
if record.SubnetId == subnetID && eniIsLiveAttachment(&record) {
- return errors.New(awserrors.ErrorDependencyViolation)
+ return awserrors.HasDependencies("subnet", subnetID)
}
}
return nil
@@ -955,6 +992,11 @@ func (s *VPCServiceImpl) clearRouteTableAssociationsForSubnet(ctx context.Contex
func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.DescribeSubnetsInput, accountID string) (*ec2.DescribeSubnetsOutput, error) {
var subnets []*ec2.Subnet
+ pageReq, err := describeSubnetsPaging.Parse(input.MaxResults, input.NextToken, len(input.SubnetIds))
+ if err != nil {
+ return nil, err
+ }
+
subnetIDs := make(map[string]bool)
for _, id := range input.SubnetIds {
if id != nil {
@@ -1021,17 +1063,20 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
found[*subnet.SubnetId] = true
}
}
- for id := range subnetIDs {
- if !found[id] {
- return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
+ for _, id := range input.SubnetIds {
+ if id != nil && !found[*id] {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidSubnetIDNotFound, "The subnet ID '%s' does not exist", *id)
}
}
}
+ subnets, nextToken := paging.EC2Page(subnets, func(s *ec2.Subnet) string { return *s.SubnetId }, pageReq)
+
slog.InfoContext(ctx, "DescribeSubnets completed", "count", len(subnets), "accountID", accountID)
return &ec2.DescribeSubnetsOutput{
- Subnets: subnets,
+ Subnets: subnets,
+ NextToken: nextToken,
}, nil
}
@@ -1149,7 +1194,7 @@ func (s *VPCServiceImpl) updateRecordTags(ctx context.Context, accountID, resour
// vpcMatchesFilters checks whether a VPCRecord satisfies all parsed awsfilters.
func vpcMatchesFilters(record *VPCRecord, accountID string, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
@@ -1181,6 +1226,20 @@ func vpcMatchesFilters(record *VPCRecord, accountID string, filters map[string][
return awsfilters.MatchesTags(filters, record.Tags)
}
+var describeVpcsPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+ TooSmall: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter VpcIds cannot be used with the parameter MaxResults",
+}
+
+var describeSubnetsPaging = paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value smaller than or equal to 1000.",
+ TooSmall: "Value ( %d ) for parameter MaxResults is invalid. Expecting a value greater than or equal to 5.",
+ WithIDs: "The parameter SubnetIds cannot be used with the parameter MaxResults",
+}
+
// describeSubnetsValidFilters defines the set of filter names accepted by DescribeSubnets.
var describeSubnetsValidFilters = map[string]bool{
"subnet-id": true,
@@ -1189,12 +1248,14 @@ var describeSubnetsValidFilters = map[string]bool{
"cidr-block": true,
"state": true,
"default-for-az": true,
+ "tag-key": true,
+ "tag-value": true,
}
// subnetMatchesFilters checks whether a SubnetRecord satisfies all parsed awsfilters.
func subnetMatchesFilters(record *SubnetRecord, filters map[string][]string) bool {
for name, values := range filters {
- if strings.HasPrefix(name, "tag:") {
+ if awsfilters.IsTagFilter(name) {
continue
}
@@ -1261,6 +1322,18 @@ func (s *VPCServiceImpl) subnetRecordToEC2(record *SubnetRecord, availableIPs in
AvailableIpAddressCount: aws.Int64(int64(availableIPs)),
OwnerId: aws.String(accountID),
MapPublicIpOnLaunch: aws.Bool(record.MapPublicIpOnLaunch),
+ SubnetArn: aws.String(arn.FormatEC2(arn.EC2Subnet, s.region(), accountID, record.SubnetId)),
+ // Spinifex subnets are IPv4-only with no customer-owned pool, and
+ // instances are named ip- with no resource-name records.
+ AssignIpv6AddressOnCreation: aws.Bool(false),
+ EnableDns64: aws.Bool(false),
+ Ipv6Native: aws.Bool(false),
+ MapCustomerOwnedIpOnLaunch: aws.Bool(false),
+ PrivateDnsNameOptionsOnLaunch: &ec2.PrivateDnsNameOptionsOnLaunch{
+ HostnameType: aws.String(ec2.HostnameTypeIpName),
+ EnableResourceNameDnsARecord: aws.Bool(false),
+ EnableResourceNameDnsAAAARecord: aws.Bool(false),
+ },
}
subnet.Tags = utils.MapToEC2Tags(record.Tags)
@@ -1309,16 +1382,29 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi
if input.VpcId == nil || *input.VpcId == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
}
- if input.EnableDnsHostnames == nil && input.EnableDnsSupport == nil && input.EnableNetworkAddressUsageMetrics == nil {
+ var set []string
+ if input.EnableDnsSupport != nil {
+ set = append(set, "enableDnsSupport")
+ }
+ if input.EnableDnsHostnames != nil {
+ set = append(set, "enableDnsHostnames")
+ }
+ if input.EnableNetworkAddressUsageMetrics != nil {
+ set = append(set, "enableNetworkAddressUsageMetrics")
+ }
+ if len(set) == 0 {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
+ if len(set) > 1 {
+ return nil, multipleAttributesError(set)
+ }
vpcID := *input.VpcId
key := utils.AccountKey(accountID, vpcID)
entry, err := s.vpcKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, vpcNotFoundError(vpcID)
}
var record VPCRecord
@@ -1366,7 +1452,7 @@ func (s *VPCServiceImpl) DescribeVpcAttribute(ctx context.Context, input *ec2.De
entry, err := s.vpcKV.Get(ctx, key)
if err != nil {
- return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound)
+ return nil, vpcNotFoundError(vpcID)
}
var record VPCRecord
diff --git a/spinifex/handlers/ec2/vpc/service_impl_test.go b/spinifex/handlers/ec2/vpc/service_impl_test.go
index e8630994d..e27cab431 100644
--- a/spinifex/handlers/ec2/vpc/service_impl_test.go
+++ b/spinifex/handlers/ec2/vpc/service_impl_test.go
@@ -103,7 +103,8 @@ func TestCreateVpc_InvalidCidr(t *testing.T) {
_, err := svc.CreateVpc(context.Background(), &ec2.CreateVpcInput{
CidrBlock: aws.String("not-a-cidr"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidVpcRange")
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+ "Value (not-a-cidr) for parameter cidrBlock is invalid. This is not a valid CIDR block.")
}
func TestCreateVpc_CidrTooLarge(t *testing.T) {
@@ -112,7 +113,7 @@ func TestCreateVpc_CidrTooLarge(t *testing.T) {
_, err := svc.CreateVpc(context.Background(), &ec2.CreateVpcInput{
CidrBlock: aws.String("10.0.0.0/8"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidVpcRange")
+ requireAWSError(t, err, "InvalidVpc.Range", "The CIDR '10.0.0.0/8' is invalid.")
}
func TestCreateVpc_CidrTooSmall(t *testing.T) {
@@ -121,7 +122,7 @@ func TestCreateVpc_CidrTooSmall(t *testing.T) {
_, err := svc.CreateVpc(context.Background(), &ec2.CreateVpcInput{
CidrBlock: aws.String("10.0.0.0/29"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidVpcRange")
+ requireAWSError(t, err, "InvalidVpc.Range", "The CIDR '10.0.0.0/29' is invalid.")
}
func TestCreateVpc_WithTags(t *testing.T) {
@@ -212,16 +213,13 @@ func TestDeleteVpc_WithSubnets(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
- subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
+ createTestSubnet(t, svc, vpcID, "10.0.1.0/24")
// Should fail because VPC has subnets
_, err := svc.DeleteVpc(context.Background(), &ec2.DeleteVpcInput{
VpcId: aws.String(vpcID),
}, testAccountID)
- assert.ErrorContains(t, err, "DependencyViolation")
- // The message must name the blocking subnet so the caller knows what to
- // delete first, not just that something is blocking.
- assert.ErrorContains(t, err, subnetID)
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The vpc '"+vpcID+"' has dependencies and cannot be deleted.")
}
func TestDescribeVpcs_All(t *testing.T) {
@@ -263,7 +261,10 @@ func TestDescribeVpcs_NotFound(t *testing.T) {
_, err := svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{
VpcIds: []*string{aws.String("vpc-nonexistent")},
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidVpcID.NotFound")
+ requireAWSError(t, err, awserrors.ErrorInvalidVpcIDNotFound, "The vpc ID 'vpc-nonexistent' does not exist")
+
+ _, err = svc.DeleteVpc(context.Background(), &ec2.DeleteVpcInput{VpcId: aws.String("vpc-nonexistent")}, testAccountID)
+ requireAWSError(t, err, awserrors.ErrorInvalidVpcIDNotFound, "The vpc ID 'vpc-nonexistent' does not exist")
}
// --- Subnet Tests ---
@@ -287,6 +288,44 @@ func TestCreateSubnet(t *testing.T) {
assert.Equal(t, int64(251), *out.Subnet.AvailableIpAddressCount)
}
+// TestSubnetResponse_AWSDefaultFields checks the ARN and the fields AWS
+// reports for a new subnet, on both the create and describe paths.
+func TestSubnetResponse_AWSDefaultFields(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ svc.config = &config.Config{Region: "ap-southeast-2"}
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+
+ out, err := svc.CreateSubnet(context.Background(), &ec2.CreateSubnetInput{
+ VpcId: aws.String(vpcID),
+ CidrBlock: aws.String("10.0.1.0/24"),
+ }, testAccountID)
+ require.NoError(t, err)
+ desc, err := svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
+ SubnetIds: []*string{out.Subnet.SubnetId},
+ }, testAccountID)
+ require.NoError(t, err)
+ require.Len(t, desc.Subnets, 1)
+
+ for name, subnet := range map[string]*ec2.Subnet{"create": out.Subnet, "describe": desc.Subnets[0]} {
+ assert.Equal(t, "arn:aws:ec2:ap-southeast-2:"+testAccountID+":subnet/"+*out.Subnet.SubnetId, aws.StringValue(subnet.SubnetArn), name)
+ for field, v := range map[string]*bool{
+ "AssignIpv6AddressOnCreation": subnet.AssignIpv6AddressOnCreation,
+ "EnableDns64": subnet.EnableDns64,
+ "Ipv6Native": subnet.Ipv6Native,
+ "MapCustomerOwnedIpOnLaunch": subnet.MapCustomerOwnedIpOnLaunch,
+ } {
+ require.NotNil(t, v, "%s %s", name, field)
+ assert.False(t, *v, "%s %s", name, field)
+ }
+ assert.Equal(t, &ec2.PrivateDnsNameOptionsOnLaunch{
+ HostnameType: aws.String("ip-name"),
+ EnableResourceNameDnsARecord: aws.Bool(false),
+ EnableResourceNameDnsAAAARecord: aws.Bool(false),
+ }, subnet.PrivateDnsNameOptionsOnLaunch, name)
+ }
+}
+
func TestCreateSubnet_MissingVpcId(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -324,7 +363,15 @@ func TestCreateSubnet_InvalidCidr(t *testing.T) {
VpcId: aws.String(vpcID),
CidrBlock: aws.String("not-a-cidr"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidSubnet.Range")
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+ "Value (not-a-cidr) for parameter cidrBlock is invalid. This is not a valid CIDR block.")
+
+ // AWS rejects the CIDR before it looks the VPC up.
+ _, err = svc.CreateSubnet(context.Background(), &ec2.CreateSubnetInput{
+ VpcId: aws.String("vpc-nonexistent"),
+ CidrBlock: aws.String("not-a-cidr"),
+ }, testAccountID)
+ requireAWSCode(t, err, awserrors.ErrorInvalidParameterValue)
}
func TestCreateSubnet_OutsideVpcCidr(t *testing.T) {
@@ -335,7 +382,7 @@ func TestCreateSubnet_OutsideVpcCidr(t *testing.T) {
VpcId: aws.String(vpcID),
CidrBlock: aws.String("192.168.1.0/24"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidSubnet.Range")
+ requireAWSError(t, err, awserrors.ErrorInvalidSubnetRange, "The CIDR '192.168.1.0/24' is invalid.")
}
func TestCreateSubnet_ConflictingCidr(t *testing.T) {
@@ -349,7 +396,7 @@ func TestCreateSubnet_ConflictingCidr(t *testing.T) {
VpcId: aws.String(vpcID),
CidrBlock: aws.String("10.0.1.0/25"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidSubnet.Conflict")
+ requireAWSError(t, err, awserrors.ErrorInvalidSubnetConflict, "The CIDR '10.0.1.0/25' conflicts with another subnet")
}
func TestCreateSubnet_WithTags(t *testing.T) {
@@ -473,9 +520,24 @@ func TestDescribeSubnets_NotFound(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
_, err := svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
- SubnetIds: []*string{aws.String("subnet-nonexistent")},
+ SubnetIds: []*string{aws.String("subnet-missing1"), aws.String("subnet-missing2")},
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidSubnetID.NotFound")
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidSubnetIDNotFound, code)
+ assert.Equal(t, "The subnet ID 'subnet-missing1' does not exist", msg)
+}
+
+func TestDeleteSubnet_NotFound(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ _, err := svc.DeleteSubnet(context.Background(), &ec2.DeleteSubnetInput{
+ SubnetId: aws.String("subnet-missing1"),
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidSubnetIDNotFound, code)
+ assert.Equal(t, "The subnet ID 'subnet-missing1' does not exist", msg)
}
func TestCreateMultipleSubnetsInVpc(t *testing.T) {
@@ -795,8 +857,7 @@ func TestDeleteVpc_RejectsAttachedInternetGateway(t *testing.T) {
_, err = svc.DeleteVpc(context.Background(), &ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}, testAccountID)
require.Error(t, err)
- assert.ErrorContains(t, err, "DependencyViolation")
- assert.ErrorContains(t, err, "igw-pending", "the caller must be told which gateway to detach")
+ requireAWSError(t, err, awserrors.ErrorDependencyViolation, "The vpc '"+vpcID+"' has dependencies and cannot be deleted.")
desc, err := svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{VpcIds: []*string{aws.String(vpcID)}}, testAccountID)
require.NoError(t, err)
@@ -1035,7 +1096,7 @@ func TestCreateSubnet_CidrTooSmall(t *testing.T) {
VpcId: aws.String(vpcID),
CidrBlock: aws.String("10.0.0.0/29"),
}, testAccountID)
- assert.ErrorContains(t, err, "InvalidSubnet.Range")
+ requireAWSError(t, err, awserrors.ErrorInvalidSubnetRange, "The CIDR '10.0.0.0/29' is invalid.")
}
func TestVpcCidrBlockAssociation(t *testing.T) {
@@ -1882,6 +1943,36 @@ func TestDescribeVpcs_FilterByTag(t *testing.T) {
assert.Equal(t, "10.0.0.0/16", *desc.Vpcs[0].CidrBlock)
}
+func TestDescribeVpcs_FilterByTagKeyAndTagValue(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+
+ out, err := svc.CreateVpc(context.Background(), &ec2.CreateVpcInput{
+ CidrBlock: aws.String("10.0.0.0/16"),
+ TagSpecifications: []*ec2.TagSpecification{{
+ ResourceType: aws.String("vpc"),
+ Tags: []*ec2.Tag{{Key: aws.String("awsdiff"), Value: aws.String("yes")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestVPC(t, svc, "172.16.0.0/16")
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "awsdiff"}, {"tag-value", "yes"}} {
+ desc, err := svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.Vpcs, 1, tc.name)
+ assert.Equal(t, *out.Vpc.VpcId, *desc.Vpcs[0].VpcId, tc.name)
+
+ desc, err = svc.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.Vpcs, tc.name)
+ }
+}
+
func TestDescribeVpcs_FilterByVpcId(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -2132,6 +2223,38 @@ func TestDescribeSubnets_FilterByTag(t *testing.T) {
assert.Equal(t, *out.Subnet.SubnetId, *desc.Subnets[0].SubnetId)
}
+func TestDescribeSubnets_FilterByTagKeyAndTagValue(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.0.0.0/16")
+
+ out, err := svc.CreateSubnet(context.Background(), &ec2.CreateSubnetInput{
+ VpcId: aws.String(vpcID),
+ CidrBlock: aws.String("10.0.1.0/24"),
+ TagSpecifications: []*ec2.TagSpecification{{
+ ResourceType: aws.String("subnet"),
+ Tags: []*ec2.Tag{{Key: aws.String("Env"), Value: aws.String("prod")}},
+ }},
+ }, testAccountID)
+ require.NoError(t, err)
+ createTestSubnet(t, svc, vpcID, "10.0.2.0/24")
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Env"}, {"tag-value", "prod"}} {
+ desc, err := svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, desc.Subnets, 1, tc.name)
+ assert.Equal(t, *out.Subnet.SubnetId, *desc.Subnets[0].SubnetId, tc.name)
+
+ desc, err = svc.DescribeSubnets(context.Background(), &ec2.DescribeSubnetsInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, desc.Subnets, tc.name)
+ }
+}
+
// --- SetExternalIPAM / GetSubnet ---
func TestGetSubnet_Success(t *testing.T) {
@@ -2322,6 +2445,35 @@ func TestApplyRecordTags_ENITagFilteredDescribe(t *testing.T) {
assert.Equal(t, "primary", findTag(out.NetworkInterfaces[0].TagSet, "Name"))
}
+func TestDescribeNetworkInterfaces_FilterByTagKeyAndTagValue(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+ vpcID := createTestVPC(t, svc, "10.5.0.0/16")
+ subnetID := createTestSubnet(t, svc, vpcID, "10.5.1.0/24")
+ eniID := createTestENI(t, svc, subnetID)
+ createTestENI(t, svc, subnetID) // untagged
+
+ require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{
+ Resources: []*string{aws.String(eniID)},
+ Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("primary")}},
+ }, testAccountID))
+
+ for _, tc := range []struct{ name, value string }{{"tag-key", "Name"}, {"tag-value", "primary"}} {
+ out, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String(tc.value)}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ require.Len(t, out.NetworkInterfaces, 1, tc.name)
+ assert.Equal(t, eniID, *out.NetworkInterfaces[0].NetworkInterfaceId, tc.name)
+
+ out, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+ Filters: []*ec2.Filter{{Name: aws.String(tc.name), Values: []*string{aws.String("zz-awsdiff-none")}}},
+ }, testAccountID)
+ require.NoError(t, err, tc.name)
+ assert.Empty(t, out.NetworkInterfaces, tc.name)
+ }
+}
+
func TestApplyRecordTags_UnknownResourceNoError(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
@@ -2341,3 +2493,22 @@ func findTag(tags []*ec2.Tag, key string) string {
}
return ""
}
+
+// AWS checks the attribute combination before it looks the VPC up, and its
+// order of names is unstable, so only the names are asserted.
+func TestVpc_ModifyVpcAttribute_MultipleAttributes(t *testing.T) {
+ t.Parallel()
+ svc := setupTestVPCService(t)
+
+ _, err := svc.ModifyVpcAttribute(context.Background(), &ec2.ModifyVpcAttributeInput{
+ VpcId: aws.String("vpc-nonexistent"),
+ EnableDnsSupport: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+ EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+ }, testAccountID)
+ code, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, awserrors.ErrorInvalidParameterCombination, code)
+ assert.Contains(t, msg, "Fields for multiple attribute types specified: ")
+ assert.Contains(t, msg, "enableDnsSupport")
+ assert.Contains(t, msg, "enableDnsHostnames")
+}
diff --git a/spinifex/handlers/ecs/tags.go b/spinifex/handlers/ecs/tags.go
index 397db4f75..28dacc1f0 100644
--- a/spinifex/handlers/ecs/tags.go
+++ b/spinifex/handlers/ecs/tags.go
@@ -8,6 +8,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ecs"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/nats-io/nats.go/jetstream"
)
@@ -37,22 +38,22 @@ func ResourceARNSegment(resourceARN string) (string, error) {
// (.../cluster/{name}, .../task-definition/{family}:{rev});
// service/task/container-instance ARNs embed the owning cluster.
func parseResourceARN(resourceARN string) (kind ecsResourceKind, cluster, id, segment string, err error) {
- // SplitN(..., 6) keeps the resource segment intact even though a
- // task-definition id ("family:1") itself contains a colon.
- parts := strings.SplitN(resourceARN, ":", 6)
- if len(parts) != 6 || parts[0] != "arn" || parts[2] != "ecs" {
+ // Split keeps the resource segment intact even though a task-definition
+ // id ("family:1") itself contains a colon.
+ service, _, _, segment, ok := arn.Split(resourceARN)
+ if !ok || service != "ecs" {
return 0, "", "", "", errors.New(awserrors.ErrorECSInvalidParameter)
}
- rtype, rest, ok := strings.Cut(parts[5], "/")
+ rtype, rest, ok := strings.Cut(segment, "/")
if !ok || rtype == "" || rest == "" {
return 0, "", "", "", errors.New(awserrors.ErrorECSInvalidParameter)
}
switch rtype {
case "cluster":
- return ecsResourceCluster, rest, rest, parts[5], nil
+ return ecsResourceCluster, rest, rest, segment, nil
case "task-definition":
- return ecsResourceTaskDefinition, "", rest, parts[5], nil
+ return ecsResourceTaskDefinition, "", rest, segment, nil
case "service", "task", "container-instance":
clusterName, name, ok := strings.Cut(rest, "/")
if !ok || clusterName == "" || name == "" {
@@ -60,11 +61,11 @@ func parseResourceARN(resourceARN string) (kind ecsResourceKind, cluster, id, se
}
switch rtype {
case "service":
- return ecsResourceService, clusterName, name, parts[5], nil
+ return ecsResourceService, clusterName, name, segment, nil
case "task":
- return ecsResourceTask, clusterName, name, parts[5], nil
+ return ecsResourceTask, clusterName, name, segment, nil
default:
- return ecsResourceContainerInstance, clusterName, name, parts[5], nil
+ return ecsResourceContainerInstance, clusterName, name, segment, nil
}
default:
return 0, "", "", "", errors.New(awserrors.ErrorECSInvalidParameter)
diff --git a/spinifex/handlers/eks/addon_catalog.go b/spinifex/handlers/eks/addon_catalog.go
index 29980b02b..90146a64e 100644
--- a/spinifex/handlers/eks/addon_catalog.go
+++ b/spinifex/handlers/eks/addon_catalog.go
@@ -2,6 +2,7 @@ package handlers_eks
import (
"fmt"
+ "maps"
"slices"
)
@@ -111,11 +112,7 @@ func (s AddonSpec) supportsVersion(version string) bool {
// catalogSpecs returns the catalog entries sorted by name for stable output.
func catalogSpecs() []AddonSpec {
- names := make([]string, 0, len(addonCatalog))
- for n := range addonCatalog {
- names = append(names, n)
- }
- slices.Sort(names)
+ names := slices.Sorted(maps.Keys(addonCatalog))
out := make([]AddonSpec, 0, len(names))
for _, n := range names {
out = append(out, addonCatalog[n])
diff --git a/spinifex/handlers/eks/addon_installer.go b/spinifex/handlers/eks/addon_installer.go
index 0063c3067..f0a323675 100644
--- a/spinifex/handlers/eks/addon_installer.go
+++ b/spinifex/handlers/eks/addon_installer.go
@@ -37,16 +37,15 @@ type StagedAddonManifest struct {
// The VM-side delivery slice applies it via the K3s auto-deploy dir; until then
// the add-on sits CREATING.
type stagingInstaller struct {
- nc *nats.Conn
- clusterSize int
+ nc *nats.Conn
}
var _ AddonInstaller = (*stagingInstaller)(nil)
// newStagingInstaller returns a stagingInstaller bound to the daemon NATS
-// connection, using clusterSize as the per-account bucket's replica count.
-func newStagingInstaller(nc *nats.Conn, clusterSize int) *stagingInstaller {
- return &stagingInstaller{nc: nc, clusterSize: clusterSize}
+// connection.
+func newStagingInstaller(nc *nats.Conn) *stagingInstaller {
+ return &stagingInstaller{nc: nc}
}
func (i *stagingInstaller) acctKV(ctx context.Context, accountID string) (jetstream.KeyValue, error) {
@@ -57,7 +56,7 @@ func (i *stagingInstaller) acctKV(ctx context.Context, accountID string) (jetstr
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- return GetOrCreateAccountBucket(ctx, js, accountID, max(i.clusterSize, 1))
+ return GetOrCreateAccountBucket(ctx, js, accountID)
}
func (i *stagingInstaller) Install(ctx context.Context, accountID, cluster string, rec *AddonRecord) error {
diff --git a/spinifex/handlers/eks/addon_status_reconciler_test.go b/spinifex/handlers/eks/addon_status_reconciler_test.go
index d199879e2..c8abfe148 100644
--- a/spinifex/handlers/eks/addon_status_reconciler_test.go
+++ b/spinifex/handlers/eks/addon_status_reconciler_test.go
@@ -15,7 +15,7 @@ import (
func acctKVForTest(t *testing.T, svc *EKSServiceImpl) jetstream.KeyValue {
t.Helper()
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
return kv
}
diff --git a/spinifex/handlers/eks/addons.go b/spinifex/handlers/eks/addons.go
index 4bf3be4fd..b60d35c27 100644
--- a/spinifex/handlers/eks/addons.go
+++ b/spinifex/handlers/eks/addons.go
@@ -331,7 +331,7 @@ func (s *EKSServiceImpl) addonInstaller() AddonInstaller {
if s.deps.AddonInstaller != nil {
return s.deps.AddonInstaller
}
- return newStagingInstaller(s.deps.NATSConn, s.deps.ClusterSize)
+ return newStagingInstaller(s.deps.NATSConn)
}
// markAddonFailed best-effort flips a record to CREATE_FAILED with the error reason.
diff --git a/spinifex/handlers/eks/addons_test.go b/spinifex/handlers/eks/addons_test.go
index 9ca5f9ba3..076158413 100644
--- a/spinifex/handlers/eks/addons_test.go
+++ b/spinifex/handlers/eks/addons_test.go
@@ -243,7 +243,7 @@ func TestStagingInstaller_StagesManifest(t *testing.T) {
require.NoError(t, err)
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
entry, err := kv.Get(t.Context(), AddonManifestKey("c1", albController))
require.NoError(t, err, "installer must stage a manifest for VM-side delivery")
diff --git a/spinifex/handlers/eks/cluster_reconciler_state_test.go b/spinifex/handlers/eks/cluster_reconciler_state_test.go
index ce85357e9..781bf5af7 100644
--- a/spinifex/handlers/eks/cluster_reconciler_state_test.go
+++ b/spinifex/handlers/eks/cluster_reconciler_state_test.go
@@ -21,9 +21,9 @@ func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*Cluster
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- leaderKV, err := InitLeaderBucket(t.Context(), js, 1)
+ leaderKV, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
diff --git a/spinifex/handlers/eks/cluster_reconciler_test.go b/spinifex/handlers/eks/cluster_reconciler_test.go
index 9faf17c20..29f12c8e9 100644
--- a/spinifex/handlers/eks/cluster_reconciler_test.go
+++ b/spinifex/handlers/eks/cluster_reconciler_test.go
@@ -58,9 +58,9 @@ func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOpti
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- leaderKV, err := InitLeaderBucket(t.Context(), js, 1)
+ leaderKV, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
@@ -72,9 +72,9 @@ func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOpti
func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- leaderKV, err := InitLeaderBucket(t.Context(), js, 1)
+ leaderKV, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "")
diff --git a/spinifex/handlers/eks/cluster_state_test.go b/spinifex/handlers/eks/cluster_state_test.go
index 730af7e8e..f23da704c 100644
--- a/spinifex/handlers/eks/cluster_state_test.go
+++ b/spinifex/handlers/eks/cluster_state_test.go
@@ -15,7 +15,7 @@ func newClusterStateTestKV(t *testing.T) jetstream.KeyValue {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
return kv
}
diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go
index 7dc136842..cd7b6a42b 100644
--- a/spinifex/handlers/eks/delete_cluster_test.go
+++ b/spinifex/handlers/eks/delete_cluster_test.go
@@ -60,7 +60,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
nlb := newFakeNLBProvisioner()
diff --git a/spinifex/handlers/eks/dns_reconcile_test.go b/spinifex/handlers/eks/dns_reconcile_test.go
index bbc0b5805..05e3b6c32 100644
--- a/spinifex/handlers/eks/dns_reconcile_test.go
+++ b/spinifex/handlers/eks/dns_reconcile_test.go
@@ -62,7 +62,7 @@ func TestDesiredDNSChanges_MetadataReadFailureIsNotAuthoritative(t *testing.T) {
require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, active))
js := testutil.NewJetStream(t, fixture.svc.deps.NATSConn)
- corruptKV, err := GetOrCreateAccountBucket(t.Context(), js, "444455556666", 1)
+ corruptKV, err := GetOrCreateAccountBucket(t.Context(), js, "444455556666")
require.NoError(t, err)
_, err = corruptKV.Put(t.Context(), ClusterMetaKey("unreadable"), []byte("{not json"))
require.NoError(t, err)
diff --git a/spinifex/handlers/eks/eks_billable_reaper.go b/spinifex/handlers/eks/eks_billable_reaper.go
index 21528da82..4198d247a 100644
--- a/spinifex/handlers/eks/eks_billable_reaper.go
+++ b/spinifex/handlers/eks/eks_billable_reaper.go
@@ -69,7 +69,7 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount, max(r.svc.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
if err != nil {
slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
continue
diff --git a/spinifex/handlers/eks/nats_bootstrap_test.go b/spinifex/handlers/eks/nats_bootstrap_test.go
index 5efe2658b..2c32e39b9 100644
--- a/spinifex/handlers/eks/nats_bootstrap_test.go
+++ b/spinifex/handlers/eks/nats_bootstrap_test.go
@@ -39,7 +39,7 @@ func newBootstrapHarness(t *testing.T) *natsBootstrapHarness {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta(bootstrapTestCluster)))
@@ -85,7 +85,7 @@ func (h *natsBootstrapHarness) waitSubsBound(base, want int) {
func TestNewNATSBootstrap_RejectsBadInputs(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
_, err = NewNATSBootstrap(nil, kv, bootstrapTestMasterKey, testAccountID, "alpha")
diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go
index 98a9d533c..0aa5844d9 100644
--- a/spinifex/handlers/eks/nodegroup.go
+++ b/spinifex/handlers/eks/nodegroup.go
@@ -195,7 +195,7 @@ func (s *EKSServiceImpl) nodegroupAcctKV(ctx context.Context, accountID string)
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, fmt.Errorf("get account bucket: %w", err)
}
diff --git a/spinifex/handlers/eks/nodegroup_test.go b/spinifex/handlers/eks/nodegroup_test.go
index 23c926560..54b4acf86 100644
--- a/spinifex/handlers/eks/nodegroup_test.go
+++ b/spinifex/handlers/eks/nodegroup_test.go
@@ -242,7 +242,7 @@ func TestReclaimOrphanedNodegroups_TerminatesStrandedWorkers(t *testing.T) {
func TestNodegroupRecord_CRUDRoundTrip(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
rec := &NodegroupRecord{
diff --git a/spinifex/handlers/eks/recovery_directive_test.go b/spinifex/handlers/eks/recovery_directive_test.go
index 78d64ded4..e868d4481 100644
--- a/spinifex/handlers/eks/recovery_directive_test.go
+++ b/spinifex/handlers/eks/recovery_directive_test.go
@@ -13,7 +13,7 @@ import (
func TestLoadRecoveryDirective_AbsentIsZeroNone(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
d, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-missing")
@@ -25,7 +25,7 @@ func TestLoadRecoveryDirective_AbsentIsZeroNone(t *testing.T) {
func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
first, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-0", RecoveryActionClusterReset, "", false)
@@ -46,7 +46,7 @@ func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) {
func TestStoreRecoveryDirective_SnapshotRequiredRoundTrips(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
stored, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed", RecoveryActionClusterReset, "etcd-frequent-20260709T010000Z.snap", true)
diff --git a/spinifex/handlers/eks/security_groups.go b/spinifex/handlers/eks/security_groups.go
index 1a28dbab8..86ec2da0d 100644
--- a/spinifex/handlers/eks/security_groups.go
+++ b/spinifex/handlers/eks/security_groups.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "slices"
"time"
"github.com/aws/aws-sdk-go/aws"
@@ -479,12 +480,9 @@ func EnsureControlPlaneHAIngress(ctx context.Context, sgp sgProvisioner, account
// sgHasLBCClusterTag reports whether the SG carries the AWS LoadBalancer
// Controller ownership tag for this cluster.
func sgHasLBCClusterTag(g *ec2.SecurityGroup, clusterName string) bool {
- for _, tag := range g.Tags {
- if tag != nil && aws.StringValue(tag.Key) == lbcClusterOwnershipTagKey && aws.StringValue(tag.Value) == clusterName {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(g.Tags, func(tag *ec2.Tag) bool {
+ return tag != nil && aws.StringValue(tag.Key) == lbcClusterOwnershipTagKey && aws.StringValue(tag.Value) == clusterName
+ })
}
func lookupSGByName(ctx context.Context, sgp sgProvisioner, accountID, vpcID, name string) (string, error) {
diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go
index 108ec2a96..024cba137 100644
--- a/spinifex/handlers/eks/service_impl.go
+++ b/spinifex/handlers/eks/service_impl.go
@@ -19,6 +19,7 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/eks"
"github.com/aws/aws-sdk-go/service/iam"
+ "github.com/mulgadc/bluebottle/pkg/auth"
"github.com/mulgadc/spinifex/spinifex/admin"
resourcearn "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -50,11 +51,6 @@ type EKSServiceDeps struct {
Region string
HolderID string
- // ClusterSize is the daemon's node count, used as the JetStream replica
- // count for the lazily-created per-account and leader KV buckets so they
- // match the cluster's other R3 streams instead of staying stuck at R1.
- ClusterSize int
-
// InternalSuffix is the AWS-parity internal DNS suffix (e.g. spinifex.internal)
// used to compose the worker's ECR registry host.
InternalSuffix string
@@ -305,7 +301,7 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
// The leader bucket outlives every request, so its open is bounded by the
// service's own background context rather than a caller's.
ctx, cancel := context.WithCancel(context.Background())
- leaderKV, err := InitLeaderBucket(ctx, js, max(deps.ClusterSize, 1))
+ leaderKV, err := InitLeaderBucket(ctx, js)
if err != nil {
cancel()
return nil, err
@@ -509,7 +505,7 @@ func (s *EKSServiceImpl) CreateCluster(ctx context.Context, input *eks.CreateClu
if err := s.requireOrchestrationDeps("CreateCluster"); err != nil {
return nil, err
}
- if err := validateCreateClusterInput(input); err != nil {
+ if err := validateCreateClusterInput(input, accountID); err != nil {
return nil, err
}
name := aws.StringValue(input.Name)
@@ -519,7 +515,7 @@ func (s *EKSServiceImpl) CreateCluster(ctx context.Context, input *eks.CreateClu
if err != nil {
return nil, logCreateErr(name, accountID, "jetstream", err)
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, logCreateErr(name, accountID, "get account bucket", err)
}
@@ -1051,7 +1047,7 @@ func (s *EKSServiceImpl) DescribeCluster(ctx context.Context, input *eks.Describ
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, fmt.Errorf("get account bucket: %w", err)
}
@@ -1070,7 +1066,7 @@ func (s *EKSServiceImpl) ListClusters(ctx context.Context, input *eks.ListCluste
if err != nil {
return nil, eksReadUnavailableOr(err, "jetstream")
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, eksReadUnavailableOr(err, "get account bucket")
}
@@ -1117,7 +1113,7 @@ func (s *EKSServiceImpl) DeleteCluster(ctx context.Context, input *eks.DeleteClu
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, fmt.Errorf("get account bucket: %w", err)
}
@@ -1536,7 +1532,7 @@ func (s *EKSServiceImpl) acctKVForCluster(ctx context.Context, accountID, cluste
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID)
if err != nil {
return nil, fmt.Errorf("get account bucket: %w", err)
}
@@ -1801,12 +1797,9 @@ func (s *EKSServiceImpl) ListAccessPolicies(ctx context.Context, _ *eks.ListAcce
// hasAssociatedPolicy reports whether the entry has the given policy ARN bound.
func hasAssociatedPolicy(rec *AccessEntryRecord, policyARN string) bool {
- for _, p := range rec.AssociatedPolicies {
- if p.PolicyARN == policyARN {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(rec.AssociatedPolicies, func(p AssociatedAccessPolicy) bool {
+ return p.PolicyARN == policyARN
+ })
}
// accessPolicyName extracts the policy short name from its ARN
@@ -2002,22 +1995,18 @@ func (s *EKSServiceImpl) accountBucket(ctx context.Context, accountID string) (j
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- return GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1))
+ return GetOrCreateAccountBucket(ctx, js, accountID)
}
// clusterNameFromARN extracts the cluster name from an EKS cluster ARN
// (arn:aws:eks:::cluster/), reporting false for any other
// ARN shape (e.g. nodegroup) the tag store does not back.
func clusterNameFromARN(arn string) (string, bool) {
- const prefix = "arn:aws:eks:"
- if !strings.HasPrefix(arn, prefix) {
+ service, _, _, resource, ok := resourcearn.Split(arn)
+ if !ok || service != "eks" {
return "", false
}
- parts := strings.SplitN(arn, ":", 6)
- if len(parts) != 6 {
- return "", false
- }
- resType, name, found := strings.Cut(parts[5], "/")
+ resType, name, found := strings.Cut(resource, "/")
if !found || resType != "cluster" || name == "" {
return "", false
}
@@ -2028,15 +2017,11 @@ func clusterNameFromARN(arn string) (string, bool) {
// (arn:aws:eks:::nodegroup///), reporting false
// for any other ARN shape.
func nodegroupRefFromARN(arn string) (string, string, bool) {
- const prefix = "arn:aws:eks:"
- if !strings.HasPrefix(arn, prefix) {
+ service, _, _, resource, ok := resourcearn.Split(arn)
+ if !ok || service != "eks" {
return "", "", false
}
- parts := strings.SplitN(arn, ":", 6)
- if len(parts) != 6 {
- return "", "", false
- }
- resType, rest, found := strings.Cut(parts[5], "/")
+ resType, rest, found := strings.Cut(resource, "/")
if !found || resType != "nodegroup" {
return "", "", false
}
@@ -2060,11 +2045,12 @@ func eksTagErr(err error) error {
func notImpl() error { return errors.New(awserrors.ErrorNotImplemented) }
-func validateCreateClusterInput(input *eks.CreateClusterInput) error {
+func validateCreateClusterInput(input *eks.CreateClusterInput, accountID string) error {
if input == nil || input.Name == nil || *input.Name == "" {
return errors.New(awserrors.ErrorInvalidParameterValue)
}
- if input.RoleArn == nil || !strings.HasPrefix(*input.RoleArn, "arn:aws:iam:") {
+ // The cluster role must be a role in the caller's own account.
+ if roleAccount, _, err := auth.ParseRoleARN(aws.StringValue(input.RoleArn)); err != nil || roleAccount != accountID {
return errors.New(awserrors.ErrorInvalidParameterValue)
}
if input.ResourcesVpcConfig == nil || len(input.ResourcesVpcConfig.SubnetIds) == 0 {
@@ -2232,7 +2218,7 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *Clust
slog.Error("spawnReconciler: jetstream", "err", err)
return
}
- acctKV, err := GetOrCreateAccountBucket(s.bgCtx, js, accountID, max(s.deps.ClusterSize, 1))
+ acctKV, err := GetOrCreateAccountBucket(s.bgCtx, js, accountID)
if err != nil {
slog.Error("spawnReconciler: account bucket", "err", err)
return
diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go
index cf333e008..956f193a4 100644
--- a/spinifex/handlers/eks/service_impl_test.go
+++ b/spinifex/handlers/eks/service_impl_test.go
@@ -90,7 +90,7 @@ func TestValidateCreateClusterInput_RejectsConfigMapAuthMode(t *testing.T) {
in.AccessConfig = &eks.CreateAccessConfigRequest{
AuthenticationMode: aws.String(eks.AuthenticationModeConfigMap),
}
- err := validateCreateClusterInput(in)
+ err := validateCreateClusterInput(in, testAccountID)
code, message, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorInvalidParameter, code)
@@ -104,7 +104,7 @@ func TestValidateCreateClusterInput_RejectsJunkAuthMode(t *testing.T) {
in.AccessConfig = &eks.CreateAccessConfigRequest{
AuthenticationMode: aws.String("NOT_A_REAL_MODE"),
}
- err := validateCreateClusterInput(in)
+ err := validateCreateClusterInput(in, testAccountID)
code, message, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorInvalidParameter, code)
@@ -119,7 +119,7 @@ func TestValidateCreateClusterInput_AcceptsAPIAndConfigMapAuthMode(t *testing.T)
in.AccessConfig = &eks.CreateAccessConfigRequest{
AuthenticationMode: aws.String(eks.AuthenticationModeApiAndConfigMap),
}
- require.NoError(t, validateCreateClusterInput(in))
+ require.NoError(t, validateCreateClusterInput(in, testAccountID))
}
func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) {
@@ -127,17 +127,17 @@ func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) {
in.AccessConfig = &eks.CreateAccessConfigRequest{
AuthenticationMode: aws.String(eks.AuthenticationModeApi),
}
- require.NoError(t, validateCreateClusterInput(in))
+ require.NoError(t, validateCreateClusterInput(in, testAccountID))
}
// An unset accessConfig (or an unset authenticationMode within it) must stay
// valid — it defaults to API, exactly as an explicit "API" does.
func TestValidateCreateClusterInput_AcceptsUnsetAuthMode(t *testing.T) {
in := createInput("alpha")
- require.NoError(t, validateCreateClusterInput(in))
+ require.NoError(t, validateCreateClusterInput(in, testAccountID))
in.AccessConfig = &eks.CreateAccessConfigRequest{}
- require.NoError(t, validateCreateClusterInput(in))
+ require.NoError(t, validateCreateClusterInput(in, testAccountID))
}
// A create request with no subnets is malformed and must be rejected before any
@@ -145,10 +145,10 @@ func TestValidateCreateClusterInput_AcceptsUnsetAuthMode(t *testing.T) {
func TestValidateCreateClusterInput_RejectsMissingSubnetIds(t *testing.T) {
in := createInput("alpha")
in.ResourcesVpcConfig = &eks.VpcConfigRequest{}
- require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameterValue)
+ require.EqualError(t, validateCreateClusterInput(in, testAccountID), awserrors.ErrorInvalidParameterValue)
in.ResourcesVpcConfig = nil
- require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameterValue)
+ require.EqualError(t, validateCreateClusterInput(in, testAccountID), awserrors.ErrorInvalidParameterValue)
}
// DescribeCluster on an absent cluster must reach the KV lookup (full deps
@@ -223,7 +223,7 @@ func TestEKSServiceImpl_NodegroupMethodsShimMode(t *testing.T) {
func seedTestCluster(t *testing.T, svc *EKSServiceImpl, cluster string) {
t.Helper()
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NoError(t, PutClusterMeta(t.Context(), kv, &ClusterMeta{Name: cluster, Status: ClusterStatusActive}))
}
@@ -414,7 +414,7 @@ func TestAccessPolicy_DisassociateRemovesProjectedGroup(t *testing.T) {
require.NoError(t, err)
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
rec, err := GetAccessEntryRecord(t.Context(), kv, "c1", testPrincipalARN)
require.NoError(t, err)
@@ -461,7 +461,7 @@ func TestEKSServiceImpl_OIDCMethodsReturnNotImplemented(t *testing.T) {
func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) {
svc := setupTestService(t)
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
const arn = "arn:aws:eks:us-east-1:111122223333:cluster/c1"
@@ -503,7 +503,7 @@ func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) {
func TestEKSServiceImpl_NodegroupTagRoundTrip(t *testing.T) {
svc := setupTestService(t)
js := testutil.NewJetStream(t, svc.deps.NATSConn)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
const arn = "arn:aws:eks:us-east-1:111122223333:nodegroup/c1/ng1/abc123"
@@ -580,3 +580,16 @@ func TestEKSServiceImpl_TagsUnsupportedARNNotImplemented(t *testing.T) {
_, err = svc.ListTagsForResource(context.Background(), &eks.ListTagsForResourceInput{ResourceArn: aws.String(fpARN)}, testAccountID)
require.EqualError(t, err, awserrors.ErrorNotImplemented)
}
+
+// The cluster role must be a role in the caller's own account.
+func TestValidateCreateClusterInput_RejectsRoleOutsideCallerAccount(t *testing.T) {
+ for _, roleARN := range []string{
+ "arn:aws:iam::999999999999:role/eks-cluster",
+ "arn:aws:iam::" + testAccountID + ":user/eks-cluster",
+ "arn:aws:iam:",
+ } {
+ in := createInput("alpha")
+ in.RoleArn = aws.String(roleARN)
+ require.EqualError(t, validateCreateClusterInput(in, testAccountID), awserrors.ErrorInvalidParameterValue, roleARN)
+ }
+}
diff --git a/spinifex/handlers/eks/store.go b/spinifex/handlers/eks/store.go
index 9efcdf64d..c8e2c1291 100644
--- a/spinifex/handlers/eks/store.go
+++ b/spinifex/handlers/eks/store.go
@@ -232,9 +232,9 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
// creating it on first use at the given replica count (clamped to a minimum
// of 1). Idempotent: subsequent calls with the same accountID return the
// existing handle.
-func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) {
+func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
bucket := AccountBucketName(accountID)
- kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, bucket, KVBucketEKSAccountHistory, replicas)
+ kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketEKSAccountHistory)
if err != nil {
return nil, fmt.Errorf("failed to create EKS per-account KV bucket %s: %w", bucket, err)
}
@@ -245,13 +245,11 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
}
// InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
-// bucket used for per-cluster reconciler leader-lease CAS locks, at the given
-// replica count (clamped to a minimum of 1).
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream, replicas int) (jetstream.KeyValue, error) {
+// bucket used for per-cluster reconciler leader-lease CAS locks.
+func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
- Name: KVBucketEKSLeader,
- TTL: KVBucketEKSLeaderTTL,
- Replicas: max(replicas, 1),
- Version: KVBucketEKSLeaderVersion,
+ Name: KVBucketEKSLeader,
+ TTL: KVBucketEKSLeaderTTL,
+ Version: KVBucketEKSLeaderVersion,
})
}
diff --git a/spinifex/handlers/eks/store_test.go b/spinifex/handlers/eks/store_test.go
index 525663099..906bfc6ed 100644
--- a/spinifex/handlers/eks/store_test.go
+++ b/spinifex/handlers/eks/store_test.go
@@ -14,11 +14,11 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NotNil(t, kv1)
- kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1)
+ kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NotNil(t, kv2)
@@ -36,7 +36,7 @@ func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 0)
+ kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
require.NoError(t, err)
require.NotNil(t, kv)
@@ -51,12 +51,12 @@ func TestInitLeaderBucket_Idempotent(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv1, err := InitLeaderBucket(t.Context(), js, 1)
+ kv1, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
require.NotNil(t, kv1)
assert.Equal(t, KVBucketEKSLeader, kv1.Bucket())
- kv2, err := InitLeaderBucket(t.Context(), js, 1)
+ kv2, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
require.NotNil(t, kv2)
assert.Equal(t, KVBucketEKSLeader, kv2.Bucket())
@@ -68,7 +68,7 @@ func TestInitLeaderBucket_ReplicasClamped(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := InitLeaderBucket(t.Context(), js, -1)
+ kv, err := InitLeaderBucket(t.Context(), js)
require.NoError(t, err)
require.NotNil(t, kv)
diff --git a/spinifex/handlers/eks/watch_source_test.go b/spinifex/handlers/eks/watch_source_test.go
index 83f73c9cc..96662b2c7 100644
--- a/spinifex/handlers/eks/watch_source_test.go
+++ b/spinifex/handlers/eks/watch_source_test.go
@@ -18,9 +18,9 @@ func TestAccountWatchBuckets_ReturnsOnePerAccountBucket(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1)
+ _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
require.NoError(t, err)
- _, err = handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "222222222222", 1)
+ _, err = handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "222222222222")
require.NoError(t, err)
_, err = kvutil.GetOrCreateBucket(t.Context(), js, "something-else", 1)
require.NoError(t, err)
@@ -51,7 +51,7 @@ func TestAccountWatchBuckets_NoAccountsIsNotAnError(t *testing.T) {
func TestAccountWatchBuckets_BucketsAreWatchable(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1)
+ _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
require.NoError(t, err)
buckets, err := handlers_eks.AccountWatchBuckets(t.Context(), nc)
diff --git a/spinifex/handlers/elbv2/haproxy.go b/spinifex/handlers/elbv2/haproxy.go
index 10b469ae9..f5c11dad1 100644
--- a/spinifex/handlers/elbv2/haproxy.go
+++ b/spinifex/handlers/elbv2/haproxy.go
@@ -218,7 +218,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
if validHTTPStatusCode(fr.StatusCode) {
hf.StatusCode = fr.StatusCode
}
- if validContentType(fr.ContentType) {
+ if contentTypeRegex.MatchString(fr.ContentType) {
hf.ContentType = fr.ContentType
}
if validFixedResponseBody(fr.MessageBody) {
@@ -529,11 +529,6 @@ func validHTTPStatusCode(s string) bool {
return true
}
-// validContentType accepts a conservative type/subtype MIME string.
-func validContentType(s string) bool {
- return contentTypeRegex.MatchString(s)
-}
-
// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive.
func validFixedResponseBody(s string) bool {
if s == "" {
diff --git a/spinifex/handlers/iam/account_lifecycle_test.go b/spinifex/handlers/iam/account_lifecycle_test.go
index 3cbe5912a..26b118d54 100644
--- a/spinifex/handlers/iam/account_lifecycle_test.go
+++ b/spinifex/handlers/iam/account_lifecycle_test.go
@@ -16,7 +16,7 @@ func newLifecycleService(t *testing.T) handlers_iam.IAMService {
masterKey, err := handlers_iam.GenerateMasterKey()
require.NoError(t, err)
- svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
return svc
}
diff --git a/spinifex/handlers/iam/account_test.go b/spinifex/handlers/iam/account_test.go
index 5b4df9082..637eb1e5d 100644
--- a/spinifex/handlers/iam/account_test.go
+++ b/spinifex/handlers/iam/account_test.go
@@ -407,7 +407,7 @@ func TestCreateAccount_PublishesEvent(t *testing.T) {
masterKey, err := GenerateMasterKey()
require.NoError(t, err)
- svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
eventCh := make(chan *nats.Msg, 1)
diff --git a/spinifex/handlers/iam/groups.go b/spinifex/handlers/iam/groups.go
index 3945704d7..1e885b7dd 100644
--- a/spinifex/handlers/iam/groups.go
+++ b/spinifex/handlers/iam/groups.go
@@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/service/iam"
"github.com/nats-io/nats.go/jetstream"
+ iamarn "github.com/mulgadc/bluebottle/pkg/auth"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
@@ -311,7 +312,7 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr
kvKey := accountID + "." + groupName
// AWS-managed ARNs are stored opaquely (like roles); customer-managed ARNs must exist.
- if !isAWSManagedPolicyARN(policyARN) {
+ if !iamarn.IsAWSManagedPolicyARN(policyARN) {
if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
return nil, err
}
@@ -377,25 +378,7 @@ func (s *IAMServiceImpl) ListAttachedGroupPolicies(accountID string, input *iam.
return nil, err
}
- var attached []*iam.AttachedPolicy
- for _, arn := range group.AttachedPolicies {
- if isAWSManagedPolicyARN(arn) {
- attached = append(attached, &iam.AttachedPolicy{
- PolicyArn: aws.String(arn),
- PolicyName: aws.String(managedPolicyNameFromARN(arn)),
- })
- continue
- }
- policy, err := s.getPolicyByARN(ctx, accountID, arn)
- if err != nil {
- slog.Warn("ListAttachedGroupPolicies: policy not found for ARN", "arn", arn, "err", err)
- continue
- }
- attached = append(attached, &iam.AttachedPolicy{
- PolicyArn: aws.String(policy.ARN),
- PolicyName: aws.String(policy.PolicyName),
- })
- }
+ attached := s.attachedPolicies(ctx, accountID, group.AttachedPolicies, aws.StringValue(input.PathPrefix))
return &iam.ListAttachedGroupPoliciesOutput{
AttachedPolicies: attached,
diff --git a/spinifex/handlers/iam/groups_test.go b/spinifex/handlers/iam/groups_test.go
index 40b3836fb..c3e8f9128 100644
--- a/spinifex/handlers/iam/groups_test.go
+++ b/spinifex/handlers/iam/groups_test.go
@@ -83,7 +83,7 @@ func TestCreateGroup_InvalidName(t *testing.T) {
GroupName: aws.String(longName),
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
func TestCreateGroup_InvalidPath(t *testing.T) {
diff --git a/spinifex/handlers/iam/instance_profiles.go b/spinifex/handlers/iam/instance_profiles.go
index 36dfc5573..99e25cb11 100644
--- a/spinifex/handlers/iam/instance_profiles.go
+++ b/spinifex/handlers/iam/instance_profiles.go
@@ -40,6 +40,10 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea
}
}
+ if err := validateTags(input.Tags, exactKeys); err != nil {
+ return nil, err
+ }
+
profileID, err := generateIAMID("AIPA")
if err != nil {
return nil, fmt.Errorf("generate instance profile ID: %w", err)
@@ -261,25 +265,28 @@ func (s *IAMServiceImpl) ResolveInstanceProfile(accountID, nameOrARN string) (*I
return s.getInstanceProfile(ctx, accountID, nameOrARN)
}
- profileAccountID, profileName, err := parseInstanceProfileARN(nameOrARN)
- if err != nil {
+ var profile *InstanceProfile
+ _, _, err := auth.ResolveInstanceProfileARN(nameOrARN, func(profileAccountID, profileName string) (string, error) {
+ if profileAccountID != accountID {
+ return "", errors.New(awserrors.ErrorAccessDenied)
+ }
+ p, err := s.getInstanceProfile(ctx, accountID, profileName)
+ if err != nil {
+ return "", err
+ }
+ profile = p
+ return p.ARN, nil
+ })
+ switch {
+ case errors.Is(err, auth.ErrInvalidInstanceProfileARN):
+ return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileArnMalformed)
+ case errors.Is(err, auth.ErrInstanceProfileARNMismatch):
+ // A non-canonical spelling names no stored profile.
+ return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+ case err != nil:
return nil, err
}
- if profileAccountID != accountID {
- return nil, errors.New(awserrors.ErrorAccessDenied)
- }
- return s.getInstanceProfile(ctx, accountID, profileName)
-}
-
-// parseInstanceProfileARN extracts accountID and profile name from an IAM
-// instance-profile ARN, reporting every malformed shape as the one AWS error
-// code this API returns.
-func parseInstanceProfileARN(arnStr string) (accountID, name string, err error) {
- accountID, name, err = auth.ParseInstanceProfileARN(arnStr)
- if err != nil {
- return "", "", errors.New(awserrors.ErrorInvalidIamInstanceProfileArnMalformed)
- }
- return accountID, name, nil
+ return profile, nil
}
// TagInstanceProfile upserts tags on an instance profile under CAS, like the
@@ -287,13 +294,13 @@ func parseInstanceProfileARN(arnStr string) (accountID, name string, err error)
// RoleName and silently undo a concurrent role attach.
func (s *IAMServiceImpl) TagInstanceProfile(accountID string, input *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) {
ctx := context.Background()
- if err := validateTags(input.Tags); err != nil {
+ if err := validateTags(input.Tags, exactKeys); err != nil {
return nil, err
}
profileName := *input.InstanceProfileName
err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
- merged := mergeTags(p.Tags, input.Tags)
+ merged := mergeTags(p.Tags, input.Tags, exactKeys)
if len(merged) > maxTagsPerResource {
return false, errors.New(awserrors.ErrorIAMLimitExceeded)
}
@@ -311,10 +318,13 @@ func (s *IAMServiceImpl) TagInstanceProfile(accountID string, input *iam.TagInst
// UntagInstanceProfile removes the named tag keys from an instance profile;
// unknown keys are a no-op.
func (s *IAMServiceImpl) UntagInstanceProfile(accountID string, input *iam.UntagInstanceProfileInput) (*iam.UntagInstanceProfileOutput, error) {
+ if err := validateTagKeys(input.TagKeys); err != nil {
+ return nil, err
+ }
ctx := context.Background()
profileName := *input.InstanceProfileName
err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
- kept := removeTagKeys(p.Tags, input.TagKeys)
+ kept := removeTagKeys(p.Tags, input.TagKeys, exactKeys)
if len(kept) == len(p.Tags) {
return false, nil
}
diff --git a/spinifex/handlers/iam/instance_profiles_test.go b/spinifex/handlers/iam/instance_profiles_test.go
index 9a24e40e2..a12f17685 100644
--- a/spinifex/handlers/iam/instance_profiles_test.go
+++ b/spinifex/handlers/iam/instance_profiles_test.go
@@ -70,7 +70,7 @@ func TestCreateInstanceProfile_InvalidName(t *testing.T) {
InstanceProfileName: aws.String(longName),
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
func TestCreateInstanceProfile_InvalidPath(t *testing.T) {
@@ -686,6 +686,29 @@ func TestResolveInstanceProfile_ARNNotFound(t *testing.T) {
assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
}
+// The store is keyed by name alone, so an ARN that spells a different path
+// must not resolve to the profile its trailing name matches.
+func TestResolveInstanceProfile_NonCanonicalARNRejected(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ createTestInstanceProfile(t, svc, "flat-profile")
+ _, err := svc.CreateInstanceProfile(testAccountID, &iam.CreateInstanceProfileInput{
+ InstanceProfileName: aws.String("pathed-profile"),
+ Path: aws.String("/team/"),
+ })
+ require.NoError(t, err)
+
+ for _, arn := range []string{
+ "arn:aws:iam::" + testAccountID + ":instance-profile/decoy/flat-profile",
+ "arn:aws:iam::" + testAccountID + ":instance-profile/pathed-profile",
+ } {
+ _, err := svc.ResolveInstanceProfile(testAccountID, arn)
+ require.Error(t, err, arn)
+ // The EC2 gateway maps this to InvalidIamInstanceProfile.NotFound by exact match.
+ assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), arn)
+ }
+}
+
func TestResolveInstanceProfile_MalformedARN(t *testing.T) {
t.Parallel()
svc := setupTestIAMService(t)
diff --git a/spinifex/handlers/iam/managed_policies.go b/spinifex/handlers/iam/managed_policies.go
index 0eeee068c..84ddc0bb8 100644
--- a/spinifex/handlers/iam/managed_policies.go
+++ b/spinifex/handlers/iam/managed_policies.go
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
+
+ iamarn "github.com/mulgadc/bluebottle/pkg/auth"
)
// awsManagedPolicyDocs maps well-known AWS-managed policy ARNs to the grant
@@ -311,7 +313,7 @@ func builtinManagedPolicyDoc(arn string) (PolicyDocument, bool) {
// failing the whole request. Customer-managed ARNs are fetched from KV and
// fail closed (error) when unresolvable.
func (s *IAMServiceImpl) resolveAttachedPolicy(ctx context.Context, accountID, arn string) (doc PolicyDocument, include bool, err error) {
- if isAWSManagedPolicyARN(arn) {
+ if iamarn.IsAWSManagedPolicyARN(arn) {
if d, ok := builtinManagedPolicyDoc(arn); ok {
return d, true, nil
}
diff --git a/spinifex/handlers/iam/managed_policies_test.go b/spinifex/handlers/iam/managed_policies_test.go
index 96c2d728b..eb2871d96 100644
--- a/spinifex/handlers/iam/managed_policies_test.go
+++ b/spinifex/handlers/iam/managed_policies_test.go
@@ -67,7 +67,7 @@ func TestNewIAMServiceImpl_PropagatesBuiltinManagedPolicyParseError(t *testing.T
builtinManagedPolicyParseErr = parseErr
t.Cleanup(func() { builtinManagedPolicyParseErr = previousErr })
- _, err := NewIAMServiceImpl(t.Context(), nil, make([]byte, 32), 1)
+ _, err := NewIAMServiceImpl(t.Context(), nil, make([]byte, 32))
require.ErrorIs(t, err, parseErr)
assert.ErrorContains(t, err, "init builtin managed policies")
}
diff --git a/spinifex/handlers/iam/oidc_providers.go b/spinifex/handlers/iam/oidc_providers.go
index 9ba46a041..8ba60ee9b 100644
--- a/spinifex/handlers/iam/oidc_providers.go
+++ b/spinifex/handlers/iam/oidc_providers.go
@@ -9,8 +9,10 @@ import (
"fmt"
"log/slog"
"net/url"
+ "slices"
"strings"
"time"
+ "unicode/utf8"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
@@ -49,8 +51,8 @@ func OIDCProviderARN(accountID, issuerHostPath string) string {
}
// GetOrCreateIAMAccountBucket opens the per-account IAM bucket, creating it on first use.
-func GetOrCreateIAMAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) {
- return kvutil.GetOrCreateBucketWithReplicas(ctx, js, IAMAccountBucketName(accountID), 1, max(replicas, 1))
+func GetOrCreateIAMAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
+ return kvutil.GetOrCreateBucket(ctx, js, IAMAccountBucketName(accountID), 1)
}
// OIDCProviderRecord is the stored shape of a registered OIDC identity provider.
@@ -120,14 +122,43 @@ func awsStrings(in []*string) []string {
return out
}
+// validateOIDCProviderLists checks every client ID and thumbprint against the
+// model's lengths, before the Url as AWS orders them. AWS names each list once,
+// however many of its entries break.
+func validateOIDCProviderLists(clientIDs, thumbprints []*string) error {
+ outside := func(minLen, maxLen int) func(*string) bool {
+ return func(v *string) bool {
+ n := utf8.RuneCountInString(aws.StringValue(v))
+ return v == nil || n < minLen || n > maxLen
+ }
+ }
+ var violations []string
+ if slices.ContainsFunc(clientIDs, outside(1, 255)) {
+ violations = append(violations, "Value at 'clientIDList' failed to satisfy constraint: Member must satisfy constraint: "+
+ "[Member must have length less than or equal to 255, Member must have length greater than or equal to 1]")
+ }
+ if slices.ContainsFunc(thumbprints, outside(40, 40)) {
+ violations = append(violations, "Value at 'thumbprintList' failed to satisfy constraint: Member must satisfy constraint: "+
+ "[Member must have length less than or equal to 40, Member must have length greater than or equal to 40]")
+ }
+ return validationError(violations)
+}
+
func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) {
ctx := context.Background()
+ if err := validateOIDCProviderLists(input.ClientIDList, input.ThumbprintList); err != nil {
+ return nil, err
+ }
issuer := aws.StringValue(input.Url)
if err := validateOIDCProviderURL(issuer); err != nil {
slog.Debug("CreateOpenIDConnectProvider: invalid Url", "url", issuer, "err", err)
return nil, awserrors.Errorf(awserrors.ErrorIAMInvalidInput, "The specified value for url is invalid: %v", err)
}
+ if err := validateTags(input.Tags, exactKeys); err != nil {
+ return nil, err
+ }
+
record := OIDCProviderRecord{
Url: issuer,
ClientIDList: awsStrings(input.ClientIDList),
@@ -140,7 +171,7 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia
return nil, fmt.Errorf("marshal OIDC provider: %w", err)
}
- kv, err := GetOrCreateIAMAccountBucket(ctx, s.js, accountID, s.replicas)
+ kv, err := GetOrCreateIAMAccountBucket(ctx, s.js, accountID)
if err != nil {
return nil, fmt.Errorf("open IAM account bucket: %w", err)
}
@@ -261,7 +292,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
// read-modify-write Put like the other writers here (no CAS).
func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.TagOpenIDConnectProviderInput) (*iam.TagOpenIDConnectProviderOutput, error) {
ctx := context.Background()
- if err := validateTags(input.Tags); err != nil {
+ if err := validateTags(input.Tags, exactKeys); err != nil {
return nil, err
}
@@ -271,7 +302,7 @@ func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.T
return nil, err
}
- merged := mergeTags(record.Tags, input.Tags)
+ merged := mergeTags(record.Tags, input.Tags, exactKeys)
if len(merged) > maxTagsPerResource {
return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
}
@@ -288,6 +319,9 @@ func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.T
// UntagOpenIDConnectProvider removes the named tag keys from an OIDC provider;
// unknown keys are a no-op.
func (s *IAMServiceImpl) UntagOpenIDConnectProvider(accountID string, input *iam.UntagOpenIDConnectProviderInput) (*iam.UntagOpenIDConnectProviderOutput, error) {
+ if err := validateTagKeys(input.TagKeys); err != nil {
+ return nil, err
+ }
ctx := context.Background()
arn := aws.StringValue(input.OpenIDConnectProviderArn)
record, err := s.getOIDCProvider(ctx, accountID, arn)
@@ -295,7 +329,7 @@ func (s *IAMServiceImpl) UntagOpenIDConnectProvider(accountID string, input *iam
return nil, err
}
- record.Tags = removeTagKeys(record.Tags, input.TagKeys)
+ record.Tags = removeTagKeys(record.Tags, input.TagKeys, exactKeys)
if err := s.putOIDCProvider(ctx, accountID, record); err != nil {
return nil, err
diff --git a/spinifex/handlers/iam/oidc_providers_test.go b/spinifex/handlers/iam/oidc_providers_test.go
index 7e5115556..a2c83fb20 100644
--- a/spinifex/handlers/iam/oidc_providers_test.go
+++ b/spinifex/handlers/iam/oidc_providers_test.go
@@ -214,3 +214,44 @@ func TestIssuerProviderARNRoundTrip(t *testing.T) {
t.Fatal("round-trip key mismatch")
}
}
+
+// The messages are the ones AWS returned; the lists are checked before the Url.
+func TestCreateOpenIDConnectProvider_ListEntryLengths(t *testing.T) {
+ t.Parallel()
+ const (
+ clientIDMsg = "1 validation error detected: Value at 'clientIDList' failed to satisfy constraint: Member must satisfy constraint: [Member must have length less than or equal to 255, Member must have length greater than or equal to 1]"
+ thumbprintMsg = "1 validation error detected: Value at 'thumbprintList' failed to satisfy constraint: Member must satisfy constraint: [Member must have length less than or equal to 40, Member must have length greater than or equal to 40]"
+ )
+ thumbprint := strings.Repeat("9", 40)
+ cases := []struct {
+ name string
+ clientIDs []string
+ thumbprints []string
+ wantMsg string
+ }{
+ {"empty client ID", []string{"sts.amazonaws.com", ""}, []string{thumbprint}, clientIDMsg},
+ {"256-character client ID", []string{strings.Repeat("c", 256)}, []string{thumbprint}, clientIDMsg},
+ {"39-character thumbprint", []string{"sts.amazonaws.com"}, []string{strings.Repeat("9", 39)}, thumbprintMsg},
+ {"41-character thumbprint", []string{"sts.amazonaws.com"}, []string{thumbprint, strings.Repeat("9", 41)}, thumbprintMsg},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ _, err := svc.CreateOpenIDConnectProvider(testAccountID, &iam.CreateOpenIDConnectProviderInput{
+ Url: aws.String("http://not-https.example.com"), ClientIDList: aws.StringSlice(tc.clientIDs), ThumbprintList: aws.StringSlice(tc.thumbprints),
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError, tc.wantMsg)
+ })
+ }
+
+ svc := setupTestIAMService(t)
+ _, err := svc.CreateOpenIDConnectProvider(testAccountID, &iam.CreateOpenIDConnectProviderInput{
+ Url: aws.String("https://oidc.example.com/id/BOUNDS"),
+ ClientIDList: aws.StringSlice([]string{"c", strings.Repeat("c", 255)}),
+ ThumbprintList: aws.StringSlice([]string{thumbprint}),
+ })
+ if err != nil {
+ t.Fatalf("entries at the length bounds: %v", err)
+ }
+}
diff --git a/spinifex/handlers/iam/provision_test.go b/spinifex/handlers/iam/provision_test.go
index 8279d8ae2..22887fa5b 100644
--- a/spinifex/handlers/iam/provision_test.go
+++ b/spinifex/handlers/iam/provision_test.go
@@ -20,7 +20,7 @@ func newIAMService(t *testing.T) *handlers_iam.IAMServiceImpl {
masterKey, err := handlers_iam.GenerateMasterKey()
require.NoError(t, err)
- svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
return svc
}
diff --git a/spinifex/handlers/iam/retry.go b/spinifex/handlers/iam/retry.go
index 126e1f31e..e523603b7 100644
--- a/spinifex/handlers/iam/retry.go
+++ b/spinifex/handlers/iam/retry.go
@@ -3,10 +3,12 @@ package handlers_iam
import (
"context"
"fmt"
- "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
"log/slog"
"time"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
+ telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+
"github.com/nats-io/nats.go"
)
@@ -16,7 +18,7 @@ import (
// leave the service nil for the process lifetime. Blocks up to maxWait, then
// returns the last error. Callers that legitimately run without a master key
// must guard the call themselves.
-func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) {
+func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl, error) {
const maxWait = 5 * time.Minute
retryDelay := 500 * time.Millisecond
start := time.Now()
@@ -24,20 +26,27 @@ func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey
for {
attempt++
- svc, err := NewIAMServiceImpl(ctx, natsConn, masterKey, clusterSize)
+ svc, err := NewIAMServiceImpl(ctx, natsConn, masterKey)
if err == nil {
if attempt > 1 {
- slog.Info("IAM service initialized after retry", "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start)))
+ slog.Info("IAM service initialized after retry", "attempts", attempt, "elapsed_ms", telemetry.Millis(time.Since(start)))
}
return svc, nil
}
+ // An undeclared cluster size is a configuration fault, not a cluster that
+ // has yet to form, so retrying it turns a clear error into five minutes
+ // of waiting followed by a misleading one.
+ if clustersize.Permanent(err) {
+ return nil, err
+ }
+
elapsed := time.Since(start)
if elapsed >= maxWait {
return nil, fmt.Errorf("IAM service unavailable after %s (%d attempts): %w", elapsed.Round(time.Second), attempt, err)
}
- slog.Warn("IAM service not ready (waiting for JetStream cluster quorum)", "error", err, "attempt", attempt, "elapsed_ms", otelsetup.Millis(elapsed), "retry_in_ms", otelsetup.Millis(retryDelay))
+ slog.Warn("IAM service not ready (waiting for JetStream cluster quorum)", "error", err, "attempt", attempt, "elapsed_ms", telemetry.Millis(elapsed), "retry_in_ms", telemetry.Millis(retryDelay))
time.Sleep(retryDelay)
retryDelay = min(retryDelay*2, 10*time.Second)
}
diff --git a/spinifex/handlers/iam/retry_test.go b/spinifex/handlers/iam/retry_test.go
new file mode 100644
index 000000000..464191d39
--- /dev/null
+++ b/spinifex/handlers/iam/retry_test.go
@@ -0,0 +1,38 @@
+package handlers_iam_test
+
+import (
+ "testing"
+ "time"
+
+ "github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
+ handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+ "github.com/stretchr/testify/require"
+)
+
+// TestNewIAMServiceWithRetry_StopsOnAConfigFault covers the difference between a
+// cluster that has yet to form and one that was misconfigured.
+//
+// The retry loop exists for the first: on a cold multi-node boot the KV store
+// needs a quorum that does not exist yet, so waiting is right. An undeclared
+// cluster size is the second, and waiting five minutes for it produces a timeout
+// that names quorum as the cause when the cause was a config the process never
+// loaded. This test bounds that: it must come back fast, and it must come back
+// saying what is actually wrong.
+func TestNewIAMServiceWithRetry_StopsOnAConfigFault(t *testing.T) {
+ _, nc, _ := testutil.StartTestJetStream(t)
+ clustersize.RedeclareForTest(t, 0)
+
+ masterKey, err := handlers_iam.GenerateMasterKey()
+ require.NoError(t, err)
+
+ start := time.Now()
+ _, err = handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey)
+ elapsed := time.Since(start)
+
+ require.Error(t, err, "an undeclared cluster size must not look like a working IAM service")
+ require.ErrorIs(t, err, clustersize.ErrUndeclared,
+ "the error has to name the config fault, not the quorum the loop was waiting for")
+ require.Lessf(t, elapsed, 10*time.Second,
+ "returned after %s; a permanent fault must not consume the retry budget", elapsed)
+}
diff --git a/spinifex/handlers/iam/roles.go b/spinifex/handlers/iam/roles.go
index f09b1d56e..c865a8fe5 100644
--- a/spinifex/handlers/iam/roles.go
+++ b/spinifex/handlers/iam/roles.go
@@ -27,7 +27,7 @@ const (
maxTrustPolicyDocumentSize = 2048
defaultMaxSessionDuration = int64(3600)
- minMaxSessionDuration = int64(900)
+ minMaxSessionDuration = int64(3600)
maxMaxSessionDuration = int64(43200)
// Bound on optimistic-concurrency retries when a concurrent writer wins the
@@ -55,6 +55,13 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput
}
}
+ if err := validateDescription(input.Description); err != nil {
+ return nil, err
+ }
+ if err := validateMaxSessionDuration(input.MaxSessionDuration); err != nil {
+ return nil, err
+ }
+
// Carry the reason: a bare code cannot tell a caller which statement or key
// was refused, and the gateway logs the returned error, so one message
// serves both the client and the operator.
@@ -66,9 +73,10 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput
maxSession := defaultMaxSessionDuration
if input.MaxSessionDuration != nil {
maxSession = *input.MaxSessionDuration
- if maxSession < minMaxSessionDuration || maxSession > maxMaxSessionDuration {
- return nil, errors.New(awserrors.ErrorValidationError)
- }
+ }
+
+ if err := validateTags(input.Tags, foldedKeys); err != nil {
+ return nil, err
}
roleID, err := generateIAMID("AROA")
@@ -210,9 +218,29 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput
return &iam.DeleteRoleOutput{}, nil
}
+// validateMaxSessionDuration enforces the role session limit of one to twelve
+// hours, in seconds, with the ValidationError AWS returns.
+func validateMaxSessionDuration(d *int64) error {
+ switch {
+ case d == nil:
+ return nil
+ case *d < minMaxSessionDuration:
+ return validationError([]string{fmt.Sprintf("Value at 'maxSessionDuration' failed to satisfy constraint: Member must have value greater than or equal to %d", minMaxSessionDuration)})
+ case *d > maxMaxSessionDuration:
+ return validationError([]string{fmt.Sprintf("Value at 'maxSessionDuration' failed to satisfy constraint: Member must have value less than or equal to %d", maxMaxSessionDuration)})
+ }
+ return nil
+}
+
func (s *IAMServiceImpl) UpdateRole(accountID string, input *iam.UpdateRoleInput) (*iam.UpdateRoleOutput, error) {
ctx := context.Background()
roleName := *input.RoleName
+ if err := validateDescription(input.Description); err != nil {
+ return nil, err
+ }
+ if err := validateMaxSessionDuration(input.MaxSessionDuration); err != nil {
+ return nil, err
+ }
role, err := s.getRole(ctx, accountID, roleName)
if err != nil {
@@ -223,11 +251,7 @@ func (s *IAMServiceImpl) UpdateRole(accountID string, input *iam.UpdateRoleInput
role.Description = *input.Description
}
if input.MaxSessionDuration != nil {
- dur := *input.MaxSessionDuration
- if dur < minMaxSessionDuration || dur > maxMaxSessionDuration {
- return nil, errors.New(awserrors.ErrorValidationError)
- }
- role.MaxSessionDuration = dur
+ role.MaxSessionDuration = *input.MaxSessionDuration
}
data, err := json.Marshal(role)
@@ -280,7 +304,7 @@ func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRol
// AmazonEKSWorkerNodePolicy, AmazonEKS_CNI_Policy, ...). Store them opaquely
// so ListAttachedRolePolicies / DescribeNodegroup round-trip instead of
// failing NoSuchEntity. Customer-managed ARNs must still exist.
- if !isAWSManagedPolicyARN(policyARN) {
+ if !iamarn.IsAWSManagedPolicyARN(policyARN) {
if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
return nil, err
}
@@ -329,25 +353,7 @@ func (s *IAMServiceImpl) ListAttachedRolePolicies(accountID string, input *iam.L
return nil, err
}
- var attached []*iam.AttachedPolicy
- for _, arn := range role.AttachedPolicies {
- if isAWSManagedPolicyARN(arn) {
- attached = append(attached, &iam.AttachedPolicy{
- PolicyArn: aws.String(arn),
- PolicyName: aws.String(managedPolicyNameFromARN(arn)),
- })
- continue
- }
- policy, err := s.getPolicyByARN(ctx, accountID, arn)
- if err != nil {
- slog.Warn("ListAttachedRolePolicies: policy not found for ARN", "arn", arn, "err", err)
- continue
- }
- attached = append(attached, &iam.AttachedPolicy{
- PolicyArn: aws.String(policy.ARN),
- PolicyName: aws.String(policy.PolicyName),
- })
- }
+ attached := s.attachedPolicies(ctx, accountID, role.AttachedPolicies, aws.StringValue(input.PathPrefix))
return &iam.ListAttachedRolePoliciesOutput{
AttachedPolicies: attached,
@@ -465,13 +471,13 @@ func (s *IAMServiceImpl) ListRolePolicies(accountID string, input *iam.ListRoleP
// TagRole upserts tags on a role under CAS, like the other role writers.
func (s *IAMServiceImpl) TagRole(accountID string, input *iam.TagRoleInput) (*iam.TagRoleOutput, error) {
ctx := context.Background()
- if err := validateTags(input.Tags); err != nil {
+ if err := validateTags(input.Tags, foldedKeys); err != nil {
return nil, err
}
roleName := *input.RoleName
err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
- merged := mergeTags(role.Tags, input.Tags)
+ merged := mergeTags(role.Tags, input.Tags, foldedKeys)
if len(merged) > maxTagsPerResource {
return false, errors.New(awserrors.ErrorIAMLimitExceeded)
}
@@ -488,10 +494,13 @@ func (s *IAMServiceImpl) TagRole(accountID string, input *iam.TagRoleInput) (*ia
// UntagRole removes the named tag keys from a role; unknown keys are a no-op.
func (s *IAMServiceImpl) UntagRole(accountID string, input *iam.UntagRoleInput) (*iam.UntagRoleOutput, error) {
+ if err := validateTagKeys(input.TagKeys); err != nil {
+ return nil, err
+ }
ctx := context.Background()
roleName := *input.RoleName
err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
- role.Tags = removeTagKeys(role.Tags, input.TagKeys)
+ role.Tags = removeTagKeys(role.Tags, input.TagKeys, foldedKeys)
return true, nil
})
if err != nil {
@@ -547,14 +556,6 @@ func (s *IAMServiceImpl) GetRolePolicies(accountID, roleName string) ([]PolicyDo
return docs, nil
}
-// isAWSManagedPolicyARN reports whether arn is an AWS-managed policy ARN
-// (arn:aws:iam::aws:policy/...). These are not provisioned in Spinifex but are
-// stored and round-tripped opaquely so stock EKS tooling that attaches them
-// works without a backing policy document.
-func isAWSManagedPolicyARN(arn string) bool {
- return iamarn.IsAWSManagedPolicyARN(arn)
-}
-
// managedPolicyNameFromARN returns the final path segment of an AWS-managed
// policy ARN, e.g. .../service-role/AmazonEKS_CNI_Policy -> AmazonEKS_CNI_Policy.
// Display-only, so an unparseable ARN falls back to itself rather than erroring.
diff --git a/spinifex/handlers/iam/roles_test.go b/spinifex/handlers/iam/roles_test.go
index e9c88850d..be69317ae 100644
--- a/spinifex/handlers/iam/roles_test.go
+++ b/spinifex/handlers/iam/roles_test.go
@@ -3,6 +3,7 @@ package handlers_iam
import (
"encoding/json"
"slices"
+ "strconv"
"strings"
"sync"
"testing"
@@ -102,7 +103,7 @@ func TestCreateRole_InvalidName(t *testing.T) {
AssumeRolePolicyDocument: aws.String(validTrustPolicy()),
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
func TestCreateRole_InvalidPath(t *testing.T) {
@@ -320,30 +321,45 @@ func TestCreateRole_TrustPolicy_DenyEffectAccepted(t *testing.T) {
require.NoError(t, err)
}
-func TestCreateRole_MaxSessionDuration_TooSmall(t *testing.T) {
+// The bounds and messages are the ones AWS returned; both calls check the
+// value before the trust policy or the role lookup.
+func TestRole_MaxSessionDurationBounds(t *testing.T) {
t.Parallel()
- svc := setupTestIAMService(t)
-
- _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
- RoleName: aws.String("short-session"),
- AssumeRolePolicyDocument: aws.String(validTrustPolicy()),
- MaxSessionDuration: aws.Int64(minMaxSessionDuration - 1),
- })
- assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
-}
-
-func TestCreateRole_MaxSessionDuration_TooLarge(t *testing.T) {
- t.Parallel()
- svc := setupTestIAMService(t)
-
- _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
- RoleName: aws.String("long-session"),
- AssumeRolePolicyDocument: aws.String(validTrustPolicy()),
- MaxSessionDuration: aws.Int64(maxMaxSessionDuration + 1),
- })
- assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
+ const (
+ tooShort = "1 validation error detected: Value at 'maxSessionDuration' failed to satisfy constraint: Member must have value greater than or equal to 3600"
+ tooLong = "1 validation error detected: Value at 'maxSessionDuration' failed to satisfy constraint: Member must have value less than or equal to 43200"
+ )
+ cases := []struct {
+ seconds int64
+ wantMsg string
+ }{
+ {900, tooShort},
+ {3599, tooShort},
+ {3600, ""},
+ {43200, ""},
+ {43201, tooLong},
+ }
+ for _, tc := range cases {
+ t.Run(strconv.FormatInt(tc.seconds, 10), func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ if tc.wantMsg != "" {
+ _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+ RoleName: aws.String("r"), AssumeRolePolicyDocument: aws.String("{"), MaxSessionDuration: aws.Int64(tc.seconds),
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError, tc.wantMsg)
+ _, err = svc.UpdateRole(testAccountID, &iam.UpdateRoleInput{RoleName: aws.String("missing"), MaxSessionDuration: aws.Int64(tc.seconds)})
+ requireIAMError(t, err, awserrors.ErrorValidationError, tc.wantMsg)
+ return
+ }
+ _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+ RoleName: aws.String("r"), AssumeRolePolicyDocument: aws.String(validTrustPolicy()), MaxSessionDuration: aws.Int64(tc.seconds),
+ })
+ require.NoError(t, err)
+ _, err = svc.UpdateRole(testAccountID, &iam.UpdateRoleInput{RoleName: aws.String("r"), MaxSessionDuration: aws.Int64(tc.seconds)})
+ require.NoError(t, err)
+ })
+ }
}
func TestCreateRole_Duplicate(t *testing.T) {
@@ -541,19 +557,6 @@ func TestUpdateRole_NotFound(t *testing.T) {
assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
}
-func TestUpdateRole_InvalidMaxSessionDuration(t *testing.T) {
- t.Parallel()
- svc := setupTestIAMService(t)
- createTestRole(t, svc, "session-role")
-
- _, err := svc.UpdateRole(testAccountID, &iam.UpdateRoleInput{
- RoleName: aws.String("session-role"),
- MaxSessionDuration: aws.Int64(60),
- })
- assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
-}
-
func TestUpdateAssumeRolePolicy(t *testing.T) {
t.Parallel()
svc := setupTestIAMService(t)
@@ -1597,3 +1600,35 @@ func TestUpdateAssumeRolePolicy_NotActionRejected(t *testing.T) {
assert.Error(t, err)
assert.Contains(t, err.Error(), awserrors.ErrorIAMMalformedPolicyDocument)
}
+
+// The message is the one AWS returned for a 1001-character description, which
+// it checks before the role lookup and the trust policy.
+func TestRoleAndPolicy_DescriptionLength(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ const msg = "1 validation error detected: Value at 'description' failed to satisfy constraint: Member must have length less than or equal to 1000"
+ tooLong := aws.String(strings.Repeat("é", maxDescriptionLength+1))
+
+ _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+ RoleName: aws.String("r"), AssumeRolePolicyDocument: aws.String("{"), Description: tooLong,
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError, msg)
+ _, err = svc.UpdateRole(testAccountID, &iam.UpdateRoleInput{RoleName: aws.String("missing"), Description: tooLong})
+ requireIAMError(t, err, awserrors.ErrorValidationError, msg)
+ _, err = svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("p"), PolicyDocument: aws.String("{"), Description: tooLong,
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError, msg)
+
+ atMax := aws.String(strings.Repeat("é", maxDescriptionLength))
+ _, err = svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+ RoleName: aws.String("r"), AssumeRolePolicyDocument: aws.String(validTrustPolicy()), Description: atMax,
+ })
+ require.NoError(t, err)
+ _, err = svc.UpdateRole(testAccountID, &iam.UpdateRoleInput{RoleName: aws.String("r"), Description: atMax})
+ require.NoError(t, err)
+ _, err = svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("p"), PolicyDocument: aws.String(validPolicyDocument()), Description: atMax,
+ })
+ require.NoError(t, err)
+}
diff --git a/spinifex/handlers/iam/service_impl.go b/spinifex/handlers/iam/service_impl.go
index a6d2fe4f0..4bbeac4e9 100644
--- a/spinifex/handlers/iam/service_impl.go
+++ b/spinifex/handlers/iam/service_impl.go
@@ -10,10 +10,12 @@ import (
"log/slog"
"maps"
"net/netip"
+ "regexp"
"slices"
"strconv"
"strings"
"time"
+ "unicode/utf8"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
@@ -54,6 +56,7 @@ const (
maxTagsPerResource = 50
maxTagKeyLength = 128
maxTagValueLength = 256
+ maxDescriptionLength = 1000
// LongLivedAccessKeyIDPrefix is the AWS-defined prefix for long-lived IAM access keys.
// The access-keys bucket rejects writes with any other prefix to prevent silent privilege escalation.
@@ -106,16 +109,13 @@ type IAMServiceImpl struct {
instanceProfilesBucket jetstream.KeyValue
groupsBucket jetstream.KeyValue
key *masterkey.Key
- // replicas is the JetStream replication factor for lazily-created per-account buckets.
- replicas int
}
var _ IAMService = (*IAMServiceImpl)(nil)
// NewIAMServiceImpl creates a new IAM service backed by NATS JetStream KV.
-// clusterSize sets the replication factor; pass 1 for single-node or test setups.
// The context bounds bucket creation and the schema migrations only.
-func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) {
+func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl, error) {
if builtinManagedPolicyParseErr != nil {
return nil, fmt.Errorf("init builtin managed policies: %w", builtinManagedPolicyParseErr)
}
@@ -123,14 +123,12 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey))
}
- replicas := max(clusterSize, 1)
-
js, err := jetstream.New(natsConn)
if err != nil {
return nil, fmt.Errorf("get JetStream context: %w", err)
}
- usersBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketUsers, 10, replicas)
+ usersBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketUsers, 10)
if err != nil {
return nil, fmt.Errorf("init users bucket: %w", err)
}
@@ -138,7 +136,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketUsers, err)
}
- accessKeysBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccessKeys, 5, replicas)
+ accessKeysBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccessKeys, 5)
if err != nil {
return nil, fmt.Errorf("init access keys bucket: %w", err)
}
@@ -146,7 +144,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketAccessKeys, err)
}
- policiesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketPolicies, 10, replicas)
+ policiesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPolicies, 10)
if err != nil {
return nil, fmt.Errorf("init policies bucket: %w", err)
}
@@ -154,7 +152,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketPolicies, err)
}
- accountsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccounts, 5, replicas)
+ accountsBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccounts, 5)
if err != nil {
return nil, fmt.Errorf("init accounts bucket: %w", err)
}
@@ -162,7 +160,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketAccounts, err)
}
- accountCounterBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccountCounter, 5, replicas)
+ accountCounterBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountCounter, 5)
if err != nil {
return nil, fmt.Errorf("init account counter bucket: %w", err)
}
@@ -170,7 +168,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketAccountCounter, err)
}
- rolesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketRoles, 10, replicas)
+ rolesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRoles, 10)
if err != nil {
return nil, fmt.Errorf("init roles bucket: %w", err)
}
@@ -178,7 +176,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketRoles, err)
}
- instanceProfilesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketInstanceProfiles, 10, replicas)
+ instanceProfilesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketInstanceProfiles, 10)
if err != nil {
return nil, fmt.Errorf("init instance profiles bucket: %w", err)
}
@@ -186,7 +184,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
return nil, fmt.Errorf("migrate %s: %w", KVBucketInstanceProfiles, err)
}
- groupsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketGroups, 10, replicas)
+ groupsBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketGroups, 10)
if err != nil {
return nil, fmt.Errorf("init groups bucket: %w", err)
}
@@ -206,8 +204,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
"accounts_bucket", KVBucketAccounts,
"roles_bucket", KVBucketRoles,
"instance_profiles_bucket", KVBucketInstanceProfiles,
- "groups_bucket", KVBucketGroups,
- "replicas", replicas)
+ "groups_bucket", KVBucketGroups)
return &IAMServiceImpl{
js: js,
@@ -221,18 +218,15 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt
instanceProfilesBucket: instanceProfilesBucket,
groupsBucket: groupsBucket,
key: key,
- replicas: replicas,
}, nil
}
-// copyTags converts SDK IAM tags into the stored Tag slice, skipping entries
-// with a nil key or value. Returns a non-nil (possibly empty) slice.
+// copyTags converts SDK IAM tags, already checked by validateTags, into the
+// stored Tag slice. Returns a non-nil (possibly empty) slice.
func copyTags(tags []*iam.Tag) []Tag {
out := make([]Tag, 0, len(tags))
for _, tag := range tags {
- if tag.Key != nil && tag.Value != nil {
- out = append(out, Tag{Key: *tag.Key, Value: *tag.Value})
- }
+ out = append(out, Tag{Key: *tag.Key, Value: *tag.Value})
}
return out
}
@@ -249,29 +243,57 @@ func tagsToSDK(tags []Tag) []*iam.Tag {
return out
}
-// validateTags enforces the AWS IAM tag limits on request input: at most 50
-// tags, key length 1-128, value length 0-256, no duplicate keys.
-func validateTags(tags []*iam.Tag) error {
- if len(tags) > maxTagsPerResource {
- return errors.New(awserrors.ErrorIAMLimitExceeded)
+// Tag key and value patterns from the IAM model, as AWS enforces them.
+var (
+ tagKeyPattern = regexp.MustCompile(`^[\p{L}\p{Z}\p{N}_.:/=+\-@]+$`)
+ tagValuePattern = regexp.MustCompile(`^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$`)
+)
+
+// keyCase is how a resource type compares tag keys: AWS ignores case on users
+// and roles, and matches exactly on policies, instance profiles and OIDC
+// providers.
+type keyCase bool
+
+const (
+ exactKeys keyCase = false
+ foldedKeys keyCase = true
+)
+
+func (c keyCase) norm(key string) string {
+ if c == foldedKeys {
+ return strings.ToLower(key)
+ }
+ return key
+}
+
+// validateTags enforces the AWS IAM tag limits on request input: the model's
+// list length and its null, length (in characters) and pattern constraints on
+// every key and value, then no reserved aws: prefix and no duplicate keys as
+// keys compares them.
+func validateTags(tags []*iam.Tag, keys keyCase) error {
+ var violations []string
+ if len(tags) > maxTagsPerResource {
+ violations = append(violations, fmt.Sprintf("Value at 'tags' failed to satisfy constraint: Member must have length less than or equal to %d", maxTagsPerResource))
}
- seen := make(map[string]struct{}, len(tags))
for i, tag := range tags {
field := fmt.Sprintf("tags.%d.member", i+1)
- if tag == nil || tag.Key == nil {
- return awserrors.Errorf(awserrors.ErrorIAMInvalidInput,
- "1 validation error detected: Value null at '%s.key' failed to satisfy constraint: Member must not be null", field)
+ if tag == nil {
+ tag = &iam.Tag{}
}
- key := *tag.Key
- if len(key) < 1 {
- return lengthViolation(key, field+".key", "greater than or equal to 1")
- }
- if len(key) > maxTagKeyLength {
- return lengthViolation(key, field+".key", fmt.Sprintf("less than or equal to %d", maxTagKeyLength))
- }
- if tag.Value != nil && len(*tag.Value) > maxTagValueLength {
- return lengthViolation(*tag.Value, field+".value", fmt.Sprintf("less than or equal to %d", maxTagValueLength))
+ violations = append(violations, tagMemberViolations(field+".key", tag.Key, 1, maxTagKeyLength, tagKeyPattern)...)
+ violations = append(violations, tagMemberViolations(field+".value", tag.Value, 0, maxTagValueLength, tagValuePattern)...)
+ }
+ if err := validationError(violations); err != nil {
+ return err
+ }
+ for _, tag := range tags {
+ if strings.HasPrefix(strings.ToLower(*tag.Key), "aws:") {
+ return awserrors.Errorf(awserrors.ErrorIAMInvalidInput, "Tag keys beginning with aws: are reserved for system use.")
}
+ }
+ seen := make(map[string]struct{}, len(tags))
+ for _, tag := range tags {
+ key := keys.norm(*tag.Key)
if _, dup := seen[key]; dup {
return awserrors.Errorf(awserrors.ErrorIAMInvalidInput,
"Duplicate tag keys found. Please note that Tag keys are case insensitive.")
@@ -281,16 +303,73 @@ func validateTags(tags []*iam.Tag) error {
return nil
}
+// validateTagKeys enforces the model constraints on an untag request's key
+// list. AWS reports every bad key as one violation of the list.
+func validateTagKeys(keys []*string) error {
+ var violations []string
+ if len(keys) > maxTagsPerResource {
+ violations = append(violations, fmt.Sprintf("Value at 'tagKeys' failed to satisfy constraint: Member must have length less than or equal to %d", maxTagsPerResource))
+ }
+ for _, key := range keys {
+ if key == nil || len(tagMemberViolations("", key, 1, maxTagKeyLength, tagKeyPattern)) > 0 {
+ violations = append(violations, fmt.Sprintf("Value at 'tagKeys' failed to satisfy constraint: Member must satisfy constraint: "+
+ "[Member must have length less than or equal to %d, Member must have length greater than or equal to 1, "+
+ "Member must satisfy regular expression pattern: %s, Member must not be null]", maxTagKeyLength, modelPattern(tagKeyPattern)))
+ break
+ }
+ }
+ return validationError(violations)
+}
+
+// validationError renders model-constraint violations as the single
+// ValidationError AWS returns for them, or nil when there are none.
+func validationError(violations []string) error {
+ if len(violations) == 0 {
+ return nil
+ }
+ noun := "errors"
+ if len(violations) == 1 {
+ noun = "error"
+ }
+ return awserrors.Errorf(awserrors.ErrorValidationError,
+ "%d validation %s detected: %s", len(violations), noun, strings.Join(violations, "; "))
+}
+
+// tagMemberViolations checks one tag key or value. AWS leaves the value out of
+// tag violation messages, so these name only the member.
+func tagMemberViolations(field string, value *string, minLen, maxLen int, pattern *regexp.Regexp) []string {
+ prefix := "Value at '" + field + "' failed to satisfy constraint: Member must "
+ if value == nil {
+ return []string{prefix + "not be null"}
+ }
+ var out []string
+ if n := utf8.RuneCountInString(*value); n < minLen {
+ out = append(out, fmt.Sprintf("%shave length greater than or equal to %d", prefix, minLen))
+ } else if n > maxLen {
+ out = append(out, fmt.Sprintf("%shave length less than or equal to %d", prefix, maxLen))
+ }
+ if !pattern.MatchString(*value) {
+ out = append(out, prefix+"satisfy regular expression pattern: "+modelPattern(pattern))
+ }
+ return out
+}
+
+// modelPattern renders a compiled tag pattern as the model writes it.
+func modelPattern(pattern *regexp.Regexp) string {
+ return strings.TrimSuffix(strings.TrimPrefix(pattern.String(), "^"), "$")
+}
+
// mergeTags upserts add into existing by key, matching AWS semantics: a
-// repeated key overwrites in place, new keys append in input order.
-func mergeTags(existing []Tag, add []*iam.Tag) []Tag {
+// repeated key, as keys compares them, is overwritten in place with the new
+// key's case; new keys append in input order.
+func mergeTags(existing []Tag, add []*iam.Tag, keys keyCase) []Tag {
out := slices.Clone(existing)
for _, tag := range add {
if tag.Key == nil {
continue
}
next := Tag{Key: *tag.Key, Value: aws.StringValue(tag.Value)}
- idx := slices.IndexFunc(out, func(t Tag) bool { return t.Key == next.Key })
+ idx := slices.IndexFunc(out, func(t Tag) bool { return keys.norm(t.Key) == keys.norm(next.Key) })
if idx >= 0 {
out[idx] = next
} else {
@@ -300,18 +379,18 @@ func mergeTags(existing []Tag, add []*iam.Tag) []Tag {
return out
}
-// removeTagKeys drops the named keys from existing; unknown keys are
-// silently ignored, matching AWS.
-func removeTagKeys(existing []Tag, keys []*string) []Tag {
+// removeTagKeys drops the named keys from existing, as match compares them;
+// unknown keys are silently ignored, matching AWS.
+func removeTagKeys(existing []Tag, keys []*string, match keyCase) []Tag {
drop := make(map[string]struct{}, len(keys))
for _, k := range keys {
if k != nil {
- drop[*k] = struct{}{}
+ drop[match.norm(*k)] = struct{}{}
}
}
out := make([]Tag, 0, len(existing))
for _, t := range existing {
- if _, gone := drop[t.Key]; !gone {
+ if _, gone := drop[match.norm(t.Key)]; !gone {
out = append(out, t)
}
}
@@ -341,6 +420,10 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput
}
}
+ if err := validateTags(input.Tags, foldedKeys); err != nil {
+ return nil, err
+ }
+
userID, err := generateIAMID("AIDA")
if err != nil {
return nil, fmt.Errorf("generate user ID: %w", err)
@@ -1192,19 +1275,23 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI
kvKey := accountID + "." + policyName
+ if err := validateDescription(input.Description); err != nil {
+ return nil, err
+ }
+ path := "/"
+ if input.Path != nil {
+ path = *input.Path
+ if err := validatePath(path); err != nil {
+ return nil, err
+ }
+ }
+
if _, err := ValidatePolicyDocument(*input.PolicyDocument); err != nil {
return nil, awserrors.Errorf(awserrors.ErrorIAMMalformedPolicyDocument,
"policy %q: %w", policyName, err)
}
- if err := validateTags(input.Tags); err != nil {
- return nil, err
- }
-
- path := aws.StringValue(input.Path)
- if path == "" {
- path = "/"
- } else if err := validatePath(path); err != nil {
+ if err := validateTags(input.Tags, exactKeys); err != nil {
return nil, err
}
@@ -1264,6 +1351,12 @@ func (s *IAMServiceImpl) GetPolicy(accountID string, input *iam.GetPolicyInput)
func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesInput) (*iam.ListPoliciesOutput, error) {
ctx := context.Background()
+ if err := checkEnum("scope", input.Scope, policyScopeValues); err != nil {
+ return nil, err
+ }
+ if err := checkEnum("policyUsageFilter", input.PolicyUsageFilter, policyUsageValues); err != nil {
+ return nil, err
+ }
keys, err := kvutil.Keys(ctx, s.policiesBucket)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
@@ -1350,7 +1443,7 @@ func policyToSDK(p *Policy, attachmentCount int64) *iam.Policy {
// policyInScope applies ListPolicies' Scope: Local is customer-managed, AWS is
// arn:aws:iam::aws:policy/..., and All (or unset) keeps both.
func policyInScope(policyARN, scope string) bool {
- awsManaged := isAWSManagedPolicyARN(policyARN)
+ awsManaged := iamarn.IsAWSManagedPolicyARN(policyARN)
switch scope {
case iam.PolicyScopeTypeLocal:
return !awsManaged
@@ -1412,6 +1505,9 @@ func (s *IAMServiceImpl) ListEntitiesForPolicy(accountID string, input *iam.List
IsTruncated: aws.Bool(false),
}
+ if err := checkEnum("policyUsageFilter", input.PolicyUsageFilter, policyUsageValues); err != nil {
+ return nil, err
+ }
// Permissions boundaries are rejected outright at policy-attachment time,
// so no entity ever uses a policy that way. The correct answer is empty.
if aws.StringValue(input.PolicyUsageFilter) == iam.PolicyUsageTypePermissionsBoundary {
@@ -1479,7 +1575,7 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse
// in Spinifex, so store them opaquely rather than failing NoSuchEntity — the
// grant document is modeled in builtinManagedPolicyDoc and resolved at
// evaluation time. Customer-managed ARNs must still exist.
- if !isAWSManagedPolicyARN(policyARN) {
+ if !iamarn.IsAWSManagedPolicyARN(policyARN) {
if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
return nil, err
}
@@ -1538,27 +1634,28 @@ func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUse
return &iam.DetachUserPolicyOutput{}, nil
}
-func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) {
- ctx := context.Background()
- user, err := s.getUser(ctx, accountID, *input.UserName)
- if err != nil {
- return nil, err
- }
-
+// attachedPolicies resolves an identity's attached policy ARNs for the
+// ListAttached*Policies calls, keeping those whose path starts with pathPrefix.
+func (s *IAMServiceImpl) attachedPolicies(ctx context.Context, accountID string, arns []string, pathPrefix string) []*iam.AttachedPolicy {
var attached []*iam.AttachedPolicy
- for _, arn := range user.AttachedPolicies {
+ for _, arn := range arns {
// AWS-managed ARNs have no KV entry; report them from the ARN itself so
// attach/list round-trips instead of silently dropping them.
- if isAWSManagedPolicyARN(arn) {
- attached = append(attached, &iam.AttachedPolicy{
- PolicyArn: aws.String(arn),
- PolicyName: aws.String(managedPolicyNameFromARN(arn)),
- })
+ if iamarn.IsAWSManagedPolicyARN(arn) {
+ if strings.HasPrefix(policyPathFromARN(arn), pathPrefix) {
+ attached = append(attached, &iam.AttachedPolicy{
+ PolicyArn: aws.String(arn),
+ PolicyName: aws.String(managedPolicyNameFromARN(arn)),
+ })
+ }
continue
}
policy, err := s.getPolicyByARN(ctx, accountID, arn)
if err != nil {
- slog.Warn("ListAttachedUserPolicies: policy not found for ARN", "arn", arn, "err", err)
+ slog.Warn("attached policy not found for ARN", "arn", arn, "err", err)
+ continue
+ }
+ if !strings.HasPrefix(policy.Path, pathPrefix) {
continue
}
attached = append(attached, &iam.AttachedPolicy{
@@ -1566,6 +1663,24 @@ func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.L
PolicyName: aws.String(policy.PolicyName),
})
}
+ return attached
+}
+
+// policyPathFromARN returns the path of a policy ARN, e.g.
+// arn:aws:iam::aws:policy/service-role/Name -> /service-role/.
+func policyPathFromARN(arn string) string {
+ _, resource, _ := strings.Cut(arn, ":policy")
+ return resource[:strings.LastIndex(resource, "/")+1]
+}
+
+func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) {
+ ctx := context.Background()
+ user, err := s.getUser(ctx, accountID, *input.UserName)
+ if err != nil {
+ return nil, err
+ }
+
+ attached := s.attachedPolicies(ctx, accountID, user.AttachedPolicies, aws.StringValue(input.PathPrefix))
return &iam.ListAttachedUserPoliciesOutput{
AttachedPolicies: attached,
@@ -1703,7 +1818,7 @@ func (s *IAMServiceImpl) ListUserPolicies(accountID string, input *iam.ListUserP
// user writers (no CAS).
func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*iam.TagUserOutput, error) {
ctx := context.Background()
- if err := validateTags(input.Tags); err != nil {
+ if err := validateTags(input.Tags, foldedKeys); err != nil {
return nil, err
}
@@ -1713,7 +1828,7 @@ func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*ia
return nil, err
}
- merged := mergeTags(user.Tags, input.Tags)
+ merged := mergeTags(user.Tags, input.Tags, foldedKeys)
if len(merged) > maxTagsPerResource {
return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
}
@@ -1733,6 +1848,9 @@ func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*ia
// UntagUser removes the named tag keys from a user; unknown keys are a no-op.
func (s *IAMServiceImpl) UntagUser(accountID string, input *iam.UntagUserInput) (*iam.UntagUserOutput, error) {
+ if err := validateTagKeys(input.TagKeys); err != nil {
+ return nil, err
+ }
ctx := context.Background()
userName := *input.UserName
user, err := s.getUser(ctx, accountID, userName)
@@ -1740,7 +1858,7 @@ func (s *IAMServiceImpl) UntagUser(accountID string, input *iam.UntagUserInput)
return nil, err
}
- user.Tags = removeTagKeys(user.Tags, input.TagKeys)
+ user.Tags = removeTagKeys(user.Tags, input.TagKeys, foldedKeys)
data, err := json.Marshal(user)
if err != nil {
@@ -1771,12 +1889,12 @@ func (s *IAMServiceImpl) ListUserTags(accountID string, input *iam.ListUserTagsI
// record cannot undo a concurrent change of default version.
func (s *IAMServiceImpl) TagPolicy(accountID string, input *iam.TagPolicyInput) (*iam.TagPolicyOutput, error) {
ctx := context.Background()
- if err := validateTags(input.Tags); err != nil {
+ if err := validateTags(input.Tags, exactKeys); err != nil {
return nil, err
}
err := s.updatePolicyCAS(ctx, accountID, *input.PolicyArn, func(policy *Policy) (bool, error) {
- merged := mergeTags(policy.Tags, input.Tags)
+ merged := mergeTags(policy.Tags, input.Tags, exactKeys)
if len(merged) > maxTagsPerResource {
return false, errors.New(awserrors.ErrorIAMLimitExceeded)
}
@@ -1793,9 +1911,12 @@ func (s *IAMServiceImpl) TagPolicy(accountID string, input *iam.TagPolicyInput)
// UntagPolicy removes the named tag keys from a policy; unknown keys are a no-op.
func (s *IAMServiceImpl) UntagPolicy(accountID string, input *iam.UntagPolicyInput) (*iam.UntagPolicyOutput, error) {
+ if err := validateTagKeys(input.TagKeys); err != nil {
+ return nil, err
+ }
ctx := context.Background()
err := s.updatePolicyCAS(ctx, accountID, *input.PolicyArn, func(policy *Policy) (bool, error) {
- policy.Tags = removeTagKeys(policy.Tags, input.TagKeys)
+ policy.Tags = removeTagKeys(policy.Tags, input.TagKeys, exactKeys)
return true, nil
})
if err != nil {
@@ -1895,34 +2016,36 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo
// ---------------------------------------------------------------------------
func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyARN string) (*Policy, error) {
- _, policyName, err := iamarn.ParsePolicyARN(policyARN)
- if err != nil {
+ var policy *Policy
+ _, _, err := iamarn.ResolvePolicyARN(policyARN, func(_, policyName string) (string, error) {
+ kvKey := accountID + "." + policyName
+ entry, err := s.policiesBucket.Get(ctx, kvKey)
+ if err != nil {
+ if errors.Is(err, jetstream.ErrKeyNotFound) {
+ return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+ }
+ return "", fmt.Errorf("get policy: %w", err)
+ }
+ var p Policy
+ if err := json.Unmarshal(entry.Value(), &p); err != nil {
+ return "", fmt.Errorf("unmarshal policy: %w", err)
+ }
+ policy = &p
+ return p.ARN, nil
+ })
+ switch {
+ case errors.Is(err, iamarn.ErrInvalidPolicyARN):
// NoSuchEntity is the response contract, so log the parse reason —
// otherwise a malformed ARN is indistinguishable from a missing policy.
slog.Debug("getPolicyByARN: unparseable policy ARN",
"accountID", accountID, "policyArn", policyARN, "err", err)
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
- }
-
- kvKey := accountID + "." + policyName
- entry, err := s.policiesBucket.Get(ctx, kvKey)
- if err != nil {
- if errors.Is(err, jetstream.ErrKeyNotFound) {
- return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
- }
- return nil, fmt.Errorf("get policy: %w", err)
- }
-
- var policy Policy
- if err := json.Unmarshal(entry.Value(), &policy); err != nil {
- return nil, fmt.Errorf("unmarshal policy: %w", err)
- }
-
- if policy.ARN != policyARN {
+ case errors.Is(err, iamarn.ErrPolicyARNMismatch):
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+ case err != nil:
+ return nil, err
}
-
- return &policy, nil
+ return policy, nil
}
// entityRef identifies one principal record found to have a given policy
@@ -2093,10 +2216,10 @@ func isIAMNameChar(c byte) bool {
// message AWS gives, which names the failing parameter.
func validateIAMName(field, name string, maxLen int) error {
if len(name) == 0 {
- return lengthViolation(name, field, "greater than or equal to 1")
+ return lengthViolation(field, "greater than or equal to 1")
}
if len(name) > maxLen {
- return lengthViolation(name, field, fmt.Sprintf("less than or equal to %d", maxLen))
+ return lengthViolation(field, fmt.Sprintf("less than or equal to %d", maxLen))
}
for i := range len(name) {
if !isIAMNameChar(name[i]) {
@@ -2107,9 +2230,33 @@ func validateIAMName(field, name string, maxLen int) error {
return nil
}
-func lengthViolation(value, field, bound string) error {
- return awserrors.Errorf(awserrors.ErrorIAMInvalidInput,
- "1 validation error detected: Value '%s' at '%s' failed to satisfy constraint: Member must have length %s", value, field, bound)
+// validateDescription enforces the 1000-character limit on a role or policy
+// description.
+func validateDescription(description *string) error {
+ if description != nil && utf8.RuneCountInString(*description) > maxDescriptionLength {
+ return lengthViolation("description", fmt.Sprintf("less than or equal to %d", maxDescriptionLength))
+ }
+ return nil
+}
+
+// lengthViolation is the ValidationError AWS returns for a member outside its
+// length bounds; IAM leaves the value out of the message.
+func lengthViolation(field, bound string) error {
+ return validationError([]string{fmt.Sprintf("Value at '%s' failed to satisfy constraint: Member must have length %s", field, bound)})
+}
+
+// Enum values in the order AWS lists them in a violation, not the model's.
+var (
+ policyScopeValues = []string{iam.PolicyScopeTypeAll, iam.PolicyScopeTypeLocal, iam.PolicyScopeTypeAws}
+ policyUsageValues = []string{iam.PolicyUsageTypePermissionsBoundary, iam.PolicyUsageTypePermissionsPolicy}
+)
+
+// checkEnum refuses an optional member set to a value outside allowed.
+func checkEnum(field string, value *string, allowed []string) error {
+ if value == nil || slices.Contains(allowed, *value) {
+ return nil
+ }
+ return enumViolation(field, allowed)
}
func enumViolation(field string, allowed []string) error {
@@ -2134,7 +2281,7 @@ func validatePath(path string) error {
"The specified value for path is invalid. It must begin and end with / and contain only alphanumeric characters and/or / characters.")
}
if len(path) > 512 {
- return lengthViolation(path, "path", "less than or equal to 512")
+ return lengthViolation("path", "less than or equal to 512")
}
return nil
}
@@ -2162,11 +2309,14 @@ func ValidatePolicyDocument(docJSON string) (*PolicyDocument, error) {
if stmt.Effect != PolicyEffectAllow && stmt.Effect != PolicyEffectDeny {
return nil, fmt.Errorf("statement %d: Effect must be Allow or Deny, got %q", i, stmt.Effect)
}
- if len(stmt.Action) == 0 {
- return nil, fmt.Errorf("statement %d: Action is required", i)
+ if err := validateSelectorPair(i, "Action", stmt.Action, stmt.NotAction); err != nil {
+ return nil, err
}
- if len(stmt.Resource) == 0 {
- return nil, fmt.Errorf("statement %d: Resource is required", i)
+ if err := validateSelectorPair(i, "Resource", stmt.Resource, stmt.NotResource); err != nil {
+ return nil, err
+ }
+ if err := validateSelectorEntries(i, stmt); err != nil {
+ return nil, err
}
if err := validateStatementRestrictions(i, stmt); err != nil {
return nil, err
@@ -2176,6 +2326,40 @@ func ValidatePolicyDocument(docJSON string) (*PolicyDocument, error) {
return &doc, nil
}
+// validateSelectorPair requires exactly one of a selector and its Not form, as
+// AWS does: a statement naming both, or neither, is malformed.
+func validateSelectorPair(i int, name string, positive, negative StringOrArr) error {
+ switch {
+ case len(positive) > 0 && len(negative) > 0:
+ return fmt.Errorf("statement %d: %s and Not%s cannot both be specified", i, name, name)
+ case len(positive) == 0 && len(negative) == 0:
+ return fmt.Errorf("statement %d: %s is required (or Not%s)", i, name, name)
+ }
+ return nil
+}
+
+// validateSelectorEntries rejects entries AWS rejects as malformed. A bad entry
+// in Action only grants less, but the same entry in NotAction excludes nothing
+// and so grants everything.
+func validateSelectorEntries(i int, stmt Statement) error {
+ for field, actions := range map[string]StringOrArr{"Action": stmt.Action, "NotAction": stmt.NotAction} {
+ for _, action := range actions {
+ vendor, name, found := strings.Cut(action, ":")
+ if action != "*" && (!found || vendor == "" || name == "") {
+ return fmt.Errorf("statement %d: %s %q: Actions/Conditions must be prefaced by a vendor, e.g. iam, sdb, ec2", i, field, action)
+ }
+ }
+ }
+ for field, resources := range map[string]StringOrArr{"Resource": stmt.Resource, "NotResource": stmt.NotResource} {
+ for _, resource := range resources {
+ if resource != "*" && !strings.HasPrefix(resource, "arn:") {
+ return fmt.Errorf("statement %d: %s %q must be in ARN format or \"*\"", i, field, resource)
+ }
+ }
+ }
+ return nil
+}
+
// validateStatementRestrictions rejects the clauses the evaluator cannot enforce,
// so an identity policy is never accepted with an inert restriction on it.
// Conditions inside the supported allowlist are accepted and enforced.
@@ -2183,17 +2367,16 @@ func validateStatementRestrictions(i int, stmt Statement) error {
if isRawJSONNonEmpty(stmt.Principal) {
return fmt.Errorf("statement %d: Principal is not valid on an identity policy; use a resource or trust policy instead", i)
}
- if len(stmt.NotAction) > 0 {
- return fmt.Errorf("statement %d: NotAction blocks are not supported in this release; use Action with an explicit list instead", i)
- }
- if len(stmt.NotResource) > 0 {
- return fmt.Errorf("statement %d: NotResource blocks are not supported in this release; use Resource with an explicit list instead", i)
- }
for _, resource := range stmt.Resource {
if err := validatePolicyVariables(i, "Resource", resource); err != nil {
return err
}
}
+ for _, resource := range stmt.NotResource {
+ if err := validatePolicyVariables(i, "NotResource", resource); err != nil {
+ return err
+ }
+ }
for op, keys := range stmt.Condition {
for key, values := range keys {
if !iampolicy.SupportedCondition(op, key) {
@@ -2228,26 +2411,43 @@ func validateConditionValues(i int, op, key string, values ConditionValue) error
if len(values) == 0 {
return fmt.Errorf("statement %d: Condition operator %q on key %q has no value", i, op, key)
}
+ // An IfExists form takes its base operator's values.
+ base, _ := iampolicy.BaseOperator(op)
for _, v := range values {
- // Only the string operators expand variables; a reference in a Bool or
- // IpAddress value is already rejected as unparseable below.
- if op == iampolicy.OpStringEquals || op == iampolicy.OpStringLike {
+ // Only the string and ARN operators expand variables; a reference in a
+ // Bool, Null, IpAddress or date value is already rejected as unparseable below.
+ switch base {
+ case iampolicy.OpIPAddress, iampolicy.OpNotIPAddress, iampolicy.OpBool, iampolicy.OpNull,
+ iampolicy.OpDateEquals, iampolicy.OpDateNotEquals, iampolicy.OpDateLessThan,
+ iampolicy.OpDateLessThanEquals, iampolicy.OpDateGreaterThan, iampolicy.OpDateGreaterThanEquals:
+ default:
if err := validatePolicyVariables(i, fmt.Sprintf("Condition %s on key %q", op, key), v); err != nil {
return err
}
}
- switch op {
- case iampolicy.OpIPAddress:
+ switch base {
+ case iampolicy.OpArnEquals, iampolicy.OpArnLike, iampolicy.OpArnNotEquals, iampolicy.OpArnNotLike:
+ if _, ok := iampolicy.SplitARN(v); !ok {
+ return fmt.Errorf("statement %d: Condition %s on key %q: %q is not an ARN of six colon-separated components",
+ i, op, key, v)
+ }
+ case iampolicy.OpIPAddress, iampolicy.OpNotIPAddress:
if _, prefixErr := netip.ParsePrefix(v); prefixErr != nil {
if _, addrErr := netip.ParseAddr(v); addrErr != nil {
return fmt.Errorf("statement %d: Condition %s on key %q: %q is not a valid IP address or CIDR block",
i, op, key, v)
}
}
- case iampolicy.OpBool:
+ case iampolicy.OpBool, iampolicy.OpNull:
if !strings.EqualFold(v, "true") && !strings.EqualFold(v, "false") {
return fmt.Errorf("statement %d: Condition %s on key %q: %q is not true or false", i, op, key, v)
}
+ case iampolicy.OpDateEquals, iampolicy.OpDateNotEquals, iampolicy.OpDateLessThan,
+ iampolicy.OpDateLessThanEquals, iampolicy.OpDateGreaterThan, iampolicy.OpDateGreaterThanEquals:
+ if _, err := iampolicy.ParseDate(v); err != nil {
+ return fmt.Errorf("statement %d: Condition %s on key %q: %q is not an ISO 8601 date or epoch seconds",
+ i, op, key, v)
+ }
}
}
return nil
diff --git a/spinifex/handlers/iam/service_impl_test.go b/spinifex/handlers/iam/service_impl_test.go
index eda08a1cb..dcf5c5ae7 100644
--- a/spinifex/handlers/iam/service_impl_test.go
+++ b/spinifex/handlers/iam/service_impl_test.go
@@ -9,6 +9,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
+ "github.com/mulgadc/bluebottle/pkg/iampolicy"
"github.com/mulgadc/spinifex/spinifex/admin"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/internal/testkit"
@@ -29,7 +30,7 @@ func setupTestIAMService(t *testing.T) *IAMServiceImpl {
masterKey, err := GenerateMasterKey()
require.NoError(t, err)
- svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
return svc
}
@@ -386,7 +387,7 @@ func TestCreateUser_InvalidName_TooLong(t *testing.T) {
UserName: aws.String(longName),
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
func TestCreateUser_InvalidName_BadChars(t *testing.T) {
@@ -422,7 +423,7 @@ func TestCreatePolicy_InvalidName(t *testing.T) {
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`),
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
func TestCreatePolicy_InvalidPath(t *testing.T) {
@@ -438,6 +439,25 @@ func TestCreatePolicy_InvalidPath(t *testing.T) {
assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
}
+// AWS refuses an empty Path, as every other Create* does, and checks it before
+// the policy document.
+func TestCreatePolicy_EmptyPathRefusedBeforeDocument(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+
+ _, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("EmptyPath"), PolicyDocument: aws.String("{"), Path: aws.String(""),
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError,
+ "The specified value for path is invalid. It must begin and end with / and contain only alphanumeric characters and/or / characters.")
+
+ out, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("NoPath"), PolicyDocument: aws.String(validPolicyDocument()),
+ })
+ require.NoError(t, err)
+ assert.Equal(t, "/", aws.StringValue(out.Policy.Path))
+}
+
func TestValidatePolicyDocument_TooLarge(t *testing.T) {
t.Parallel()
largeDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"` + strings.Repeat("a", maxPolicyDocumentSize) + `"}]}`
@@ -1264,7 +1284,7 @@ func TestCreatePolicy_InvalidTag(t *testing.T) {
Tags: []*iam.Tag{{Key: aws.String(""), Value: aws.String("x")}},
})
assert.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorIAMInvalidInput)
+ assert.Contains(t, err.Error(), awserrors.ErrorValidationError)
// The rejected policy must not have been stored.
_, err = svc.GetPolicy(testAccountID, &iam.GetPolicyInput{
@@ -1700,6 +1720,37 @@ func TestDeletePolicy_AttachedConflict_Group(t *testing.T) {
// ListEntitiesForPolicy Tests
// ============================================================================
+// The messages are the ones AWS returned for the same values.
+func TestListPolicyCalls_EnumViolations(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ const (
+ scopeMsg = "1 validation error detected: Value at 'scope' failed to satisfy constraint: Member must satisfy enum value set: [All, Local, AWS]"
+ usageMsg = "1 validation error detected: Value at 'policyUsageFilter' failed to satisfy constraint: Member must satisfy enum value set: [PermissionsBoundary, PermissionsPolicy]"
+ )
+ _, err := svc.ListPolicies(testAccountID, &iam.ListPoliciesInput{Scope: aws.String("Bogus")})
+ requireIAMError(t, err, awserrors.ErrorValidationError, scopeMsg)
+ _, err = svc.ListPolicies(testAccountID, &iam.ListPoliciesInput{PolicyUsageFilter: aws.String("Bogus")})
+ requireIAMError(t, err, awserrors.ErrorValidationError, usageMsg)
+ _, err = svc.ListEntitiesForPolicy(testAccountID, &iam.ListEntitiesForPolicyInput{
+ PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), PolicyUsageFilter: aws.String("Bogus"),
+ })
+ requireIAMError(t, err, awserrors.ErrorValidationError, usageMsg)
+
+ for _, scope := range iam.PolicyScopeType_Values() {
+ _, err = svc.ListPolicies(testAccountID, &iam.ListPoliciesInput{Scope: aws.String(scope)})
+ require.NoError(t, err, scope)
+ }
+ for _, usage := range iam.PolicyUsageType_Values() {
+ _, err = svc.ListPolicies(testAccountID, &iam.ListPoliciesInput{PolicyUsageFilter: aws.String(usage)})
+ require.NoError(t, err, usage)
+ _, err = svc.ListEntitiesForPolicy(testAccountID, &iam.ListEntitiesForPolicyInput{
+ PolicyArn: aws.String("arn:aws:iam::aws:policy/ReadOnlyAccess"), PolicyUsageFilter: aws.String(usage),
+ })
+ require.NoError(t, err, usage)
+ }
+}
+
func TestListEntitiesForPolicy_Empty(t *testing.T) {
t.Parallel()
svc := setupTestIAMService(t)
@@ -2002,6 +2053,78 @@ func TestListAttachedUserPolicies(t *testing.T) {
assert.True(t, names["ListPolicy2"])
}
+func TestListAttachedPolicies_FilterByPathPrefix(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ root := createTestPolicy(t, svc, "RootPolicy")
+ team, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("TeamPolicy"), Path: aws.String("/team/a/"), PolicyDocument: aws.String(validPolicyDocument()),
+ })
+ require.NoError(t, err)
+ createTestUser(t, svc, "u")
+ createTestRole(t, svc, "r")
+ createTestGroup(t, svc, "g")
+ for _, arn := range []*string{root.Arn, team.Policy.Arn} {
+ _, err = svc.AttachUserPolicy(testAccountID, &iam.AttachUserPolicyInput{UserName: aws.String("u"), PolicyArn: arn})
+ require.NoError(t, err)
+ _, err = svc.AttachRolePolicy(testAccountID, &iam.AttachRolePolicyInput{RoleName: aws.String("r"), PolicyArn: arn})
+ require.NoError(t, err)
+ _, err = svc.AttachGroupPolicy(testAccountID, &iam.AttachGroupPolicyInput{GroupName: aws.String("g"), PolicyArn: arn})
+ require.NoError(t, err)
+ }
+
+ list := map[string]func(prefix *string) ([]*iam.AttachedPolicy, error){
+ "user": func(prefix *string) ([]*iam.AttachedPolicy, error) {
+ out, err := svc.ListAttachedUserPolicies(testAccountID, &iam.ListAttachedUserPoliciesInput{UserName: aws.String("u"), PathPrefix: prefix})
+ if err != nil {
+ return nil, err
+ }
+ return out.AttachedPolicies, nil
+ },
+ "role": func(prefix *string) ([]*iam.AttachedPolicy, error) {
+ out, err := svc.ListAttachedRolePolicies(testAccountID, &iam.ListAttachedRolePoliciesInput{RoleName: aws.String("r"), PathPrefix: prefix})
+ if err != nil {
+ return nil, err
+ }
+ return out.AttachedPolicies, nil
+ },
+ "group": func(prefix *string) ([]*iam.AttachedPolicy, error) {
+ out, err := svc.ListAttachedGroupPolicies(testAccountID, &iam.ListAttachedGroupPoliciesInput{GroupName: aws.String("g"), PathPrefix: prefix})
+ if err != nil {
+ return nil, err
+ }
+ return out.AttachedPolicies, nil
+ },
+ }
+ cases := []struct {
+ prefix *string
+ want []string
+ }{
+ {nil, []string{"RootPolicy", "TeamPolicy"}},
+ {aws.String("/"), []string{"RootPolicy", "TeamPolicy"}},
+ {aws.String("/team/"), []string{"TeamPolicy"}},
+ {aws.String("/other/"), nil},
+ }
+ for identity, fn := range list {
+ for _, tc := range cases {
+ got, err := fn(tc.prefix)
+ require.NoError(t, err)
+ var names []string
+ for _, p := range got {
+ names = append(names, aws.StringValue(p.PolicyName))
+ }
+ assert.ElementsMatch(t, tc.want, names, "%s with PathPrefix %v", identity, aws.StringValue(tc.prefix))
+ }
+ }
+}
+
+func TestPolicyPathFromARN(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, "/", policyPathFromARN("arn:aws:iam::aws:policy/ReadOnlyAccess"))
+ assert.Equal(t, "/service-role/", policyPathFromARN("arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM"))
+ assert.Equal(t, "/team/a/", policyPathFromARN("arn:aws:iam::000000000000:policy/team/a/TeamPolicy"))
+}
+
func TestListAttachedUserPolicies_Empty(t *testing.T) {
t.Parallel()
svc := setupTestIAMService(t)
@@ -2162,6 +2285,42 @@ func TestValidatePolicyDocument_SupportedCondition(t *testing.T) {
doc.Statement[0].Condition["IpAddress"]["aws:SourceIp"])
}
+// Null takes a JSON boolean or its string form, and IfExists forms reach the
+// write path on the keys their base operator does.
+func TestValidatePolicyDocument_NullAndIfExists(t *testing.T) {
+ t.Parallel()
+ for _, cond := range []string{
+ `{"Null":{"aws:username":true}}`,
+ `{"Null":{"s3:prefix":"false"}}`,
+ `{"StringEqualsIfExists":{"aws:username":"alice"}}`,
+ `{"NotIpAddressIfExists":{"aws:SourceIp":"10.0.0.0/8"}}`,
+ `{"BoolIfExists":{"aws:SecureTransport":"true"}}`,
+ } {
+ _, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Action":"s3:*","Resource":"*","Condition":` + cond + `}]}`)
+ assert.NoError(t, err, cond)
+ }
+}
+
+// Date values take either W3C ISO 8601 or epoch seconds, the latter as a JSON
+// number too, on both date-valued keys.
+func TestValidatePolicyDocument_DateOperators(t *testing.T) {
+ t.Parallel()
+ for _, cond := range []string{
+ `{"DateGreaterThan":{"aws:CurrentTime":"2026-10-01T12:00:00Z"}}`,
+ `{"DateLessThan":{"aws:CurrentTime":"2027-01-01"}}`,
+ `{"DateEquals":{"aws:EpochTime":"2026-10-01"}}`,
+ `{"DateNotEquals":{"aws:EpochTime":1790856000}}`,
+ `{"DateLessThanEquals":{"aws:EpochTime":["1790856000","2026-10-01T22:00+10:00"]}}`,
+ `{"DateGreaterThanEqualsIfExists":{"aws:CurrentTime":"2026-10-01T12:00:00.5Z"}}`,
+ `{"Null":{"aws:CurrentTime":"false"}}`,
+ } {
+ _, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow",
+ "Action":"s3:*","Resource":"*","Condition":` + cond + `}]}`)
+ assert.NoError(t, err, cond)
+ }
+}
+
func TestValidatePolicyDocument_UnsupportedConditionOperator(t *testing.T) {
t.Parallel()
_, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*",
@@ -2196,12 +2355,62 @@ func TestValidatePolicyDocument_RejectsPrincipal(t *testing.T) {
assert.Contains(t, err.Error(), "Principal is not valid on an identity policy")
}
-func TestValidatePolicyDocument_RejectsNotAction(t *testing.T) {
+func TestValidatePolicyDocument_Selectors(t *testing.T) {
t.Parallel()
- _, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"*","Resource":"*",
- "NotAction":"sts:AssumeRole"}]}`)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "NotAction blocks are not supported")
+ tests := []struct {
+ name string
+ stmt string
+ wantErr string
+ }{
+ {"NotAction", `"NotAction":"iam:*","Resource":"*"`, ""},
+ {"NotResource", `"Action":"s3:*","NotResource":["arn:aws:s3:::secret","arn:aws:s3:::secret/*"]`, ""},
+ {"NotAction and NotResource", `"NotAction":"iam:*","NotResource":"arn:aws:s3:::secret/*"`, ""},
+ {"NotResource with a variable", `"Action":"s3:*","NotResource":"arn:aws:s3:::home/${aws:username}/*"`, ""},
+ {"Action and NotAction", `"Action":"*","NotAction":"sts:AssumeRole","Resource":"*"`, "Action and NotAction cannot both be specified"},
+ {"Resource and NotResource", `"Action":"*","Resource":"*","NotResource":"arn:aws:s3:::public/*"`, "Resource and NotResource cannot both be specified"},
+ {"NotResource with an unknown variable", `"Action":"s3:*","NotResource":"arn:aws:s3:::${aws:PrincipalOrgID}/*"`, "NotResource"},
+ {"empty NotAction entry", `"NotAction":"","Resource":"*"`, "must be prefaced by a vendor"},
+ {"NotAction without a vendor", `"NotAction":["iam"],"Resource":"*"`, "must be prefaced by a vendor"},
+ {"NotAction with an empty name", `"NotAction":"iam:","Resource":"*"`, "must be prefaced by a vendor"},
+ {"Action without a vendor", `"Action":"RunInstances","Resource":"*"`, "must be prefaced by a vendor"},
+ {"empty NotResource entry", `"Action":"s3:*","NotResource":[""]`, "must be in ARN format"},
+ {"NotResource not an ARN", `"Action":"s3:*","NotResource":"secret-bucket"`, "must be in ARN format"},
+ {"Resource not an ARN", `"Action":"s3:*","Resource":"secret-bucket"`, "must be in ARN format"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ for _, effect := range []string{"Allow", "Deny"} {
+ _, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"` + effect + `",` + tc.stmt + `}]}`)
+ if tc.wantErr == "" {
+ assert.NoError(t, err, effect)
+ } else {
+ require.Error(t, err, effect)
+ assert.Contains(t, err.Error(), tc.wantErr, effect)
+ }
+ }
+ })
+ }
+}
+
+// NotAction reaches CreatePolicy as an accepted identity policy, and both
+// selectors together come back as MalformedPolicyDocument.
+func TestCreatePolicy_NotAction(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+
+ _, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("PowerUser"),
+ PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","NotAction":"iam:*","Resource":"*"}]}`),
+ })
+ require.NoError(t, err)
+
+ _, err = svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("Both"),
+ PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","NotAction":"iam:*","Resource":"*"}]}`),
+ })
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), awserrors.ErrorIAMMalformedPolicyDocument)
}
// An allowlisted operator over a malformed value is still an inert grant: the
@@ -2220,6 +2429,20 @@ func TestValidatePolicyDocument_RejectsMalformedConditionValues(t *testing.T) {
{"numeric bool", `{"Bool":{"aws:SecureTransport":1}}`, "not true or false"},
{"yes", `{"Bool":{"aws:SecureTransport":"yes"}}`, "not true or false"},
{"empty array", `{"IpAddress":{"aws:SourceIp":[]}}`, "has no value"},
+ {"NotIpAddress hostname", `{"NotIpAddress":{"aws:SourceIp":"office.example.com"}}`, "not a valid IP address or CIDR block"},
+ {"NotIpAddress empty array", `{"NotIpAddress":{"aws:SourceIp":[]}}`, "has no value"},
+ {"Null yes", `{"Null":{"aws:username":"yes"}}`, "not true or false"},
+ {"Null variable", `{"Null":{"aws:username":"${aws:username}"}}`, "not true or false"},
+ {"IpAddressIfExists hostname", `{"IpAddressIfExists":{"aws:SourceIp":"office.example.com"}}`, "not a valid IP address or CIDR block"},
+ {"BoolIfExists yes", `{"BoolIfExists":{"aws:SecureTransport":"yes"}}`, "not true or false"},
+ {"StringEqualsIfExists unknown variable", `{"StringEqualsIfExists":{"aws:username":"${aws:bogus}"}}`, `references policy variable "aws:bogus"`},
+ {"NullIfExists", `{"NullIfExists":{"aws:username":"true"}}`, "is not supported in this release"},
+ {"DateGreaterThan prose", `{"DateGreaterThan":{"aws:CurrentTime":"next week"}}`, "not an ISO 8601 date or epoch seconds"},
+ {"DateLessThan no time zone", `{"DateLessThan":{"aws:CurrentTime":"2026-10-01T12:00:00"}}`, "not an ISO 8601 date or epoch seconds"},
+ {"DateEquals fractional epoch", `{"DateEquals":{"aws:EpochTime":"1790856000.5"}}`, "not an ISO 8601 date or epoch seconds"},
+ {"DateEquals variable", `{"DateEquals":{"aws:CurrentTime":"${aws:CurrentTime}"}}`, "not an ISO 8601 date or epoch seconds"},
+ {"DateLessThanIfExists empty", `{"DateLessThanIfExists":{"aws:EpochTime":""}}`, "not an ISO 8601 date or epoch seconds"},
+ {"DateGreaterThan on a string key", `{"DateGreaterThan":{"aws:username":"2026"}}`, "is not supported in this release"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -2232,6 +2455,38 @@ func TestValidatePolicyDocument_RejectsMalformedConditionValues(t *testing.T) {
}
}
+// No supported key carries an Arn operator yet, so the write path cannot reach
+// this arm; it is exercised directly so the key that unlocks it lands validated.
+func TestValidateConditionValues_ArnOperators(t *testing.T) {
+ t.Parallel()
+ const key = "aws:SourceArn"
+ tests := []struct {
+ name string
+ value string
+ wantErr string
+ }{
+ {"exact ARN", "arn:aws:ecs:us-east-1:111122223333:task/c/abc", ""},
+ {"wildcard components", "arn:aws:ecs:*:111122223333:*", ""},
+ {"resource with colons", "arn:aws:logs:us-east-1:111122223333:log-group:app:*", ""},
+ {"variable", "arn:aws:iam::${aws:PrincipalAccount}:role/*", ""},
+ {"too few components", "arn:aws:ecs:*", "not an ARN of six colon-separated components"},
+ {"bare wildcard", "*", "not an ARN of six colon-separated components"},
+ {"unknown variable", "arn:aws:iam::${aws:bogus}:role/*", `references policy variable "aws:bogus"`},
+ }
+ for _, op := range []string{
+ iampolicy.OpArnEquals, iampolicy.OpArnLike, iampolicy.OpArnNotEquals, iampolicy.OpArnNotLike,
+ } {
+ for _, tt := range tests {
+ err := validateConditionValues(0, op, key, ConditionValue{tt.value})
+ if tt.wantErr == "" {
+ assert.NoError(t, err, "%s %s", op, tt.name)
+ continue
+ }
+ assert.ErrorContains(t, err, tt.wantErr, "%s %s", op, tt.name)
+ }
+ }
+}
+
// A reference the evaluator cannot resolve makes an Allow select nothing and a
// Deny select everything it named, so the policy does not do what it reads as.
// Rejecting it on write is the only place an operator can be told.
@@ -2250,6 +2505,15 @@ func TestValidatePolicyDocument_RejectsUnresolvableVariables(t *testing.T) {
{"MFA in a StringLike value",
`"Action":"s3:*","Resource":"*","Condition":{"StringLike":{"s3:prefix":"${aws:MultiFactorAuthPresent}/*"}}`,
`references policy variable "aws:MultiFactorAuthPresent"`},
+ {"MFA in a StringNotEquals value",
+ `"Action":"s3:*","Resource":"*","Condition":{"StringNotEquals":{"aws:username":"${aws:MultiFactorAuthPresent}"}}`,
+ `references policy variable "aws:MultiFactorAuthPresent"`},
+ {"MFA in a StringNotLike value",
+ `"Action":"s3:*","Resource":"*","Condition":{"StringNotLike":{"s3:prefix":"${aws:MultiFactorAuthPresent}/*"}}`,
+ `references policy variable "aws:MultiFactorAuthPresent"`},
+ {"unterminated in a StringEqualsIgnoreCase value",
+ `"Action":"s3:*","Resource":"*","Condition":{"StringEqualsIgnoreCase":{"aws:username":"${aws:username"}}`,
+ "unterminated policy variable reference"},
{"unknown key", `"Action":"s3:*","Resource":"arn:aws:s3:::home/${aws:bogus}/*"`,
`references policy variable "aws:bogus"`},
{"condition key that is not substitutable",
@@ -2322,14 +2586,6 @@ func TestValidatePolicyDocument_RejectsUnsupportedAlongsideSupported(t *testing.
assert.Contains(t, err.Error(), "aws:PrincipalOrgID")
}
-func TestValidatePolicyDocument_RejectsNotResource(t *testing.T) {
- t.Parallel()
- _, err := ValidatePolicyDocument(`{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"*","Resource":"*",
- "NotResource":"arn:aws:s3:::public/*"}]}`)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "NotResource blocks are not supported")
-}
-
// ============================================================================
// Sensitive Data Not Logged Tests
// ============================================================================
@@ -2381,7 +2637,7 @@ func TestSensitiveDataNotLogged_MasterKey(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
masterKey, err := GenerateMasterKey()
require.NoError(t, err)
- _, err = NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ _, err = NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
logOutput := buf.String()
@@ -2460,8 +2716,8 @@ func TestInputValidation_PathLength(t *testing.T) {
func TestInputValidation_PolicyDocumentSize(t *testing.T) {
t.Parallel()
// 6144 bytes — should pass
- filler6144 := strings.Repeat("a", 6144-len(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"`)-len(`"}]}`))
- doc6144 := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"` + filler6144 + `"}]}`
+ filler6144 := strings.Repeat("a", 6144-len(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"arn:aws:s3:::`)-len(`"}]}`))
+ doc6144 := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"arn:aws:s3:::` + filler6144 + `"}]}`
assert.Len(t, doc6144, 6144)
_, err := ValidatePolicyDocument(doc6144)
assert.NoError(t, err, "6144-byte policy document should be valid")
@@ -2584,7 +2840,7 @@ func TestIsIAMNameChar(t *testing.T) {
func TestValidateIAMName(t *testing.T) {
t.Parallel()
const charset = "It must contain only alphanumeric characters and/or the following: +=,.@_-"
- invalid, validation := awserrors.ErrorIAMInvalidInput, awserrors.ErrorValidationError
+ validation := awserrors.ErrorValidationError
tests := []struct {
name string
field string
@@ -2598,10 +2854,10 @@ func TestValidateIAMName(t *testing.T) {
{"valid single char", "userName", "a", 64, "", ""},
{"valid at max length", "userName", strings.Repeat("a", 64), 64, "", ""},
{"valid policy at max length", "policyName", strings.Repeat("x", 128), 128, "", ""},
- {"empty", "userName", "", 64, invalid,
- "1 validation error detected: Value '' at 'userName' failed to satisfy constraint: Member must have length greater than or equal to 1"},
- {"too long", "policyName", strings.Repeat("x", 129), 128, invalid,
- "1 validation error detected: Value '" + strings.Repeat("x", 129) + "' at 'policyName' failed to satisfy constraint: Member must have length less than or equal to 128"},
+ {"empty", "userName", "", 64, validation,
+ "1 validation error detected: Value at 'userName' failed to satisfy constraint: Member must have length greater than or equal to 1"},
+ {"too long", "policyName", strings.Repeat("x", 129), 128, validation,
+ "1 validation error detected: Value at 'policyName' failed to satisfy constraint: Member must have length less than or equal to 128"},
{"space", "userName", "bad name!", 64, validation, "The specified value for userName is invalid. " + charset},
{"slash", "roleName", "alice/bob", 64, validation, "The specified value for roleName is invalid. " + charset},
{"colon", "groupName", "alice:bob", 128, validation, "The specified value for groupName is invalid. " + charset},
@@ -2632,7 +2888,7 @@ func TestValidatePath(t *testing.T) {
t.Parallel()
const shape = "The specified value for path is invalid. It must begin and end with / and contain only alphanumeric characters and/or / characters."
tooLong := "/" + strings.Repeat("a", 511) + "/"
- invalid, validation := awserrors.ErrorIAMInvalidInput, awserrors.ErrorValidationError
+ validation := awserrors.ErrorValidationError
tests := []struct {
name string
input string
@@ -2646,8 +2902,8 @@ func TestValidatePath(t *testing.T) {
{"empty string", "", validation, shape},
{"just text", "noslash", validation, shape},
{"max length 512", "/" + strings.Repeat("a", 510) + "/", "", ""},
- {"over max length 513", tooLong, invalid,
- "1 validation error detected: Value '" + tooLong + "' at 'path' failed to satisfy constraint: Member must have length less than or equal to 512"},
+ {"over max length 513", tooLong, validation,
+ "1 validation error detected: Value at 'path' failed to satisfy constraint: Member must have length less than or equal to 512"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/spinifex/handlers/iam/tags_test.go b/spinifex/handlers/iam/tags_test.go
index 925e619c8..cadbc7427 100644
--- a/spinifex/handlers/iam/tags_test.go
+++ b/spinifex/handlers/iam/tags_test.go
@@ -30,22 +30,34 @@ func TestValidateTags(t *testing.T) {
}{
{"nil slice", nil, ""},
{"valid tags", []*iam.Tag{sdkTag("env", "prod"), sdkTag("team", "")}, ""},
+ {"every allowed symbol", []*iam.Tag{sdkTag("ok_.:/=+-@ key", "ok_.:/=+-@ value")}, ""},
+ {"non-ASCII letters", []*iam.Tag{sdkTag("ключ", "значение")}, ""},
{"max key length", []*iam.Tag{sdkTag(strings.Repeat("k", maxTagKeyLength), "v")}, ""},
{"max value length", []*iam.Tag{sdkTag("k", strings.Repeat("v", maxTagValueLength))}, ""},
- {"nil value allowed", []*iam.Tag{{Key: aws.String("k")}}, ""},
- {"over 50 tags", tooMany, awserrors.ErrorIAMLimitExceeded},
- {"nil tag entry", []*iam.Tag{nil}, awserrors.ErrorIAMInvalidInput},
- {"nil key", []*iam.Tag{{Value: aws.String("v")}}, awserrors.ErrorIAMInvalidInput},
- {"empty key", []*iam.Tag{sdkTag("", "v")}, awserrors.ErrorIAMInvalidInput},
- {"over-length key", []*iam.Tag{sdkTag(strings.Repeat("k", maxTagKeyLength+1), "v")}, awserrors.ErrorIAMInvalidInput},
- {"over-length value", []*iam.Tag{sdkTag("k", strings.Repeat("v", maxTagValueLength+1))}, awserrors.ErrorIAMInvalidInput},
+ {"max key length in multibyte characters", []*iam.Tag{sdkTag(strings.Repeat("é", maxTagKeyLength), "v")}, ""},
+ {"max value length in multibyte characters", []*iam.Tag{sdkTag("k", strings.Repeat("é", maxTagValueLength))}, ""},
+ {"over 50 tags", tooMany, awserrors.ErrorValidationError},
+ {"reserved aws: prefix", []*iam.Tag{sdkTag("aws:thing", "v")}, awserrors.ErrorIAMInvalidInput},
+ {"reserved prefix in any case", []*iam.Tag{sdkTag("AWS:thing", "v")}, awserrors.ErrorIAMInvalidInput},
+ {"aws prefix without colon", []*iam.Tag{sdkTag("awsthing", "v")}, ""},
+ {"constraint break wins over reserved prefix", []*iam.Tag{sdkTag("aws:thing", "v"), sdkTag("bad#", "v")}, awserrors.ErrorValidationError},
+ {"nil tag entry", []*iam.Tag{nil}, awserrors.ErrorValidationError},
+ {"nil key", []*iam.Tag{{Value: aws.String("v")}}, awserrors.ErrorValidationError},
+ {"nil value", []*iam.Tag{{Key: aws.String("k")}}, awserrors.ErrorValidationError},
+ {"empty key", []*iam.Tag{sdkTag("", "v")}, awserrors.ErrorValidationError},
+ {"over-length key", []*iam.Tag{sdkTag(strings.Repeat("é", maxTagKeyLength+1), "v")}, awserrors.ErrorValidationError},
+ {"over-length value", []*iam.Tag{sdkTag("k", strings.Repeat("é", maxTagValueLength+1))}, awserrors.ErrorValidationError},
+ {"key outside pattern", []*iam.Tag{sdkTag("bad#key", "v")}, awserrors.ErrorValidationError},
+ {"value outside pattern", []*iam.Tag{sdkTag("k", "bad\nvalue")}, awserrors.ErrorValidationError},
{"duplicate keys", []*iam.Tag{sdkTag("k", "1"), sdkTag("k", "2")}, awserrors.ErrorIAMInvalidInput},
+ {"duplicate keys differing in case", []*iam.Tag{sdkTag("Dup", "1"), sdkTag("dup", "2")}, awserrors.ErrorIAMInvalidInput},
+ {"constraint break wins over duplicate", []*iam.Tag{sdkTag("k", "1"), sdkTag("K", "2"), sdkTag("bad#", "v")}, awserrors.ErrorValidationError},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- err := validateTags(tc.tags)
+ err := validateTags(tc.tags, foldedKeys)
if tc.wantErr == "" {
require.NoError(t, err)
return
@@ -58,28 +70,104 @@ func TestValidateTags(t *testing.T) {
}
}
-func TestValidateTags_NamesFailingMember(t *testing.T) {
+// The messages are the ones AWS returned to TagUser for the same input.
+func TestValidateTags_Messages(t *testing.T) {
t.Parallel()
cases := []struct {
- name string
- tags []*iam.Tag
- wantMsg string
+ name string
+ tags []*iam.Tag
+ wantCode string
+ wantMsg string
}{
- {"nil key", []*iam.Tag{sdkTag("a", "1"), {Value: aws.String("v")}},
- "1 validation error detected: Value null at 'tags.2.member.key' failed to satisfy constraint: Member must not be null"},
- {"over-length value", []*iam.Tag{sdkTag("a", "1"), sdkTag("b", "2"), sdkTag("k", strings.Repeat("v", maxTagValueLength+1))},
- "1 validation error detected: Value '" + strings.Repeat("v", maxTagValueLength+1) + "' at 'tags.3.member.value' failed to satisfy constraint: Member must have length less than or equal to 256"},
- {"duplicate keys", []*iam.Tag{sdkTag("k", "1"), sdkTag("k", "2")},
+ {"nil value", []*iam.Tag{{Key: aws.String("novalue")}}, awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'tags.1.member.value' failed to satisfy constraint: Member must not be null"},
+ {"key outside pattern", []*iam.Tag{sdkTag("bad#key", "v")}, awserrors.ErrorValidationError,
+ `1 validation error detected: Value at 'tags.1.member.key' failed to satisfy constraint: Member must satisfy regular expression pattern: [\p{L}\p{Z}\p{N}_.:/=+\-@]+`},
+ {"value outside pattern", []*iam.Tag{sdkTag("k", "bad#value")}, awserrors.ErrorValidationError,
+ `1 validation error detected: Value at 'tags.1.member.value' failed to satisfy constraint: Member must satisfy regular expression pattern: [\p{L}\p{Z}\p{N}_.:/=+\-@]*`},
+ {"over-length key", []*iam.Tag{sdkTag(strings.Repeat("é", maxTagKeyLength+1), "v")}, awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'tags.1.member.key' failed to satisfy constraint: Member must have length less than or equal to 128"},
+ {"over-length value", []*iam.Tag{sdkTag("k", strings.Repeat("é", maxTagValueLength+1))}, awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'tags.1.member.value' failed to satisfy constraint: Member must have length less than or equal to 256"},
+ {"empty key breaks length and pattern", []*iam.Tag{sdkTag("", "v")}, awserrors.ErrorValidationError,
+ "2 validation errors detected: Value at 'tags.1.member.key' failed to satisfy constraint: Member must have length greater than or equal to 1; " +
+ `Value at 'tags.1.member.key' failed to satisfy constraint: Member must satisfy regular expression pattern: [\p{L}\p{Z}\p{N}_.:/=+\-@]+`},
+ {"names the failing member", []*iam.Tag{sdkTag("a", "1"), {Value: aws.String("v")}}, awserrors.ErrorValidationError,
+ "1 validation error detected: Value at 'tags.2.member.key' failed to satisfy constraint: Member must not be null"},
+ {"duplicate keys differing in case", []*iam.Tag{sdkTag("Dup", "1"), sdkTag("dup", "2")}, awserrors.ErrorIAMInvalidInput,
"Duplicate tag keys found. Please note that Tag keys are case insensitive."},
+ {"reserved aws: prefix", []*iam.Tag{sdkTag("aws:thing", "v")}, awserrors.ErrorIAMInvalidInput,
+ "Tag keys beginning with aws: are reserved for system use."},
+ {"over 50 tags with a bad key", append(tagsNamed(maxTagsPerResource), sdkTag("bad#key", "v")), awserrors.ErrorValidationError,
+ "2 validation errors detected: Value at 'tags' failed to satisfy constraint: Member must have length less than or equal to 50; " +
+ `Value at 'tags.51.member.key' failed to satisfy constraint: Member must satisfy regular expression pattern: [\p{L}\p{Z}\p{N}_.:/=+\-@]+`},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- requireIAMError(t, validateTags(tc.tags), awserrors.ErrorIAMInvalidInput, tc.wantMsg)
+ requireIAMError(t, validateTags(tc.tags, foldedKeys), tc.wantCode, tc.wantMsg)
})
}
}
+func TestValidateTagKeys(t *testing.T) {
+ t.Parallel()
+ keySet := `Value at 'tagKeys' failed to satisfy constraint: Member must satisfy constraint: [Member must have length less than or equal to 128, ` +
+ `Member must have length greater than or equal to 1, Member must satisfy regular expression pattern: [\p{L}\p{Z}\p{N}_.:/=+\-@]+, Member must not be null]`
+ tooMany := make([]*string, maxTagsPerResource+1)
+ for i := range tooMany {
+ tooMany[i] = aws.String(fmt.Sprintf("k%d", i))
+ }
+ cases := []struct {
+ name string
+ keys []*string
+ wantMsg string
+ }{
+ {"valid keys", []*string{aws.String("env"), aws.String(strings.Repeat("é", maxTagKeyLength))}, ""},
+ {"empty key", []*string{aws.String("")}, "1 validation error detected: " + keySet},
+ {"key outside pattern", []*string{aws.String("bad#key")}, "1 validation error detected: " + keySet},
+ {"over-length key", []*string{aws.String(strings.Repeat("k", maxTagKeyLength+1))}, "1 validation error detected: " + keySet},
+ {"several bad keys are one violation", []*string{aws.String("bad#"), aws.String("k"), aws.String("bad*")}, "1 validation error detected: " + keySet},
+ {"nil key", []*string{nil}, "1 validation error detected: " + keySet},
+ {"over 50 keys", tooMany,
+ "1 validation error detected: Value at 'tagKeys' failed to satisfy constraint: Member must have length less than or equal to 50"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ err := validateTagKeys(tc.keys)
+ if tc.wantMsg == "" {
+ require.NoError(t, err)
+ return
+ }
+ requireIAMError(t, err, awserrors.ErrorValidationError, tc.wantMsg)
+ })
+ }
+}
+
+// AWS checks the keys before looking the resource up.
+func TestUntag_InvalidKeyRefusedBeforeLookup(t *testing.T) {
+ t.Parallel()
+ for _, ops := range allTagOps() {
+ t.Run(ops.resource, func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ err := ops.untag(svc, ops.missingID, []*string{aws.String("bad#key")})
+ code, ok := awserrors.ResolveErrorCode(err)
+ require.True(t, ok, "error must carry a registered code: %v", err)
+ assert.Equal(t, awserrors.ErrorValidationError, code)
+ })
+ }
+}
+
+func tagsNamed(n int) []*iam.Tag {
+ tags := make([]*iam.Tag, n)
+ for i := range tags {
+ tags[i] = sdkTag(fmt.Sprintf("k%d", i), "v")
+ }
+ return tags
+}
+
func TestMergeTags(t *testing.T) {
t.Parallel()
cases := []struct {
@@ -95,6 +183,12 @@ func TestMergeTags(t *testing.T) {
[]*iam.Tag{sdkTag("a", "9"), sdkTag("c", "3")},
[]Tag{{Key: "a", Value: "9"}, {Key: "b", Value: "2"}, {Key: "c", Value: "3"}},
},
+ {
+ "upsert ignores case and takes the new key",
+ []Tag{{Key: "Env", Value: "base"}, {Key: "b", Value: "2"}},
+ []*iam.Tag{sdkTag("env", "lower")},
+ []Tag{{Key: "env", Value: "lower"}, {Key: "b", Value: "2"}},
+ },
{
"nil key skipped",
[]Tag{{Key: "a", Value: "1"}},
@@ -112,16 +206,28 @@ func TestMergeTags(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- got := mergeTags(tc.existing, tc.add)
+ got := mergeTags(tc.existing, tc.add, foldedKeys)
assert.Equal(t, tc.want, got)
})
}
}
+func TestMergeAndRemoveTags_ExactKeys(t *testing.T) {
+ t.Parallel()
+ existing := []Tag{{Key: "Env", Value: "base"}}
+ merged := mergeTags(existing, []*iam.Tag{sdkTag("env", "lower")}, exactKeys)
+ assert.Equal(t, []Tag{{Key: "Env", Value: "base"}, {Key: "env", Value: "lower"}}, merged)
+ assert.Equal(t, merged, removeTagKeys(merged, []*string{aws.String("ENV")}, exactKeys))
+ assert.Equal(t, []Tag{{Key: "env", Value: "lower"}}, removeTagKeys(merged, []*string{aws.String("Env")}, exactKeys))
+ require.NoError(t, validateTags([]*iam.Tag{sdkTag("Dup", "1"), sdkTag("dup", "2")}, exactKeys))
+ requireIAMError(t, validateTags([]*iam.Tag{sdkTag("dup", "1"), sdkTag("dup", "2")}, exactKeys), awserrors.ErrorIAMInvalidInput,
+ "Duplicate tag keys found. Please note that Tag keys are case insensitive.")
+}
+
func TestMergeTags_DoesNotMutateExisting(t *testing.T) {
t.Parallel()
existing := []Tag{{Key: "a", Value: "1"}}
- _ = mergeTags(existing, []*iam.Tag{sdkTag("a", "9")})
+ _ = mergeTags(existing, []*iam.Tag{sdkTag("a", "9")}, foldedKeys)
assert.Equal(t, []Tag{{Key: "a", Value: "1"}}, existing)
}
@@ -135,6 +241,7 @@ func TestRemoveTagKeys(t *testing.T) {
want []Tag
}{
{"remove one", []*string{aws.String("b")}, []Tag{{Key: "a", Value: "1"}, {Key: "c", Value: "3"}}},
+ {"key case ignored", []*string{aws.String("B")}, []Tag{{Key: "a", Value: "1"}, {Key: "c", Value: "3"}}},
{"unknown key ignored", []*string{aws.String("zzz")}, existing},
{"nil key ignored", []*string{nil, aws.String("a")}, []Tag{{Key: "b", Value: "2"}, {Key: "c", Value: "3"}}},
{"remove all", []*string{aws.String("a"), aws.String("b"), aws.String("c")}, []Tag{}},
@@ -143,7 +250,7 @@ func TestRemoveTagKeys(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- got := removeTagKeys(existing, tc.keys)
+ got := removeTagKeys(existing, tc.keys, foldedKeys)
assert.Equal(t, tc.want, got)
})
}
@@ -152,21 +259,29 @@ func TestRemoveTagKeys(t *testing.T) {
// tagOps abstracts the per-resource tag/untag/list triple so the round-trip
// behaviour is asserted identically for every taggable resource.
type tagOps struct {
- resource string
- create func(t *testing.T, svc *IAMServiceImpl) string
- missingID string
- tag func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error
- untag func(svc *IAMServiceImpl, id string, keys []*string) error
- list func(svc *IAMServiceImpl, id string) ([]*iam.Tag, error)
+ resource string
+ keys keyCase
+ create func(t *testing.T, svc *IAMServiceImpl) string
+ // createTagged creates the resource with tags and returns the ID it would have.
+ createTagged func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error)
+ missingID string
+ tag func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error
+ untag func(svc *IAMServiceImpl, id string, keys []*string) error
+ list func(svc *IAMServiceImpl, id string) ([]*iam.Tag, error)
}
func allTagOps() []tagOps {
return []tagOps{
{
resource: "user",
+ keys: foldedKeys,
create: func(t *testing.T, svc *IAMServiceImpl) string {
return *createTestUser(t, svc, "tagme").UserName
},
+ createTagged: func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error) {
+ _, err := svc.CreateUser(testAccountID, &iam.CreateUserInput{UserName: aws.String("tagged"), Tags: tags})
+ return "tagged", err
+ },
missingID: "no-such-user",
tag: func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error {
_, err := svc.TagUser(testAccountID, &iam.TagUserInput{UserName: aws.String(id), Tags: tags})
@@ -186,9 +301,16 @@ func allTagOps() []tagOps {
},
{
resource: "role",
+ keys: foldedKeys,
create: func(t *testing.T, svc *IAMServiceImpl) string {
return *createTestRole(t, svc, "tagme").RoleName
},
+ createTagged: func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error) {
+ _, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+ RoleName: aws.String("tagged"), AssumeRolePolicyDocument: aws.String(validTrustPolicy()), Tags: tags,
+ })
+ return "tagged", err
+ },
missingID: "no-such-role",
tag: func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error {
_, err := svc.TagRole(testAccountID, &iam.TagRoleInput{RoleName: aws.String(id), Tags: tags})
@@ -208,6 +330,7 @@ func allTagOps() []tagOps {
},
{
resource: "policy",
+ keys: exactKeys,
create: func(t *testing.T, svc *IAMServiceImpl) string {
out, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
PolicyName: aws.String("tagme"),
@@ -216,6 +339,12 @@ func allTagOps() []tagOps {
require.NoError(t, err)
return *out.Policy.Arn
},
+ createTagged: func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error) {
+ _, err := svc.CreatePolicy(testAccountID, &iam.CreatePolicyInput{
+ PolicyName: aws.String("tagged"), PolicyDocument: aws.String(validPolicyDocument()), Tags: tags,
+ })
+ return "arn:aws:iam::" + testAccountID + ":policy/tagged", err
+ },
missingID: "arn:aws:iam::" + testAccountID + ":policy/no-such-policy",
tag: func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error {
_, err := svc.TagPolicy(testAccountID, &iam.TagPolicyInput{PolicyArn: aws.String(id), Tags: tags})
@@ -235,9 +364,16 @@ func allTagOps() []tagOps {
},
{
resource: "instance profile",
+ keys: exactKeys,
create: func(t *testing.T, svc *IAMServiceImpl) string {
return *createTestInstanceProfile(t, svc, "tagme").InstanceProfileName
},
+ createTagged: func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error) {
+ _, err := svc.CreateInstanceProfile(testAccountID, &iam.CreateInstanceProfileInput{
+ InstanceProfileName: aws.String("tagged"), Tags: tags,
+ })
+ return "tagged", err
+ },
missingID: "no-such-profile",
tag: func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error {
_, err := svc.TagInstanceProfile(testAccountID, &iam.TagInstanceProfileInput{InstanceProfileName: aws.String(id), Tags: tags})
@@ -257,6 +393,7 @@ func allTagOps() []tagOps {
},
{
resource: "OIDC provider",
+ keys: exactKeys,
create: func(t *testing.T, svc *IAMServiceImpl) string {
out, err := svc.CreateOpenIDConnectProvider(testAccountID, &iam.CreateOpenIDConnectProviderInput{
Url: aws.String("https://oidc.example.com/id/TAGME"),
@@ -264,6 +401,12 @@ func allTagOps() []tagOps {
require.NoError(t, err)
return *out.OpenIDConnectProviderArn
},
+ createTagged: func(svc *IAMServiceImpl, tags []*iam.Tag) (string, error) {
+ _, err := svc.CreateOpenIDConnectProvider(testAccountID, &iam.CreateOpenIDConnectProviderInput{
+ Url: aws.String("https://oidc.example.com/id/TAGGED"), Tags: tags,
+ })
+ return "arn:aws:iam::" + testAccountID + ":oidc-provider/oidc.example.com/id/TAGGED", err
+ },
missingID: "arn:aws:iam::" + testAccountID + ":oidc-provider/oidc.example.com/id/MISSING",
tag: func(svc *IAMServiceImpl, id string, tags []*iam.Tag) error {
_, err := svc.TagOpenIDConnectProvider(testAccountID, &iam.TagOpenIDConnectProviderInput{OpenIDConnectProviderArn: aws.String(id), Tags: tags})
@@ -292,6 +435,91 @@ func tagsAsMap(tags []*iam.Tag) map[string]string {
return m
}
+func TestCreate_InvalidTagsRefusedAndNotStored(t *testing.T) {
+ t.Parallel()
+ cases := []struct {
+ name string
+ tags []*iam.Tag
+ wantCode string
+ }{
+ {"nil value", []*iam.Tag{{Key: aws.String("k")}}, awserrors.ErrorValidationError},
+ {"key outside pattern", []*iam.Tag{sdkTag("bad#key", "v")}, awserrors.ErrorValidationError},
+ {"duplicate keys", []*iam.Tag{sdkTag("dup", "1"), sdkTag("dup", "2")}, awserrors.ErrorIAMInvalidInput},
+ {"reserved aws: prefix", []*iam.Tag{sdkTag("aws:thing", "v")}, awserrors.ErrorIAMInvalidInput},
+ }
+ for _, ops := range allTagOps() {
+ for _, tc := range cases {
+ t.Run(ops.resource+"/"+tc.name, func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ id, err := ops.createTagged(svc, tc.tags)
+ require.Error(t, err)
+ code, ok := awserrors.ResolveErrorCode(err)
+ require.True(t, ok, "error must carry a registered code: %v", err)
+ assert.Equal(t, tc.wantCode, code)
+
+ _, err = ops.list(svc, id)
+ require.Error(t, err)
+ assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+ })
+ }
+ }
+}
+
+func TestCreate_ValidTagsStored(t *testing.T) {
+ t.Parallel()
+ for _, ops := range allTagOps() {
+ t.Run(ops.resource, func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ id, err := ops.createTagged(svc, []*iam.Tag{sdkTag("env", "prod"), sdkTag("team", "")})
+ require.NoError(t, err)
+ got, err := ops.list(svc, id)
+ require.NoError(t, err)
+ assert.Equal(t, map[string]string{"env": "prod", "team": ""}, tagsAsMap(got))
+ })
+ }
+}
+
+// AWS compares tag keys ignoring case on users and roles only: elsewhere Dup
+// and dup are two tags, and neither tagging nor untagging in another case
+// touches the stored key.
+func TestResourceTagging_KeyCase(t *testing.T) {
+ t.Parallel()
+ for _, ops := range allTagOps() {
+ t.Run(ops.resource, func(t *testing.T) {
+ t.Parallel()
+ svc := setupTestIAMService(t)
+ id := ops.create(t, svc)
+
+ err := ops.tag(svc, id, []*iam.Tag{sdkTag("Dup", "1"), sdkTag("dup", "2")})
+ if ops.keys == foldedKeys {
+ requireIAMError(t, err, awserrors.ErrorIAMInvalidInput, "Duplicate tag keys found. Please note that Tag keys are case insensitive.")
+ } else {
+ require.NoError(t, err)
+ }
+
+ require.NoError(t, ops.tag(svc, id, []*iam.Tag{sdkTag("Case", "upper")}))
+ require.NoError(t, ops.tag(svc, id, []*iam.Tag{sdkTag("case", "lower")}))
+ got, err := ops.list(svc, id)
+ require.NoError(t, err)
+ want := map[string]string{"Dup": "1", "dup": "2", "Case": "upper", "case": "lower"}
+ if ops.keys == foldedKeys {
+ want = map[string]string{"case": "lower"}
+ }
+ assert.Equal(t, want, tagsAsMap(got))
+
+ require.NoError(t, ops.untag(svc, id, []*string{aws.String("CASE")}))
+ got, err = ops.list(svc, id)
+ require.NoError(t, err)
+ if ops.keys == foldedKeys {
+ want = map[string]string{}
+ }
+ assert.Equal(t, want, tagsAsMap(got))
+ })
+ }
+}
+
func TestResourceTagging_RoundTrip(t *testing.T) {
t.Parallel()
for _, ops := range allTagOps() {
diff --git a/spinifex/handlers/ochrevector/backend.go b/spinifex/handlers/ochrevector/backend.go
index 0bbd16033..9f18e89c4 100644
--- a/spinifex/handlers/ochrevector/backend.go
+++ b/spinifex/handlers/ochrevector/backend.go
@@ -27,11 +27,11 @@ type VectorRow struct {
// stored metadata, source provenance, and cosine similarity Score in [0,1]
// (D8; 1 - cosine_distance).
type QueryResult struct {
- Chunk string
- Metadata map[string]any
- SourceKey string
- SourceOffset int
- Score float32
+ Chunk string `json:"Chunk"`
+ Metadata map[string]any `json:"Metadata"`
+ SourceKey string `json:"SourceKey"`
+ SourceOffset int `json:"SourceOffset"`
+ Score float32 `json:"Score"`
}
// defaultQueryK and maxQueryK bound Query's k (D8/D10): k<=0 defaults to
diff --git a/spinifex/handlers/ochrevector/filter.go b/spinifex/handlers/ochrevector/filter.go
index 4cfe1e0d0..2a7a5ee50 100644
--- a/spinifex/handlers/ochrevector/filter.go
+++ b/spinifex/handlers/ochrevector/filter.go
@@ -46,10 +46,10 @@ var filterKeyPattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]{0,63}$`)
// orAll) over Children. Construct via the Equals/GreaterThan/.../AndAll
// helpers rather than the struct literal directly.
type Filter struct {
- Op FilterOp
- Key string
- Value any
- Children []*Filter
+ Op FilterOp `json:"Op"`
+ Key string `json:"Key"`
+ Value any `json:"Value"`
+ Children []*Filter `json:"Children"`
}
// Equals builds an `equals` leaf filter: metadata[key] as text equals value.
diff --git a/spinifex/handlers/quota/bedrock.go b/spinifex/handlers/quota/bedrock.go
index 9fafaaf9c..2f1ada5ca 100644
--- a/spinifex/handlers/quota/bedrock.go
+++ b/spinifex/handlers/quota/bedrock.go
@@ -167,11 +167,11 @@ type bedrockRPMSnapshot struct {
// RunBedrockRPMSync periodically snapshots every account this gateway has
// rate-limited into kv, until ctx is cancelled. A no-op when the RPM
// dimension is disabled, so a default-off deployment starts no ticker.
-func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream, replicas int) {
+func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream) {
if s == nil || !s.limits.RequestsPerMinuteEnabled {
return
}
- kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketBedrockRPMUsage, 1, replicas)
+ kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
if err != nil {
slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err)
return
diff --git a/spinifex/handlers/quota/bedrock_test.go b/spinifex/handlers/quota/bedrock_test.go
index 7411d8e4d..6af343c2e 100644
--- a/spinifex/handlers/quota/bedrock_test.go
+++ b/spinifex/handlers/quota/bedrock_test.go
@@ -195,7 +195,7 @@ func TestRunBedrockRPMSync_DisabledNoop(t *testing.T) {
s := New(Limits{}, nil)
done := make(chan struct{})
go func() {
- s.RunBedrockRPMSync(context.Background(), nil, 1)
+ s.RunBedrockRPMSync(context.Background(), nil)
close(done)
}()
select {
diff --git a/spinifex/handlers/quota/records_test.go b/spinifex/handlers/quota/records_test.go
index dd5ae9047..c1f42717a 100644
--- a/spinifex/handlers/quota/records_test.go
+++ b/spinifex/handlers/quota/records_test.go
@@ -21,9 +21,8 @@ func recordStore(t *testing.T) *kvstore.Store[vm.InstanceRecord] {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
return kvstore.New[vm.InstanceRecord](testutil.NewJetStream(t, nc), kvstore.Config{
- Name: "spinifex-instance-state",
- History: 1,
- Replicas: 1,
+ Name: "spinifex-instance-state",
+ History: 1,
})
}
diff --git a/spinifex/handlers/rds/backup.go b/spinifex/handlers/rds/backup.go
index 355b70b2a..8475c35c7 100644
--- a/spinifex/handlers/rds/backup.go
+++ b/spinifex/handlers/rds/backup.go
@@ -395,7 +395,10 @@ func AutomatedBackupStamp(at time.Time) string {
// listable through DescribeDBSnapshots --snapshot-type automated.
func (s *Service) DescribeDBInstanceAutomatedBackups(ctx context.Context,
input *rds.DescribeDBInstanceAutomatedBackupsInput, accountID string) (*rds.DescribeDBInstanceAutomatedBackupsOutput, error) {
- wanted, err := validateDescribeAutomatedBackupsRequest(input)
+ if input == nil {
+ input = &rds.DescribeDBInstanceAutomatedBackupsInput{}
+ }
+ wanted, filters, err := validateDescribeAutomatedBackupsRequest(input)
if err != nil {
return nil, err
}
@@ -434,31 +437,68 @@ func (s *Service) DescribeDBInstanceAutomatedBackups(ctx context.Context,
if err != nil {
return nil, err
}
- backups = append(backups, s.projectAutomatedBackup(&rec, len(stamps)))
+ backup := s.projectAutomatedBackup(&rec, len(stamps))
+ if !matchesFilters(filters, func(name string) (string, bool) {
+ switch name {
+ case filterDBInstanceID:
+ return aws.StringValue(backup.DBInstanceIdentifier), true
+ case filterStatus:
+ return aws.StringValue(backup.Status), true
+ }
+ return "", false
+ }) {
+ continue
+ }
+ backups = append(backups, backup)
}
- return &rds.DescribeDBInstanceAutomatedBackupsOutput{DBInstanceAutomatedBackups: backups}, nil
+ backups, next, err := Page(backups, automatedBackupPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBInstanceAutomatedBackupsOutput{DBInstanceAutomatedBackups: backups, Marker: next}, nil
}
-// A filter this phase cannot honour is rejected rather than dropped, since a
+// A scope this phase cannot honour is rejected rather than dropped, since a
// silently unfiltered list reads as a complete answer. Returns the DB instance to
-// report on, empty for every one.
-func validateDescribeAutomatedBackupsRequest(input *rds.DescribeDBInstanceAutomatedBackupsInput) (string, error) {
+// report on, empty for every one, and the Filters.
+func validateDescribeAutomatedBackupsRequest(input *rds.DescribeDBInstanceAutomatedBackupsInput) (string, []Filter, error) {
if input == nil {
- return "", nil
+ return "", nil, nil
}
- if len(input.Filters) > 0 {
- return "", unimplemented("Filters", "DescribeDBInstanceAutomatedBackups filters on DBInstanceIdentifier only")
+ filters, err := ReadFilters(input.Filters, filterDBInstanceID, filterDbiResourceID, filterStatus)
+ if err != nil {
+ return "", nil, err
+ }
+ for _, filter := range filters {
+ switch filter.Name {
+ case filterDbiResourceID:
+ return "", nil, unimplemented("Filter dbi-resource-id", automatedBackupNoResourceID)
+ case filterStatus:
+ for _, value := range filter.Values {
+ if !slices.Contains(awsAutomatedBackupStatuses, value) {
+ return "", nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "The parameter Filter: status is not a valid Automated Backup status. Allowed values are creating, active, retained, pending, replicating, deleting.")
+ }
+ }
+ }
}
if aws.StringValue(input.DbiResourceId) != "" {
- return "", unimplemented("DbiResourceId", "a DB instance has no resource ID distinct from its identifier here")
+ return "", nil, unimplemented("DbiResourceId", automatedBackupNoResourceID)
}
if aws.StringValue(input.DBInstanceAutomatedBackupsArn) != "" {
- return "", unimplemented("DBInstanceAutomatedBackupsArn",
+ return "", nil, unimplemented("DBInstanceAutomatedBackupsArn",
"cross-region automated backup replication is not offered, so no replicated backup has an ARN")
}
- return aws.StringValue(input.DBInstanceIdentifier), nil
+ return aws.StringValue(input.DBInstanceIdentifier), filters, nil
}
+// Why DbiResourceId and the dbi-resource-id filter are refused: an instance has
+// one, but its automated backups are addressed by its identifier only.
+const automatedBackupNoResourceID = "automated backups are addressed by DBInstanceIdentifier only"
+
+// AWS's whole status vocabulary; the ones never reported here match nothing.
+var awsAutomatedBackupStatuses = []string{"creating", "active", "retained", "pending", "replicating", "deleting"}
+
// RestoreWindow and LatestRestorableTime are deliberately absent: this phase
// backs discrete daily snapshots, and reporting a restore window would tell a
// client it can recover to any instant inside it.
diff --git a/spinifex/handlers/rds/backup_test.go b/spinifex/handlers/rds/backup_test.go
index 4d0535b40..32b1c697f 100644
--- a/spinifex/handlers/rds/backup_test.go
+++ b/spinifex/handlers/rds/backup_test.go
@@ -766,14 +766,37 @@ func TestDescribeDBInstanceAutomatedBackups_RejectsAnUnknownInstance(t *testing.
assert.Contains(t, err.Error(), awserrors.ErrorDBInstanceNotFound)
}
-// A filter this phase cannot honour is rejected, because a silently
-// unfiltered list reads as a complete answer.
+func TestDescribeDBInstanceAutomatedBackups_FiltersByInstanceAndStatus(t *testing.T) {
+ t.Parallel()
+ h := newSnapshotHarness(t, false)
+ seedInstance(t, h.svc, backupReadyRecord())
+
+ for name, tc := range map[string]struct {
+ filter *rds.Filter
+ want int
+ }{
+ "own instance": {&rds.Filter{Name: aws.String("db-instance-id"), Values: aws.StringSlice([]string{testDBID})}, 1},
+ "other instance": {&rds.Filter{Name: aws.String("db-instance-id"), Values: aws.StringSlice([]string{"other-db"})}, 0},
+ "own status": {&rds.Filter{Name: aws.String("status"), Values: aws.StringSlice([]string{"retained", "creating"})}, 1},
+ "other status": {&rds.Filter{Name: aws.String("status"), Values: aws.StringSlice([]string{"active"})}, 0},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ out, err := h.svc.DescribeDBInstanceAutomatedBackups(t.Context(),
+ &rds.DescribeDBInstanceAutomatedBackupsInput{Filters: []*rds.Filter{tc.filter}}, testAccountID)
+ require.NoError(t, err)
+ assert.Len(t, out.DBInstanceAutomatedBackups, tc.want)
+ })
+ }
+}
+
+// A scope this phase cannot honour is rejected, because a silently unfiltered
+// list reads as a complete answer; so are names and statuses AWS refuses.
func TestDescribeDBInstanceAutomatedBackups_RejectsUnimplementedFilters(t *testing.T) {
cases := map[string]*rds.DescribeDBInstanceAutomatedBackupsInput{
- "Filters": {Filters: []*rds.Filter{{
- Name: aws.String("db-instance-id"),
- Values: aws.StringSlice([]string{testDBID}),
- }}},
+ "dbi-resource-id filter": {Filters: filter("dbi-resource-id", "db-ABCDEF")},
+ "unknown filter": {Filters: filter("engine", "postgres")},
+ "bad status": {Filters: filter("status", "available")},
"DbiResourceId": {DbiResourceId: aws.String("db-ABCDEF")},
"DBInstanceAutomatedBackupsArn": {DBInstanceAutomatedBackupsArn: aws.String("arn:aws:rds:::auto-backup:x")},
}
diff --git a/spinifex/handlers/rds/describe.go b/spinifex/handlers/rds/describe.go
index 8dfaf3754..0563c8b29 100644
--- a/spinifex/handlers/rds/describe.go
+++ b/spinifex/handlers/rds/describe.go
@@ -4,6 +4,8 @@ import (
"context"
"errors"
"slices"
+ "strings"
+ "unicode/utf8"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/rds"
@@ -11,42 +13,36 @@ import (
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
)
-// The prefix AWS gives a DB instance's immutable resource ID, and the two filter
-// names DescribeDBInstances accepts against it.
-const (
- dbiResourceIDPrefix = "db"
-
- filterDbiResourceID = "dbi-resource-id"
- filterDBInstanceID = "db-instance-id"
-)
+// The prefix AWS gives a DB instance's immutable resource ID.
+const dbiResourceIDPrefix = "db"
// Named DB instances that do not exist are an error, matching AWS: a client
// polling a create would otherwise read an empty list as "gone" rather than
// "not ready".
func (s *Service) DescribeDBInstances(ctx context.Context, input *rds.DescribeDBInstancesInput, accountID string) (*rds.DescribeDBInstancesOutput, error) {
+ if input == nil {
+ input = &rds.DescribeDBInstancesInput{}
+ }
kv, err := s.bucket(ctx, accountID)
if err != nil {
return nil, err
}
- var matches func(*DBInstanceRecord) bool
- if input != nil {
- matches, err = dbInstanceFilterMatcher(input.Filters)
+ matches, err := dbInstanceFilterMatcher(input.Filters)
+ if err != nil {
+ return nil, err
+ }
+ if id := aws.StringValue(input.DBInstanceIdentifier); id != "" {
+ rec, _, err := s.getDBInstance(ctx, kv, id)
if err != nil {
return nil, err
}
- if id := aws.StringValue(input.DBInstanceIdentifier); id != "" {
- rec, _, err := s.getDBInstance(ctx, kv, id)
- if err != nil {
- return nil, err
- }
- // A named instance the filters exclude is reported as absent rather than
- // returned anyway, so the two halves of one request cannot disagree.
- if matches != nil && !matches(rec) {
- return nil, errors.New(awserrors.ErrorDBInstanceNotFound)
- }
- return &rds.DescribeDBInstancesOutput{DBInstances: []*rds.DBInstance{s.projectDBInstance(rec)}}, nil
+ // A named instance the filters exclude is reported as absent rather than
+ // returned anyway, so the two halves of one request cannot disagree.
+ if matches != nil && !matches(rec) {
+ return nil, errors.New(awserrors.ErrorDBInstanceNotFound)
}
+ return &rds.DescribeDBInstancesOutput{DBInstances: []*rds.DBInstance{s.projectDBInstance(rec)}}, nil
}
ids, err := ListDBInstanceIDs(ctx, kv)
@@ -72,49 +68,65 @@ func (s *Service) DescribeDBInstances(ctx context.Context, input *rds.DescribeDB
}
instances = append(instances, s.projectDBInstance(&rec))
}
- return &rds.DescribeDBInstancesOutput{DBInstances: instances}, nil
+ instances, next, err := Page(instances, dbInstancePageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBInstancesOutput{DBInstances: instances, Marker: next}, nil
}
-// The filter names AWS documents for this action and clients actually send.
-// Ignoring them is not a safe default here: the Terraform provider keys its
+// Ignoring a filter is not a safe default here: the Terraform provider keys its
// state off DbiResourceId and reads an instance back by filtering on it, so an
// unapplied filter answers with every instance in the account, which the
-// provider rejects as an ambiguous match. An unrecognised name is refused for
-// the same reason — a filter silently dropped returns rows the caller asked to
-// exclude.
+// provider rejects as an ambiguous match.
//
// Returns nil when there is nothing to filter on, so the common path allocates
// no closure.
-func dbInstanceFilterMatcher(filters []*rds.Filter) (func(*DBInstanceRecord) bool, error) {
- if len(filters) == 0 {
- return nil, nil
+func dbInstanceFilterMatcher(entries []*rds.Filter) (func(*DBInstanceRecord) bool, error) {
+ filters, err := ReadFilters(entries, filterDBClusterID, filterDBInstanceID, filterDbiResourceID, filterDomain, filterEngine)
+ if err != nil || len(filters) == 0 {
+ return nil, err
}
for _, filter := range filters {
- switch name := aws.StringValue(filter.Name); name {
- case filterDbiResourceID, filterDBInstanceID:
- default:
- return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
- "unrecognized filter name: %s", name)
+ if filter.Name != filterDomain {
+ continue
+ }
+ for _, value := range filter.Values {
+ if !validDomainID(value) {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "The parameter Filter: domain is not a valid Domain identifier. Domain identifier must begin with d- prefix, must contain only ASCII characters, and must not be empty")
+ }
}
}
- // AWS's own semantics: a record must match every filter, and matches a filter
- // by carrying any one of its values.
+ // No instance belongs to a cluster or joins a directory domain, so those two
+ // filters match nothing.
return func(rec *DBInstanceRecord) bool {
- for _, filter := range filters {
- var value string
- if aws.StringValue(filter.Name) == filterDbiResourceID {
- value = rec.DbiResourceID
- } else {
- value = rec.DBInstanceIdentifier
+ return matchesFilters(filters, func(name string) (string, bool) {
+ switch name {
+ case filterDbiResourceID:
+ return rec.DbiResourceID, true
+ case filterDBInstanceID:
+ return rec.DBInstanceIdentifier, true
+ case filterEngine:
+ return rec.Engine, true
}
- if !slices.Contains(aws.StringValueSlice(filter.Values), value) {
- return false
- }
- }
- return true
+ return "", false
+ })
}, nil
}
+func validDomainID(value string) bool {
+ if !strings.HasPrefix(value, "d-") {
+ return false
+ }
+ for i := 0; i < len(value); i++ {
+ if value[i] >= utf8.RuneSelf {
+ return false
+ }
+ }
+ return true
+}
+
// Returns the record plus its revision, for callers that follow with a CAS.
func (s *Service) getDBInstance(ctx context.Context, kv *kvstore.Bucket, id string) (*DBInstanceRecord, uint64, error) {
var rec DBInstanceRecord
diff --git a/spinifex/handlers/rds/describe_test.go b/spinifex/handlers/rds/describe_test.go
index b7fb27b38..24543259e 100644
--- a/spinifex/handlers/rds/describe_test.go
+++ b/spinifex/handlers/rds/describe_test.go
@@ -203,18 +203,61 @@ func TestDescribeDBInstances_EchoesOmittedInertSettingsAsDisabled(t *testing.T)
assert.Zero(t, aws.Int64Value(out.DBInstances[0].MonitoringInterval))
}
-// A filter nobody implemented is refused rather than dropped: dropping it
-// returns exactly the rows the caller asked to exclude.
-func TestDescribeDBInstances_RejectsAnUnrecognizedFilter(t *testing.T) {
+// AWS refuses a name it does not document for this call, and an ill-formed
+// domain identifier, rather than dropping the filter.
+func TestDescribeDBInstances_RejectsUnrecognizedFiltersAndBadDomains(t *testing.T) {
t.Parallel()
h := newCreateHarness(t, "")
seedCreated(t, h, testDBInstanceID)
- _, err := h.svc.DescribeDBInstances(t.Context(), &rds.DescribeDBInstancesInput{
- Filters: []*rds.Filter{{Name: aws.String("engine"), Values: aws.StringSlice([]string{"postgres"})}},
- }, testAccountID)
- require.Error(t, err)
- assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
+ for name, tc := range map[string]struct {
+ filter *rds.Filter
+ msg string
+ }{
+ "unknown name": {&rds.Filter{Name: aws.String("snapshot-type"), Values: aws.StringSlice([]string{"manual"})},
+ "Unrecognized filter name: snapshot-type"},
+ "domain without prefix": {&rds.Filter{Name: aws.String(filterDomain), Values: aws.StringSlice([]string{"nope"})},
+ "is not a valid Domain identifier"},
+ "non-ASCII domain": {&rds.Filter{Name: aws.String(filterDomain), Values: aws.StringSlice([]string{"d-é"})},
+ "is not a valid Domain identifier"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ _, err := h.svc.DescribeDBInstances(t.Context(), &rds.DescribeDBInstancesInput{
+ Filters: []*rds.Filter{tc.filter},
+ }, testAccountID)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
+ assert.Contains(t, err.Error(), tc.msg)
+ })
+ }
+}
+
+// engine narrows like the identifiers do; no instance is in a cluster or a
+// directory domain, so a well-formed value for either matches nothing.
+func TestDescribeDBInstances_FiltersOnEngineClusterAndDomain(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, "")
+ rec := seedCreated(t, h, testDBInstanceID)
+
+ for name, tc := range map[string]struct {
+ filter *rds.Filter
+ want int
+ }{
+ "own engine": {&rds.Filter{Name: aws.String(filterEngine), Values: aws.StringSlice([]string{"mariadb", rec.Engine})}, 1},
+ "other engine": {&rds.Filter{Name: aws.String(filterEngine), Values: aws.StringSlice([]string{"mariadb"})}, 0},
+ "cluster": {&rds.Filter{Name: aws.String(filterDBClusterID), Values: aws.StringSlice([]string{"any-cluster"})}, 0},
+ "domain": {&rds.Filter{Name: aws.String(filterDomain), Values: aws.StringSlice([]string{"d-1234567890"})}, 0},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ out, err := h.svc.DescribeDBInstances(t.Context(), &rds.DescribeDBInstancesInput{
+ Filters: []*rds.Filter{tc.filter},
+ }, testAccountID)
+ require.NoError(t, err)
+ assert.Len(t, out.DBInstances, tc.want)
+ })
+ }
}
// A named instance and a filter that excludes it disagree; reporting the
diff --git a/spinifex/handlers/rds/events.go b/spinifex/handlers/rds/events.go
index 27890caa1..92bec9c3b 100644
--- a/spinifex/handlers/rds/events.go
+++ b/spinifex/handlers/rds/events.go
@@ -33,9 +33,8 @@ const (
// once, so the CAS is retried rather than dropped on first contention.
eventWriteAttempts = 8
- // AWS's defaults for the read: a one-hour window and a 100-record page.
+ // AWS's default window for the read.
defaultEventDuration = time.Hour
- maxEventRecords = 100
)
// AWS's SourceType values. Only the ones some phase actually writes are
@@ -158,6 +157,11 @@ func trimEvents(events []Event) []Event {
// to a resource, so an unfiltered call reports the whole account's recent
// history, including resources that have since been deleted.
func (s *Service) DescribeEvents(ctx context.Context, input *rds.DescribeEventsInput, accountID string) (*rds.DescribeEventsOutput, error) {
+ // AWS recognises these names. They are not applied, as recognised names are
+ // not on the other calls documented as not supporting Filters.
+ if _, err := ReadFilters(input.Filters, filterEventCategory, filterDBInstanceID); err != nil {
+ return nil, err
+ }
window, err := eventWindow(input)
if err != nil {
return nil, err
@@ -209,16 +213,14 @@ func (s *Service) DescribeEvents(ctx context.Context, input *rds.DescribeEventsI
// Oldest first across resources, as AWS returns them, so a client that has
// already read up to a timestamp can resume from it.
slices.SortFunc(events, func(a, b Event) int {
- if cmp := a.Date.Compare(b.Date); cmp != 0 {
- return cmp
- }
- return strings.Compare(a.SourceIdentifier, b.SourceIdentifier)
+ return strings.Compare(eventPageKey(a), eventPageKey(b))
})
- if limit := eventRecordLimit(input); len(events) > limit {
- events = events[:limit]
+ events, marker, err := Page(events, eventPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
}
- out := &rds.DescribeEventsOutput{Events: make([]*rds.Event, 0, len(events))}
+ out := &rds.DescribeEventsOutput{Events: make([]*rds.Event, 0, len(events)), Marker: marker}
for _, event := range events {
out.Events = append(out.Events, s.projectEvent(accountID, event))
}
@@ -267,12 +269,11 @@ func eventWindow(input *rds.DescribeEventsInput) (timeWindow, error) {
return window, nil
}
-func eventRecordLimit(input *rds.DescribeEventsInput) int {
- requested := aws.Int64Value(input.MaxRecords)
- if requested <= 0 || requested > maxEventRecords {
- return maxEventRecords
- }
- return int(requested)
+// Fixed-width nanoseconds lead, so the key sorts by date, and the remaining
+// fields break a tie the same way on every read.
+func eventPageKey(event Event) string {
+ return pageKey(fmt.Sprintf("%020d", event.Date.UnixNano()),
+ event.SourceIdentifier, event.SourceType, event.Message)
}
// A fully qualified read is one Get. Anything broader has to enumerate, because
@@ -305,10 +306,7 @@ func matchesCategories(event Event, wanted []string) bool {
if len(wanted) == 0 {
return true
}
- for _, category := range wanted {
- if slices.Contains(event.Categories, category) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(wanted, func(category string) bool {
+ return slices.Contains(event.Categories, category)
+ })
}
diff --git a/spinifex/handlers/rds/events_test.go b/spinifex/handlers/rds/events_test.go
index 5c2db1bb7..0fa65d841 100644
--- a/spinifex/handlers/rds/events_test.go
+++ b/spinifex/handlers/rds/events_test.go
@@ -102,6 +102,28 @@ func TestDescribeEvents_ReadsAcrossResourcesAndScopesToOneWhenAsked(t *testing.T
assert.Equal(t, []string{"snapshot event"}, messagesOf(scoped))
}
+// A truncated read carries a Marker, so a client can tell the list was cut and
+// resume from it.
+func TestDescribeEvents_PagesOldestFirst(t *testing.T) {
+ t.Parallel()
+ svc := newTestService(t)
+ now := time.Now().UTC()
+ seedEvent(t, svc, EventSourceTypeDBInstance, testDBID, now.Add(-3*time.Minute), "first")
+ seedEvent(t, svc, EventSourceTypeDBInstance, testDBID, now.Add(-2*time.Minute), "second")
+ seedEvent(t, svc, EventSourceTypeDBInstance, testDBID, now.Add(-time.Minute), "third")
+
+ first, err := svc.DescribeEvents(t.Context(), &rds.DescribeEventsInput{MaxRecords: aws.Int64(2)}, testAccountID)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"first", "second"}, messagesOf(first.Events))
+ require.NotNil(t, first.Marker)
+
+ rest, err := svc.DescribeEvents(t.Context(),
+ &rds.DescribeEventsInput{MaxRecords: aws.Int64(2), Marker: first.Marker}, testAccountID)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"third"}, messagesOf(rest.Events))
+ assert.Nil(t, rest.Marker)
+}
+
func TestDescribeEvents_RejectsMalformedRequests(t *testing.T) {
cases := []struct {
name string
diff --git a/spinifex/handlers/rds/filters.go b/spinifex/handlers/rds/filters.go
new file mode 100644
index 000000000..82c8ce6f7
--- /dev/null
+++ b/spinifex/handlers/rds/filters.go
@@ -0,0 +1,68 @@
+package handlers_rds
+
+import (
+ "slices"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/rds"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Filter names, as AWS spells them. Each Describe call recognises its own subset.
+const (
+ filterDBClusterID = "db-cluster-id"
+ filterDBInstanceID = "db-instance-id"
+ filterDBParameterGroupFamily = "db-parameter-group-family"
+ filterDBSnapshotID = "db-snapshot-id"
+ filterDataType = "data-type"
+ filterDbiResourceID = "dbi-resource-id"
+ filterDomain = "domain"
+ filterEngine = "engine"
+ filterEventCategory = "event-category"
+ filterParameterName = "parameter-name"
+ filterResourceID = "resource-id"
+ filterSnapshotType = "snapshot-type"
+ filterStatus = "status"
+)
+
+// Filter is one validated Filters entry.
+type Filter struct {
+ Name string
+ Values []string
+}
+
+// ReadFilters validates Filters against the names one Describe call recognises,
+// in AWS's order: an unknown name, compared case-sensitively, before an absent
+// value list. The query parser reads an empty list as absent, so both answer
+// the same.
+func ReadFilters(filters []*rds.Filter, recognised ...string) ([]Filter, error) {
+ out := make([]Filter, 0, len(filters))
+ for _, entry := range filters {
+ name := "null"
+ if entry != nil && entry.Name != nil {
+ name = *entry.Name
+ }
+ if entry == nil || entry.Name == nil || !slices.Contains(recognised, name) {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: %s", name)
+ }
+ if len(entry.Values) == 0 {
+ return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterCombination,
+ "The values list cannot be null for the filter %s.", name)
+ }
+ out = append(out, Filter{Name: name, Values: aws.StringValueSlice(entry.Values)})
+ }
+ return out, nil
+}
+
+// matchesFilters is AWS's rule: a row must match every filter, and matches one
+// by carrying any of its values. field returns the row's value for a filter
+// name; a name the row has no field for matches nothing.
+func matchesFilters(filters []Filter, field func(name string) (string, bool)) bool {
+ for _, filter := range filters {
+ value, ok := field(filter.Name)
+ if !ok || !slices.Contains(filter.Values, value) {
+ return false
+ }
+ }
+ return true
+}
diff --git a/spinifex/handlers/rds/filters_test.go b/spinifex/handlers/rds/filters_test.go
new file mode 100644
index 000000000..bd8c77f7d
--- /dev/null
+++ b/spinifex/handlers/rds/filters_test.go
@@ -0,0 +1,155 @@
+//test:in-package — reuses the in-package create harness and event seeding, and
+// the filter-name constants are unexported
+
+package handlers_rds
+
+import (
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/rds"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func filter(name string, values ...string) []*rds.Filter {
+ return []*rds.Filter{{Name: aws.String(name), Values: aws.StringSlice(values)}}
+}
+
+func TestReadFilters_AnswersAsAWSDoes(t *testing.T) {
+ cases := map[string]struct {
+ filters []*rds.Filter
+ code string
+ msg string
+ }{
+ "unknown name": {filter("bogus", "x"), awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: bogus"},
+ "name case": {filter("Engine", "x"), awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: Engine"},
+ "missing name": {[]*rds.Filter{{Values: aws.StringSlice([]string{"x"})}}, awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: null"},
+ "nil entry": {[]*rds.Filter{nil}, awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: null"},
+ "unknown first": {[]*rds.Filter{{Name: aws.String("bogus")}}, awserrors.ErrorInvalidParameterValue, "Unrecognized filter name: bogus"},
+ "no values": {[]*rds.Filter{{Name: aws.String("engine")}}, awserrors.ErrorInvalidParameterCombination,
+ "The values list cannot be null for the filter engine."},
+ }
+ for name, tc := range cases {
+ t.Run(name, func(t *testing.T) {
+ _, err := ReadFilters(tc.filters, filterEngine)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), tc.code)
+ assert.Contains(t, err.Error(), tc.msg)
+ })
+ }
+
+ got, err := ReadFilters(append(filter(filterEngine, "postgres"), filter(filterEngine, "mariadb")...), filterEngine)
+ require.NoError(t, err)
+ assert.Equal(t, []Filter{{filterEngine, []string{"postgres"}}, {filterEngine, []string{"mariadb"}}}, got)
+}
+
+// AWS applies parameter-name, matching names in any case, and accepts data-type
+// without narrowing by it.
+func TestDescribeDBParameters_FiltersOnParameterName(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, testBaseDomain)
+ _, err := h.svc.CreateDBParameterGroup(t.Context(), parameterGroupInput(testParameterGroup), testAccountID)
+ require.NoError(t, err)
+ describe := func(filters []*rds.Filter) (*rds.DescribeDBParametersOutput, error) {
+ return h.svc.DescribeDBParameters(t.Context(), &rds.DescribeDBParametersInput{
+ DBParameterGroupName: aws.String(testParameterGroup),
+ Filters: filters,
+ MaxRecords: aws.Int64(100),
+ }, testAccountID)
+ }
+
+ byName, err := describe(filter(filterParameterName, "nope", "WORK_MEM"))
+ require.NoError(t, err)
+ require.Len(t, byName.Parameters, 1)
+ assert.Equal(t, "work_mem", aws.StringValue(byName.Parameters[0].ParameterName))
+
+ all, err := describe(nil)
+ require.NoError(t, err)
+ none, err := describe(filter(filterParameterName, "nope"))
+ require.NoError(t, err)
+ assert.NotNil(t, none.Parameters, "AWS answers an empty list, not an absent one")
+ assert.Empty(t, none.Parameters)
+
+ byType, err := describe(filter(filterDataType, "nope"))
+ require.NoError(t, err)
+ assert.Len(t, byType.Parameters, len(all.Parameters))
+
+ _, err = describe(filter("source", "user"))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Unrecognized filter name: source")
+}
+
+// On these calls AWS refuses a name it does not recognise but narrows by none
+// of the names it does.
+func TestRecognisedFiltersThatDoNotNarrow(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, "")
+ input := validCreateInput()
+ input.Tags = awsTags("env", "prod")
+ _, err := h.svc.CreateDBInstance(t.Context(), input, testAccountID)
+ require.NoError(t, err)
+ h.svc.RecordEvent(t.Context(), testAccountID, EventSourceTypeDBInstance, testDBInstanceID,
+ "DB instance stopped.", EventCategoryAvailability)
+ arn := FormatARN(ResourceKindDBInstance, testRegion, testAccountID, testDBInstanceID)
+
+ groups := func(filters []*rds.Filter) (int, error) {
+ out, err := h.svc.DescribeDBParameterGroups(t.Context(), &rds.DescribeDBParameterGroupsInput{Filters: filters}, testAccountID)
+ if err != nil {
+ return 0, err
+ }
+ return len(out.DBParameterGroups), nil
+ }
+ tags := func(filters []*rds.Filter) (int, error) {
+ out, err := h.svc.ListTagsForResource(t.Context(), &rds.ListTagsForResourceInput{ResourceName: aws.String(arn), Filters: filters}, testAccountID)
+ if err != nil {
+ return 0, err
+ }
+ return len(out.TagList), nil
+ }
+ events := func(filters []*rds.Filter) (int, error) {
+ out, err := h.svc.DescribeEvents(t.Context(), &rds.DescribeEventsInput{Filters: filters}, testAccountID)
+ if err != nil {
+ return 0, err
+ }
+ return len(out.Events), nil
+ }
+
+ for name, tc := range map[string]struct {
+ call func([]*rds.Filter) (int, error)
+ recognised [][]*rds.Filter
+ unknown string
+ }{
+ "DescribeDBParameterGroups": {groups,
+ [][]*rds.Filter{filter(filterDBParameterGroupFamily, "mysql8.0"), filter(filterEngine, "mysql")}, "db-parameter-group-name"},
+ "ListTagsForResource": {tags, [][]*rds.Filter{filter(filterResourceID, "nope")}, "tag-key"},
+ "DescribeEvents": {events,
+ [][]*rds.Filter{filter(filterEventCategory, "backup"), filter(filterDBInstanceID, "other-db")}, "source-id"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ want, err := tc.call(nil)
+ require.NoError(t, err)
+ require.NotZero(t, want)
+ for _, filters := range tc.recognised {
+ got, err := tc.call(filters)
+ require.NoError(t, err)
+ assert.Equal(t, want, got, aws.StringValue(filters[0].Name))
+ }
+ _, err = tc.call(filter(tc.unknown, "x"))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Unrecognized filter name: "+tc.unknown)
+ })
+ }
+}
+
+// AWS ignores Filters here entirely, even an unknown or malformed entry.
+func TestDescribeDBSubnetGroups_IgnoresFilters(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, "")
+ _, err := h.svc.DescribeDBSubnetGroups(t.Context(), &rds.DescribeDBSubnetGroupsInput{
+ Filters: []*rds.Filter{{Name: aws.String("bogus")}, {Values: aws.StringSlice([]string{"x"})}},
+ }, testAccountID)
+ require.NoError(t, err)
+}
diff --git a/spinifex/handlers/rds/paging.go b/spinifex/handlers/rds/paging.go
new file mode 100644
index 000000000..8cf4a26f7
--- /dev/null
+++ b/spinifex/handlers/rds/paging.go
@@ -0,0 +1,74 @@
+package handlers_rds
+
+import (
+ "encoding/base64"
+ "slices"
+ "strings"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/rds"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// AWS's page size when a Describe call names none.
+const defaultMaxRecords = 100
+
+// Joins the fields of a compound page key. It sorts below every printable byte,
+// so a key compares field by field exactly as the fields do.
+const pageKeySeparator = "\x00"
+
+// One page of items, already in ascending key order. The Marker is base64 of the
+// next item's key, as AWS issues it; a resume starts at the first key not below
+// it, so that item's deletion skips nothing. MaxRecords outside 1-100 is 100.
+func Page[T any](items []T, key func(T) string, maxRecords *int64, marker *string) ([]T, *string, error) {
+ start := 0
+ if encoded := aws.StringValue(marker); encoded != "" {
+ decoded, err := base64.StdEncoding.DecodeString(encoded)
+ if err != nil {
+ return nil, nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "Marker %q was not issued by this service", encoded)
+ }
+ start, _ = slices.BinarySearchFunc(items, string(decoded), func(item T, target string) int {
+ return strings.Compare(key(item), target)
+ })
+ }
+
+ limit := defaultMaxRecords
+ if requested := aws.Int64Value(maxRecords); requested > 0 && requested < defaultMaxRecords {
+ limit = int(requested)
+ }
+ rest := items[start:]
+ if len(rest) <= limit {
+ return rest, nil, nil
+ }
+ return rest[:limit], aws.String(base64.StdEncoding.EncodeToString([]byte(key(rest[limit])))), nil
+}
+
+func pageKey(fields ...string) string {
+ return strings.Join(fields, pageKeySeparator)
+}
+
+// The key each paged listing is already sorted by.
+func parameterPageKey(p *rds.Parameter) string { return aws.StringValue(p.ParameterName) }
+
+func dbInstancePageKey(i *rds.DBInstance) string { return aws.StringValue(i.DBInstanceIdentifier) }
+
+func dbSnapshotPageKey(s *rds.DBSnapshot) string { return aws.StringValue(s.DBSnapshotIdentifier) }
+
+func automatedBackupPageKey(b *rds.DBInstanceAutomatedBackup) string {
+ return aws.StringValue(b.DBInstanceIdentifier)
+}
+
+func subnetGroupPageKey(g *rds.DBSubnetGroup) string { return aws.StringValue(g.DBSubnetGroupName) }
+
+func parameterGroupPageKey(g *rds.DBParameterGroup) string {
+ return aws.StringValue(g.DBParameterGroupName)
+}
+
+func EngineVersionPageKey(v *rds.DBEngineVersion) string {
+ return pageKey(aws.StringValue(v.Engine), aws.StringValue(v.EngineVersion))
+}
+
+func OrderableOptionPageKey(o *rds.OrderableDBInstanceOption) string {
+ return pageKey(aws.StringValue(o.Engine), aws.StringValue(o.EngineVersion), aws.StringValue(o.DBInstanceClass))
+}
diff --git a/spinifex/handlers/rds/paging_test.go b/spinifex/handlers/rds/paging_test.go
new file mode 100644
index 000000000..38dcc26e7
--- /dev/null
+++ b/spinifex/handlers/rds/paging_test.go
@@ -0,0 +1,107 @@
+package handlers_rds
+
+import (
+ "encoding/base64"
+ "fmt"
+ "slices"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/rds"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func identity(s string) string { return s }
+
+func pageNames(n int) []string {
+ names := make([]string, 0, n)
+ for i := range n {
+ names = append(names, fmt.Sprintf("item-%03d", i))
+ }
+ return names
+}
+
+// Follows every Marker to the end, returning the pages as served.
+func walkPages(t *testing.T, items []string, maxRecords int64) [][]string {
+ t.Helper()
+ var pages [][]string
+ var marker *string
+ for {
+ page, next, err := Page(items, identity, aws.Int64(maxRecords), marker)
+ require.NoError(t, err)
+ pages = append(pages, page)
+ if next == nil {
+ return pages
+ }
+ marker = next
+ }
+}
+
+func TestPage_WalksEveryItemOnceInOrder(t *testing.T) {
+ t.Parallel()
+ items := pageNames(53)
+ pages := walkPages(t, items, 20)
+
+ require.Len(t, pages, 3)
+ assert.Len(t, pages[0], 20)
+ assert.Len(t, pages[1], 20)
+ assert.Len(t, pages[2], 13)
+ assert.Equal(t, items, slices.Concat(pages...))
+}
+
+// AWS's marker is base64 of the next row's name, and the last page carries none.
+func TestPage_MarkerNamesTheNextItem(t *testing.T) {
+ t.Parallel()
+ items := pageNames(25)
+ _, next, err := Page(items, identity, aws.Int64(20), nil)
+ require.NoError(t, err)
+ assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("item-020")), aws.StringValue(next))
+
+ last, next, err := Page(items, identity, aws.Int64(20), next)
+ require.NoError(t, err)
+ assert.Equal(t, items[20:], last)
+ assert.Nil(t, next)
+}
+
+func TestPage_ResumesPastAnItemDeletedBetweenPages(t *testing.T) {
+ t.Parallel()
+ items := pageNames(30)
+ _, marker, err := Page(items, identity, aws.Int64(10), nil)
+ require.NoError(t, err)
+
+ remaining := slices.Delete(slices.Clone(items), 10, 11)
+ page, _, err := Page(remaining, identity, aws.Int64(10), marker)
+ require.NoError(t, err)
+ assert.Equal(t, items[11:21], page)
+}
+
+func TestPage_ClampsAnUnsetOrOutOfRangeMaxRecordsToTheDefault(t *testing.T) {
+ t.Parallel()
+ items := pageNames(defaultMaxRecords + 5)
+ for _, maxRecords := range []*int64{nil, aws.Int64(0), aws.Int64(-1), aws.Int64(500)} {
+ page, next, err := Page(items, identity, maxRecords, nil)
+ require.NoError(t, err)
+ assert.Len(t, page, defaultMaxRecords)
+ assert.NotNil(t, next)
+ }
+}
+
+func TestPage_RejectsAMarkerItDidNotIssue(t *testing.T) {
+ t.Parallel()
+ _, _, err := Page(pageNames(5), identity, nil, aws.String("not base64!"))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue)
+}
+
+// The compound keys the catalog listings page by must sort as the listings do,
+// which a separator above '-' or '.' would break for engine names sharing a prefix.
+func TestPageKey_SortsFieldByField(t *testing.T) {
+ t.Parallel()
+ keys := []string{
+ OrderableOptionPageKey(&rds.OrderableDBInstanceOption{Engine: aws.String("aurora"), DBInstanceClass: aws.String("db.t3.micro")}),
+ OrderableOptionPageKey(&rds.OrderableDBInstanceOption{Engine: aws.String("aurora-mysql"), DBInstanceClass: aws.String("db.m5.large")}),
+ }
+ assert.True(t, slices.IsSorted(keys))
+}
diff --git a/spinifex/handlers/rds/parametergroup.go b/spinifex/handlers/rds/parametergroup.go
index cc185e3e5..e0880a7ff 100644
--- a/spinifex/handlers/rds/parametergroup.go
+++ b/spinifex/handlers/rds/parametergroup.go
@@ -84,20 +84,25 @@ func (s *Service) CreateDBParameterGroup(ctx context.Context, input *rds.CreateD
// or not anything has materialised it yet — a client that lists groups before
// creating its first instance must still see the one it can name.
func (s *Service) DescribeDBParameterGroups(ctx context.Context, input *rds.DescribeDBParameterGroupsInput, accountID string) (*rds.DescribeDBParameterGroupsOutput, error) {
+ if input == nil {
+ input = &rds.DescribeDBParameterGroupsInput{}
+ }
+ // AWS recognises both names and narrows by neither.
+ if _, err := ReadFilters(input.Filters, filterDBParameterGroupFamily, filterEngine); err != nil {
+ return nil, err
+ }
kv, err := s.bucket(ctx, accountID)
if err != nil {
return nil, err
}
- if input != nil {
- if name := aws.StringValue(input.DBParameterGroupName); name != "" {
- rec, _, err := getDBParameterGroup(ctx, kv, accountID, name)
- if err != nil {
- return nil, err
- }
- return &rds.DescribeDBParameterGroupsOutput{
- DBParameterGroups: []*rds.DBParameterGroup{s.projectParameterGroupRecord(rec)},
- }, nil
+ if name := aws.StringValue(input.DBParameterGroupName); name != "" {
+ rec, _, err := getDBParameterGroup(ctx, kv, accountID, name)
+ if err != nil {
+ return nil, err
}
+ return &rds.DescribeDBParameterGroupsOutput{
+ DBParameterGroups: []*rds.DBParameterGroup{s.projectParameterGroupRecord(rec)},
+ }, nil
}
names, err := ListDBParameterGroupNames(ctx, kv)
@@ -125,7 +130,11 @@ func (s *Service) DescribeDBParameterGroups(ctx context.Context, input *rds.Desc
}
groups = append(groups, s.projectParameterGroupRecord(rec))
}
- return &rds.DescribeDBParameterGroupsOutput{DBParameterGroups: groups}, nil
+ groups, next, err := Page(groups, parameterGroupPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBParameterGroupsOutput{DBParameterGroups: groups, Marker: next}, nil
}
// Stores validated overrides, one KV key per parameter, so a modify touching one
@@ -243,6 +252,11 @@ func (s *Service) DescribeDBParameters(ctx context.Context, input *rds.DescribeD
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"Source %q is not one of %s or %s", source, ParameterSourceUser, ParameterSourceEngineDefault)
}
+ // AWS narrows by parameter-name, matching names in any case, and not by data-type.
+ filters, err := ReadFilters(input.Filters, filterParameterName, filterDataType)
+ if err != nil {
+ return nil, err
+ }
kv, err := s.bucket(ctx, accountID)
if err != nil {
@@ -267,7 +281,8 @@ func (s *Service) DescribeDBParameters(ctx context.Context, input *rds.DescribeD
return nil, err
}
- out := &rds.DescribeDBParametersOutput{}
+ // Non-nil, so a filter that matches nothing answers with an empty list, as AWS does.
+ out := &rds.DescribeDBParametersOutput{Parameters: []*rds.Parameter{}}
for _, param := range engine.CatalogParameterNames() {
spec, _ := engine.LookupParameter(param)
override, isOverride := overrides[param]
@@ -278,11 +293,29 @@ func (s *Service) DescribeDBParameters(ctx context.Context, input *rds.DescribeD
if source != "" && source != parameterSource(isOverride) {
continue
}
+ if !parameterNameMatches(filters, param) {
+ continue
+ }
out.Parameters = append(out.Parameters, projectParameter(spec, value, override.ApplyMethod, isOverride))
}
+ out.Parameters, out.Marker, err = Page(out.Parameters, parameterPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
return out, nil
}
+func parameterNameMatches(filters []Filter, param string) bool {
+ for _, filter := range filters {
+ if filter.Name == filterParameterName && !slices.ContainsFunc(filter.Values, func(v string) bool {
+ return strings.EqualFold(v, param)
+ }) {
+ return false
+ }
+ }
+ return true
+}
+
// Refused for a default group, and while any instance still references it —
// including one that is only deleting, so a destroy that races the teardown
// fails cleanly rather than leaving a live engine's configuration unreadable.
diff --git a/spinifex/handlers/rds/parametergroup_test.go b/spinifex/handlers/rds/parametergroup_test.go
index 60ea45daa..68dcaac26 100644
--- a/spinifex/handlers/rds/parametergroup_test.go
+++ b/spinifex/handlers/rds/parametergroup_test.go
@@ -1,6 +1,7 @@
package handlers_rds
import (
+ "encoding/base64"
"strings"
"testing"
@@ -599,6 +600,56 @@ func TestDescribeDBParameters_FiltersOnSource(t *testing.T) {
assert.Len(t, defaults.Parameters, len(enginePostgres.CatalogParameterNames())-1)
}
+// AWS's observed first page of a fresh postgres18 group at MaxRecords 20: twenty
+// parameters by name, and a Marker that is base64 of the twenty-first's name.
+func TestDescribeDBParameters_PagesByName(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, testBaseDomain)
+ catalog := enginePostgres.CatalogParameterNames()
+
+ var listed []string
+ var marker *string
+ for page := 0; ; page++ {
+ out, err := h.svc.DescribeDBParameters(t.Context(), &rds.DescribeDBParametersInput{
+ DBParameterGroupName: aws.String(testDefaultPG),
+ MaxRecords: aws.Int64(20),
+ Marker: marker,
+ }, testAccountID)
+ require.NoError(t, err)
+ if page == 0 {
+ require.Len(t, out.Parameters, 20)
+ assert.Equal(t, base64.StdEncoding.EncodeToString([]byte(catalog[20])), aws.StringValue(out.Marker))
+ }
+ for _, p := range out.Parameters {
+ listed = append(listed, aws.StringValue(p.ParameterName))
+ }
+ if out.Marker == nil {
+ break
+ }
+ marker = out.Marker
+ }
+ assert.Equal(t, catalog, listed)
+}
+
+func TestDescribeDBParameterGroups_PagesByName(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, testBaseDomain)
+ _, err := h.svc.CreateDBParameterGroup(t.Context(), parameterGroupInput("alpha"), testAccountID)
+ require.NoError(t, err)
+
+ first, err := h.svc.DescribeDBParameterGroups(t.Context(),
+ &rds.DescribeDBParameterGroupsInput{MaxRecords: aws.Int64(2)}, testAccountID)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"alpha", testDefaultMariaDB}, parameterGroupNames(first))
+ require.NotNil(t, first.Marker)
+
+ rest, err := h.svc.DescribeDBParameterGroups(t.Context(),
+ &rds.DescribeDBParameterGroupsInput{MaxRecords: aws.Int64(2), Marker: first.Marker}, testAccountID)
+ require.NoError(t, err)
+ assert.Equal(t, []string{testDefaultPG}, parameterGroupNames(rest))
+ assert.Nil(t, rest.Marker)
+}
+
func TestDeleteDBParameterGroup_RemovesTheGroupAndItsValues(t *testing.T) {
t.Parallel()
h := newCreateHarness(t, testBaseDomain)
diff --git a/spinifex/handlers/rds/reconciler.go b/spinifex/handlers/rds/reconciler.go
index c1da52ca9..7d2cccce5 100644
--- a/spinifex/handlers/rds/reconciler.go
+++ b/spinifex/handlers/rds/reconciler.go
@@ -154,7 +154,7 @@ func (r *Reconciler) watchBuckets(ctx context.Context) ([]*kvstore.Bucket, error
// One pass, and when the loop should run again with nothing having changed.
func (r *Reconciler) reconcilePass(ctx context.Context) (time.Duration, error) {
- if !r.isLeader() {
+ if !r.lease.Held() {
// Leadership changes without anything being written, so a follower has
// to keep asking. The pass itself is a lease check and nothing more.
return leaseRefresh, nil
@@ -172,11 +172,7 @@ func (r *Reconciler) reconcilePass(ctx context.Context) (time.Duration, error) {
// cluster-singular and held continuously rather than claimed per sweep, so
// holding it is the whole answer and there is nothing for the caller to release.
func (r *Reconciler) AcquireClusterLease() (func(), bool) {
- return func() {}, r.isLeader()
-}
-
-func (r *Reconciler) isLeader() bool {
- return r.lease.Held()
+ return func() {}, r.lease.Held()
}
func (r *Reconciler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
diff --git a/spinifex/handlers/rds/reconciler_test.go b/spinifex/handlers/rds/reconciler_test.go
index 229f45e91..468adb276 100644
--- a/spinifex/handlers/rds/reconciler_test.go
+++ b/spinifex/handlers/rds/reconciler_test.go
@@ -396,7 +396,7 @@ func TestReconciler_RunHoldsLeadershipIndependentlyOfTheReconcileLoop(t *testing
done := make(chan struct{})
go func() { defer close(done); h.rec.Run(ctx) }()
- require.Eventually(t, h.rec.isLeader, 2*time.Second, 10*time.Millisecond,
+ require.Eventually(t, h.rec.lease.Held, 2*time.Second, 10*time.Millisecond,
"Run must elect without waiting for a reconcile tick")
cancel()
diff --git a/spinifex/handlers/rds/snapshot.go b/spinifex/handlers/rds/snapshot.go
index 2a2a09281..a12bad91a 100644
--- a/spinifex/handlers/rds/snapshot.go
+++ b/spinifex/handlers/rds/snapshot.go
@@ -222,7 +222,7 @@ func (s *Service) releaseQuiesce(ctx context.Context, accountID, dbInstanceIdent
// empty list, matching AWS: a client polling a create would otherwise read
// "gone" for "not ready".
func (s *Service) DescribeDBSnapshots(ctx context.Context, input *rds.DescribeDBSnapshotsInput, accountID string) (*rds.DescribeDBSnapshotsOutput, error) {
- snapshotType, err := validateDescribeSnapshotsRequest(input)
+ snapshotType, filters, err := validateDescribeSnapshotsRequest(input)
if err != nil {
return nil, err
}
@@ -236,7 +236,7 @@ func (s *Service) DescribeDBSnapshots(ctx context.Context, input *rds.DescribeDB
if err != nil {
return nil, err
}
- if !snapshotMatches(rec, aws.StringValue(input.DBInstanceIdentifier), snapshotType) {
+ if !snapshotMatches(rec, aws.StringValue(input.DBInstanceIdentifier), snapshotType, filters) {
return &rds.DescribeDBSnapshotsOutput{DBSnapshots: []*rds.DBSnapshot{}}, nil
}
return &rds.DescribeDBSnapshotsOutput{DBSnapshots: []*rds.DBSnapshot{s.projectDBSnapshot(rec)}}, nil
@@ -257,20 +257,39 @@ func (s *Service) DescribeDBSnapshots(ctx context.Context, input *rds.DescribeDB
}
// A snapshot deleted between the key listing and this read is simply gone,
// which is what a describe one tick later would report too.
- if !found || !snapshotMatches(&rec, aws.StringValue(input.DBInstanceIdentifier), snapshotType) {
+ if !found || !snapshotMatches(&rec, aws.StringValue(input.DBInstanceIdentifier), snapshotType, filters) {
continue
}
snapshots = append(snapshots, s.projectDBSnapshot(&rec))
}
- return &rds.DescribeDBSnapshotsOutput{DBSnapshots: snapshots}, nil
+ snapshots, next, err := Page(snapshots, dbSnapshotPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBSnapshotsOutput{DBSnapshots: snapshots, Marker: next}, nil
}
-// An empty filter matches everything, as AWS does.
-func snapshotMatches(rec *DBSnapshotRecord, dbInstanceIdentifier, snapshotType string) bool {
+// An empty parameter matches everything, as AWS does.
+func snapshotMatches(rec *DBSnapshotRecord, dbInstanceIdentifier, snapshotType string, filters []Filter) bool {
if dbInstanceIdentifier != "" && rec.DBInstanceIdentifier != dbInstanceIdentifier {
return false
}
- return snapshotType == "" || rec.SnapshotType == snapshotType
+ if snapshotType != "" && rec.SnapshotType != snapshotType {
+ return false
+ }
+ return matchesFilters(filters, func(name string) (string, bool) {
+ switch name {
+ case filterDBInstanceID:
+ return rec.DBInstanceIdentifier, true
+ case filterDBSnapshotID:
+ return rec.DBSnapshotIdentifier, true
+ case filterSnapshotType:
+ return rec.SnapshotType, true
+ case filterEngine:
+ return rec.Engine, true
+ }
+ return "", false
+ })
}
// Removes the snapshot and, when it was the last thing holding a data volume
@@ -481,31 +500,53 @@ func validateCreateSnapshotRequest(input *rds.CreateDBSnapshotInput) (*validated
}, nil
}
-// Returns the snapshot type to filter on, empty for "any". A filter this
-// phase cannot honour is rejected rather than dropped, because a silently
-// unfiltered list reads as a complete answer.
-func validateDescribeSnapshotsRequest(input *rds.DescribeDBSnapshotsInput) (string, error) {
+// Returns the snapshot type to filter on, empty for "any", and the Filters. A
+// scope this phase cannot honour is rejected rather than dropped, because a
+// silently unfiltered list reads as a complete answer.
+func validateDescribeSnapshotsRequest(input *rds.DescribeDBSnapshotsInput) (string, []Filter, error) {
if input == nil {
- return "", nil
+ return "", nil, nil
}
- if len(input.Filters) > 0 {
- return "", unimplemented("Filters", "DescribeDBSnapshots filters on DBSnapshotIdentifier, DBInstanceIdentifier and SnapshotType only")
+ filters, err := ReadFilters(input.Filters,
+ filterDBInstanceID, filterDBSnapshotID, filterDbiResourceID, filterSnapshotType, filterEngine)
+ if err != nil {
+ return "", nil, err
+ }
+ for _, filter := range filters {
+ switch filter.Name {
+ case filterDbiResourceID:
+ return "", nil, unimplemented("Filter dbi-resource-id", snapshotNoResourceID)
+ case filterSnapshotType:
+ // AWS's whole vocabulary is accepted; the types never offered here match nothing.
+ for _, value := range filter.Values {
+ if !slices.Contains(awsSnapshotTypes, value) {
+ return "", nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "Invalid snapshot type. If specified, must be one of: public, shared, manual, awsbackup, automated.")
+ }
+ }
+ }
}
if aws.StringValue(input.DbiResourceId) != "" {
- return "", unimplemented("DbiResourceId", "a DB instance has no resource ID distinct from its identifier here")
+ return "", nil, unimplemented("DbiResourceId", snapshotNoResourceID)
}
if aws.BoolValue(input.IncludeShared) || aws.BoolValue(input.IncludePublic) {
- return "", unimplemented("IncludeShared/IncludePublic", "cross-account snapshot sharing is not offered")
+ return "", nil, unimplemented("IncludeShared/IncludePublic", "cross-account snapshot sharing is not offered")
}
switch snapshotType := aws.StringValue(input.SnapshotType); snapshotType {
case "", SnapshotTypeManual, SnapshotTypeAutomated:
- return snapshotType, nil
+ return snapshotType, filters, nil
default:
- return "", awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ return "", nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"SnapshotType %q is not offered; use %q or %q", snapshotType, SnapshotTypeManual, SnapshotTypeAutomated)
}
}
+// Why DbiResourceId and the dbi-resource-id filter are refused: an instance has
+// one, but a snapshot does not record it.
+const snapshotNoResourceID = "a snapshot does not record its source instance's resource ID"
+
+var awsSnapshotTypes = []string{"public", "shared", SnapshotTypeManual, "awsbackup", SnapshotTypeAutomated}
+
// A name the caller is minting: a customer snapshot, or a final snapshot at
// delete. The rds: namespace belongs to automated backups, so it is refused
// wherever a name is created — and accepted by validateDBSnapshotReference, which
diff --git a/spinifex/handlers/rds/snapshot_test.go b/spinifex/handlers/rds/snapshot_test.go
index 2fb38165f..627eb892e 100644
--- a/spinifex/handlers/rds/snapshot_test.go
+++ b/spinifex/handlers/rds/snapshot_test.go
@@ -341,6 +341,34 @@ func TestDescribeDBSnapshots_FiltersByInstanceAndType(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
assert.Empty(t, byType.DBSnapshots, "nothing in this phase is automated")
+
+ for name, tc := range map[string]struct {
+ filters []*rds.Filter
+ want []string
+ }{
+ "instance": {[]*rds.Filter{{Name: aws.String("db-instance-id"), Values: aws.StringSlice([]string{"reports-db"})}},
+ []string{"reports-db-daily"}},
+ "snapshot": {[]*rds.Filter{{Name: aws.String("db-snapshot-id"), Values: aws.StringSlice([]string{testSnapshotID, "nope"})}},
+ []string{testSnapshotID}},
+ "type": {[]*rds.Filter{{Name: aws.String("snapshot-type"), Values: aws.StringSlice([]string{"shared", SnapshotTypeManual})}},
+ []string{testSnapshotID, "reports-db-daily"}},
+ "engine": {[]*rds.Filter{{Name: aws.String("engine"), Values: aws.StringSlice([]string{"mariadb"})}}, nil},
+ "every filter must match": {[]*rds.Filter{
+ {Name: aws.String("snapshot-type"), Values: aws.StringSlice([]string{SnapshotTypeManual})},
+ {Name: aws.String("db-instance-id"), Values: aws.StringSlice([]string{testDBID})},
+ }, []string{testSnapshotID}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ out, err := h.svc.DescribeDBSnapshots(t.Context(), &rds.DescribeDBSnapshotsInput{Filters: tc.filters}, testAccountID)
+ require.NoError(t, err)
+ var got []string
+ for _, snap := range out.DBSnapshots {
+ got = append(got, aws.StringValue(snap.DBSnapshotIdentifier))
+ }
+ assert.ElementsMatch(t, tc.want, got)
+ })
+ }
}
// A client polling a create would otherwise read "gone" for "not ready".
@@ -361,10 +389,12 @@ func TestDescribeDBSnapshots_RejectsUnhonouredScoping(t *testing.T) {
h := newSnapshotHarness(t, false)
cases := map[string]*rds.DescribeDBSnapshotsInput{
- "Filters": {Filters: []*rds.Filter{{Name: aws.String("engine")}}},
- "DbiResourceId": {DbiResourceId: aws.String("db-ABC123")},
- "IncludeShared": {IncludeShared: aws.Bool(true)},
- "SnapshotType": {SnapshotType: aws.String("shared")},
+ "dbi-resource-id filter": {Filters: filter("dbi-resource-id", "db-ABC123")},
+ "unknown filter": {Filters: filter("status", "available")},
+ "bad snapshot-type": {Filters: filter("snapshot-type", "bogus")},
+ "DbiResourceId": {DbiResourceId: aws.String("db-ABC123")},
+ "IncludeShared": {IncludeShared: aws.Bool(true)},
+ "SnapshotType": {SnapshotType: aws.String("shared")},
}
for name, input := range cases {
t.Run(name, func(t *testing.T) {
diff --git a/spinifex/handlers/rds/subnetgroup.go b/spinifex/handlers/rds/subnetgroup.go
index 514acbba9..d7e599429 100644
--- a/spinifex/handlers/rds/subnetgroup.go
+++ b/spinifex/handlers/rds/subnetgroup.go
@@ -89,20 +89,22 @@ func (s *Service) CreateDBSubnetGroup(ctx context.Context, input *rds.CreateDBSu
}
// A named group that does not exist is an error, matching AWS; an unnamed
-// request lists the account's groups.
+// request lists the account's groups. Filters is not read: AWS ignores it here,
+// even an unknown or malformed entry.
func (s *Service) DescribeDBSubnetGroups(ctx context.Context, input *rds.DescribeDBSubnetGroupsInput, accountID string) (*rds.DescribeDBSubnetGroupsOutput, error) {
+ if input == nil {
+ input = &rds.DescribeDBSubnetGroupsInput{}
+ }
kv, err := s.bucket(ctx, accountID)
if err != nil {
return nil, err
}
- if input != nil {
- if name := aws.StringValue(input.DBSubnetGroupName); name != "" {
- rec, _, err := getDBSubnetGroup(ctx, kv, name)
- if err != nil {
- return nil, err
- }
- return &rds.DescribeDBSubnetGroupsOutput{DBSubnetGroups: []*rds.DBSubnetGroup{s.projectSubnetGroup(rec)}}, nil
+ if name := aws.StringValue(input.DBSubnetGroupName); name != "" {
+ rec, _, err := getDBSubnetGroup(ctx, kv, name)
+ if err != nil {
+ return nil, err
}
+ return &rds.DescribeDBSubnetGroupsOutput{DBSubnetGroups: []*rds.DBSubnetGroup{s.projectSubnetGroup(rec)}}, nil
}
names, err := listNames(ctx, kv, DBSubnetGroupsPrefix())
@@ -123,7 +125,11 @@ func (s *Service) DescribeDBSubnetGroups(ctx context.Context, input *rds.Describ
}
groups = append(groups, s.projectSubnetGroup(&rec))
}
- return &rds.DescribeDBSubnetGroupsOutput{DBSubnetGroups: groups}, nil
+ groups, next, err := Page(groups, subnetGroupPageKey, input.MaxRecords, input.Marker)
+ if err != nil {
+ return nil, err
+ }
+ return &rds.DescribeDBSubnetGroupsOutput{DBSubnetGroups: groups, Marker: next}, nil
}
// Refused while any instance still names the group, including one that is only
@@ -280,11 +286,15 @@ func (s *Service) projectSubnetGroup(rec *DBSubnetGroupRecord) *rds.DBSubnetGrou
DBSubnetGroupArn: aws.String(FormatARN(ResourceKindDBSubnetGroup, s.region, rec.AccountID, rec.Name)),
SubnetGroupStatus: aws.String(subnetGroupStatusComplete),
VpcId: aws.String(rec.VpcID),
+ SupportedNetworkTypes: aws.StringSlice([]string{networkTypeIPv4}),
}
for _, subnet := range rec.Subnets {
+ // Empty rather than nil: AWS reports SubnetOutpost as {} on every subnet,
+ // and the XML marshaller omits a nil pointer entirely.
member := &rds.Subnet{
SubnetIdentifier: aws.String(subnet.SubnetID),
SubnetStatus: aws.String("Active"),
+ SubnetOutpost: &rds.Outpost{},
}
if subnet.AvailabilityZone != "" {
member.SubnetAvailabilityZone = &rds.AvailabilityZone{Name: aws.String(subnet.AvailabilityZone)}
diff --git a/spinifex/handlers/rds/subnetgroup_test.go b/spinifex/handlers/rds/subnetgroup_test.go
index e516500c2..ea8249f25 100644
--- a/spinifex/handlers/rds/subnetgroup_test.go
+++ b/spinifex/handlers/rds/subnetgroup_test.go
@@ -1,12 +1,17 @@
package handlers_rds
import (
+ "bytes"
+ "encoding/json"
+ "encoding/xml"
"testing"
"github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/rds"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -180,6 +185,33 @@ func TestDescribeDBSubnetGroups_ListsAndNames(t *testing.T) {
assert.Equal(t, "zeta", aws.StringValue(named.DBSubnetGroups[0].DBSubnetGroupName))
}
+// AWS reports SupportedNetworkTypes on the group and SubnetOutpost as {} on each
+// subnet. Read back through the NATS JSON hop and the gateway's XML, as a client
+// sees it, since an empty Outpost that either hop drops reads as absent.
+func TestDescribeDBSubnetGroups_ReportsNetworkTypesAndAnEmptyOutpost(t *testing.T) {
+ t.Parallel()
+ h := newCreateHarness(t, testBaseDomain)
+ _, err := h.svc.CreateDBSubnetGroup(t.Context(), subnetGroupInput(testSubnetGroup, "subnet-alpha"), testAccountID)
+ require.NoError(t, err)
+ described, err := h.svc.DescribeDBSubnetGroups(t.Context(), &rds.DescribeDBSubnetGroupsInput{}, testAccountID)
+ require.NoError(t, err)
+
+ wire, err := json.Marshal(described)
+ require.NoError(t, err)
+ var relayed rds.DescribeDBSubnetGroupsOutput
+ require.NoError(t, json.Unmarshal(wire, &relayed))
+ body, err := utils.MarshalToXML(utils.GenerateIAMXMLPayload("DescribeDBSubnetGroups", &relayed))
+ require.NoError(t, err)
+ var parsed rds.DescribeDBSubnetGroupsOutput
+ require.NoError(t, xmlutil.UnmarshalXML(&parsed, xml.NewDecoder(bytes.NewReader(body)), "DescribeDBSubnetGroupsResult"))
+
+ require.Len(t, parsed.DBSubnetGroups, 1)
+ group := parsed.DBSubnetGroups[0]
+ assert.Equal(t, []string{"IPV4"}, aws.StringValueSlice(group.SupportedNetworkTypes))
+ require.Len(t, group.Subnets, 1)
+ assert.Equal(t, &rds.Outpost{}, group.Subnets[0].SubnetOutpost)
+}
+
// A client polling a create would read an empty list as "gone" rather than
// "not there", so a named group that does not exist is an error.
func TestDescribeDBSubnetGroups_RejectsAnUnknownName(t *testing.T) {
diff --git a/spinifex/handlers/rds/tags.go b/spinifex/handlers/rds/tags.go
index 61441d82d..dbfddeb49 100644
--- a/spinifex/handlers/rds/tags.go
+++ b/spinifex/handlers/rds/tags.go
@@ -75,6 +75,10 @@ func (s *Service) ListTagsForResource(ctx context.Context, input *rds.ListTagsFo
if input == nil {
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "empty request")
}
+ // AWS recognises the name and does not narrow by it.
+ if _, err := ReadFilters(input.Filters, filterResourceID); err != nil {
+ return nil, err
+ }
resource, parsed, err := s.resolveTaggable(aws.StringValue(input.ResourceName), accountID)
if err != nil {
return nil, err
diff --git a/spinifex/handlers/rds/window.go b/spinifex/handlers/rds/window.go
index 4425fda1c..2bc5b2a6d 100644
--- a/spinifex/handlers/rds/window.go
+++ b/spinifex/handlers/rds/window.go
@@ -3,6 +3,8 @@ package handlers_rds
import (
"fmt"
"hash/fnv"
+ "math"
+ "slices"
"strconv"
"strings"
"time"
@@ -242,12 +244,9 @@ func (w weeklyWindow) segments() [][2]time.Duration {
}
func withinSegments(segments [][2]time.Duration, offset time.Duration) bool {
- for _, segment := range segments {
- if offset >= segment[0] && offset < segment[1] {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(segments, func(segment [2]time.Duration) bool {
+ return offset >= segment[0] && offset < segment[1]
+ })
}
func sinceMidnight(t time.Time) time.Duration {
@@ -292,7 +291,10 @@ func assignDailySlot(block dailyWindow, identifier string, shift int64) dailyWin
if slots < 1 {
return block
}
- slot := int64(windowHash(identifier) % uint64(slots)) //nolint:gosec // the modulus bounds it to the slot count
+ var slot int64
+ if h := windowHash(identifier) % uint64(slots); h <= math.MaxInt64 {
+ slot = int64(h)
+ }
start := (block.start + time.Duration((slot+shift)%slots)*windowSlot) % oneDay
return dailyWindow{start: start, end: (start + windowSlot) % oneDay}
}
diff --git a/spinifex/handlers/sts/assume_role.go b/spinifex/handlers/sts/assume_role.go
index f8b8cf5b5..5361b7124 100644
--- a/spinifex/handlers/sts/assume_role.go
+++ b/spinifex/handlers/sts/assume_role.go
@@ -13,18 +13,17 @@ import (
"log/slog"
"regexp"
"slices"
- "strconv"
"strings"
"time"
- "unicode/utf8"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/sts"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
- handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/utils"
)
@@ -75,10 +74,6 @@ const (
// ASCII range. `:` and `/` are excluded so the assumed-role ARN stays unambiguously parseable.
var roleSessionNameRegex = regexp.MustCompile(`^[A-Za-z0-9_+=,.@-]{2,64}$`)
-// roleSessionNameCharsetRegex is roleSessionNameRegex without the length bound, so the
-// charset and length constraints are reported separately, as AWS does.
-var roleSessionNameCharsetRegex = regexp.MustCompile(`^[A-Za-z0-9_+=,.@-]*$`)
-
// AssumeRole mints temporary credentials after evaluating the target role's
// trust policy against the caller.
func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity string, input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
@@ -94,28 +89,9 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s
// AWS reports every failed constraint at once, durationSeconds first, and before
// resolving the role, so a bad value against a missing role is not AccessDenied.
- var violations []constraintViolation
- if d := input.DurationSeconds; d != nil {
- value := strconv.FormatInt(*d, 10)
- if *d < minDurationSeconds {
- violations = append(violations, constraintViolation{"durationSeconds", value,
- fmt.Sprintf("Member must have value greater than or equal to %d", minDurationSeconds)})
- } else if *d > maxDurationSeconds {
- violations = append(violations, constraintViolation{"durationSeconds", value,
- fmt.Sprintf("Member must have value less than or equal to %d", maxDurationSeconds)})
- }
- }
- if !roleSessionNameCharsetRegex.MatchString(sessionName) {
- violations = append(violations, constraintViolation{"roleSessionName", sessionName,
- `Member must satisfy regular expression pattern: [\w+=,.@-]*`})
- }
- if n := utf8.RuneCountInString(sessionName); n < minRoleSessionNameLength {
- violations = append(violations, constraintViolation{"roleSessionName", sessionName,
- fmt.Sprintf("Member must have length greater than or equal to %d", minRoleSessionNameLength)})
- } else if n > maxRoleSessionNameLength {
- violations = append(violations, constraintViolation{"roleSessionName", sessionName,
- fmt.Sprintf("Member must have length less than or equal to %d", maxRoleSessionNameLength)})
- }
+ violations := durationViolations(input.DurationSeconds, maxDurationSeconds)
+ violations = append(violations, roleSessionNameConstraint.check("roleSessionName", sessionName)...)
+ violations = append(violations, mfaViolations(input.SerialNumber, input.TokenCode)...)
if err := validationError(violations); err != nil {
return nil, err
}
@@ -128,7 +104,7 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s
return nil, awserrors.Errorf(awserrors.ErrorValidationError,
"Session tags are not supported in this release; omit Tags and TransitiveTagKeys")
}
- if aws.StringValue(input.SerialNumber) != "" || aws.StringValue(input.TokenCode) != "" {
+ if input.SerialNumber != nil || input.TokenCode != nil {
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"MFA is not supported in this release; omit SerialNumber and TokenCode")
}
@@ -353,12 +329,9 @@ func assumeRoleConditionsHold(raw json.RawMessage, sourceAccount string) bool {
}
func matchTrustAction(actions []string) bool {
- for _, a := range actions {
- if a == stsActionAssumeRole || a == stsActionWildcard || a == globalWildcard {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(actions, func(a string) bool {
+ return a == stsActionAssumeRole || a == stsActionWildcard || a == globalWildcard
+ })
}
// serviceSourcesForCaller returns the service principals an authenticated HTTPS
@@ -466,7 +439,7 @@ func matchAWSPrincipalEntry(clause, callerARN string) bool {
return true
}
if utils.IsAccountID(clause) {
- clause = fmt.Sprintf("arn:aws:iam::%s:root", clause)
+ clause = arn.FormatIAMRoot(clause)
}
if clause == callerARN {
return true
@@ -513,11 +486,11 @@ type principalARN struct {
}
func parsePrincipalARN(arnStr string) (principalARN, bool) {
- parts := strings.SplitN(arnStr, ":", 6)
- if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" || parts[3] != "" {
+ service, region, account, resource, ok := arn.Split(arnStr)
+ if !ok || region != "" {
return principalARN{}, false
}
- return principalARN{service: parts[2], account: parts[4], resource: parts[5]}, true
+ return principalARN{service: service, account: account, resource: resource}, true
}
func lastPathSegment(s string) string {
diff --git a/spinifex/handlers/sts/assume_role_test.go b/spinifex/handlers/sts/assume_role_test.go
index 6207171d8..21b50fdcf 100644
--- a/spinifex/handlers/sts/assume_role_test.go
+++ b/spinifex/handlers/sts/assume_role_test.go
@@ -497,8 +497,58 @@ func TestAssumeRole_RejectsMFA(t *testing.T) {
requireMFANotSupported(t, err)
})
}
+
+ for _, tc := range mfaConstraintCases {
+ t.Run(tc.name, func(t *testing.T) {
+ input := basicAssumeRoleInput(*role.Arn, "sess")
+ input.SerialNumber, input.TokenCode = tc.serialNumber, tc.tokenCode
+ _, err := svc.AssumeRole(testCallerAccountID, testCallerARN(), testCallerUserName, input)
+ requireAWSError(t, err, awserrors.ErrorValidationError, tc.wantMessage)
+ })
+ }
}
+// mfaConstraintCases break the model's serialNumber and tokenCode constraints, which
+// are checked before MFA is refused as unsupported.
+var mfaConstraintCases = func() []struct {
+ name string
+ serialNumber, tokenCode *string
+ wantMessage string
+} {
+ const (
+ serialPattern = `Member must satisfy regular expression pattern: [\w+=/:,.@-]*`
+ serialShort = "Member must have length greater than or equal to 9"
+ serialLong = "Member must have length less than or equal to 256"
+ tokenPattern = `Member must satisfy regular expression pattern: [\d]*`
+ tokenShort = "Member must have length greater than or equal to 6"
+ tokenLong = "Member must have length less than or equal to 6"
+ )
+ violation := func(field, value, constraint string) string {
+ return fmt.Sprintf("Value '%s' at '%s' failed to satisfy constraint: %s", value, field, constraint)
+ }
+ one := func(field, value, constraint string) string {
+ return "1 validation error detected: " + violation(field, value, constraint)
+ }
+ long := strings.Repeat("a", 257)
+ return []struct {
+ name string
+ serialNumber, tokenCode *string
+ wantMessage string
+ }{
+ {"serial_empty", aws.String(""), nil, one("serialNumber", "", serialShort)},
+ {"serial_short", aws.String("GAHT1234"), nil, one("serialNumber", "GAHT1234", serialShort)},
+ {"serial_long", aws.String(long), nil, one("serialNumber", long, serialLong)},
+ {"serial_charset", aws.String("GAHT 12345"), nil, one("serialNumber", "GAHT 12345", serialPattern)},
+ {"token_empty", nil, aws.String(""), one("tokenCode", "", tokenShort)},
+ {"token_long", nil, aws.String("1234567"), one("tokenCode", "1234567", tokenLong)},
+ {"token_charset", nil, aws.String("12345a"), one("tokenCode", "12345a", tokenPattern)},
+ {"token_charset_and_short", nil, aws.String("abc"), "2 validation errors detected: " +
+ violation("tokenCode", "abc", tokenPattern) + "; " + violation("tokenCode", "abc", tokenShort)},
+ {"both", aws.String(""), aws.String(""), "2 validation errors detected: " +
+ violation("serialNumber", "", serialShort) + "; " + violation("tokenCode", "", tokenShort)},
+ }
+}()
+
func TestAssumeRole_RejectsInvalidSessionName(t *testing.T) {
svc, _ := newTestSetup(t)
role := createRoleInAccount(t, svc, testCallerAccountID, "name", trustPolicyAllowingUser(testCallerARN()))
diff --git a/spinifex/handlers/sts/assume_role_with_web_identity_test.go b/spinifex/handlers/sts/assume_role_with_web_identity_test.go
index 79a75e0e3..a9617ce76 100644
--- a/spinifex/handlers/sts/assume_role_with_web_identity_test.go
+++ b/spinifex/handlers/sts/assume_role_with_web_identity_test.go
@@ -74,7 +74,7 @@ func newWebIdentityFixture(t *testing.T, svc *STSServiceImpl, accountID string)
federatedARN := handlers_iam.OIDCProviderARN(accountID, issuerHostPath)
// Publish JWKS to the per-cluster EKS bucket.
- kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, accountID, 1)
+ kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, accountID)
require.NoError(t, err)
raw, err := json.Marshal(jwks)
require.NoError(t, err)
@@ -364,7 +364,7 @@ func TestAssumeRoleWithWebIdentity_ProviderNotRegistered(t *testing.T) {
federatedARN := handlers_iam.OIDCProviderARN(testCallerAccountID, issuerHostPath)
// Publish JWKS — but skip the IAM provider registration.
- kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, testCallerAccountID, 1)
+ kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, testCallerAccountID)
require.NoError(t, err)
jwk := signingJWK(t, &priv.PublicKey)
jwk.Alg, jwk.Use, jwk.Kid = "ES256", "sig", kid
diff --git a/spinifex/handlers/sts/get_session_token.go b/spinifex/handlers/sts/get_session_token.go
index e96ae9423..26c120712 100644
--- a/spinifex/handlers/sts/get_session_token.go
+++ b/spinifex/handlers/sts/get_session_token.go
@@ -29,6 +29,13 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller
input = &sts.GetSessionTokenInput{}
}
+ // Model constraints are checked before the caller, as AWS does for AssumeRole.
+ violations := durationViolations(input.DurationSeconds, getSessionTokenMaxDuration)
+ violations = append(violations, mfaViolations(input.SerialNumber, input.TokenCode)...)
+ if err := validationError(violations); err != nil {
+ return nil, err
+ }
+
if callerPrincipalType != principalTypeUser {
slog.Warn("GetSessionToken denied: caller is not a long-lived user principal",
"account_id", callerAccountID, "principal_type", callerPrincipalType)
@@ -50,14 +57,14 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller
}
// MFA is out of scope: reject rather than silently ignore, so callers don't believe MFA was enforced.
- if aws.StringValue(input.SerialNumber) != "" || aws.StringValue(input.TokenCode) != "" {
+ if input.SerialNumber != nil || input.TokenCode != nil {
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"MFA is not supported in this release; omit SerialNumber and TokenCode")
}
duration := getSessionTokenDefaultDuration
if input.DurationSeconds != nil {
- duration = clampGetSessionTokenDuration(*input.DurationSeconds)
+ duration = *input.DurationSeconds
}
// Resolved here rather than threaded from the gateway: the gateway's value is
@@ -100,18 +107,6 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller
}, nil
}
-// clampGetSessionTokenDuration coerces a duration into [900, 129600].
-// A nil DurationSeconds is handled by the caller; AWS clamps explicit values rather than rejecting them.
-func clampGetSessionTokenDuration(requested int64) int64 {
- if requested < minDurationSeconds {
- return minDurationSeconds
- }
- if requested > getSessionTokenMaxDuration {
- return getSessionTokenMaxDuration
- }
- return requested
-}
-
// userEnvelope is the session envelope for a GetSessionToken user session:
// PrincipalType "user", SessionName = the IAM user name, no assumed-role fields.
func userEnvelope(accountID, userName, userID string) sessionEnvelope {
diff --git a/spinifex/handlers/sts/get_session_token_test.go b/spinifex/handlers/sts/get_session_token_test.go
index a7bc8dc1f..5c4c96a0d 100644
--- a/spinifex/handlers/sts/get_session_token_test.go
+++ b/spinifex/handlers/sts/get_session_token_test.go
@@ -4,6 +4,7 @@ import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
+ "fmt"
"strings"
"testing"
"time"
@@ -79,29 +80,47 @@ func TestGetSessionToken_NilInput_DefaultsToTwelveHours(t *testing.T) {
assertStoredDuration(t, svc, out, getSessionTokenDefaultDuration)
}
-func TestGetSessionToken_DurationClamp(t *testing.T) {
+func TestGetSessionToken_DurationBounds(t *testing.T) {
svc, _ := newTestSetup(t)
seedUser(t, svc, testCallerAccountID, testCallerUserName)
- cases := []struct {
- name string
- requested int64
- want int64
- }{
- {"below_minimum_clamps_up", minDurationSeconds - 1, minDurationSeconds},
- {"at_minimum", minDurationSeconds, minDurationSeconds},
- {"in_range", 7200, 7200},
- {"at_maximum", getSessionTokenMaxDuration, getSessionTokenMaxDuration},
- {"above_maximum_clamps_down", getSessionTokenMaxDuration + 1, getSessionTokenMaxDuration},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
+ for _, seconds := range []int64{minDurationSeconds, 7200, getSessionTokenMaxDuration} {
+ t.Run(fmt.Sprintf("accepts_%d", seconds), func(t *testing.T) {
out, err := svc.GetSessionToken(testCallerAccountID, testCallerUserName, principalTypeUser, testCallerAccessKeyID,
- &sts.GetSessionTokenInput{DurationSeconds: aws.Int64(tc.requested)})
+ &sts.GetSessionTokenInput{DurationSeconds: aws.Int64(seconds)})
require.NoError(t, err)
- assertStoredDuration(t, svc, out, tc.want)
+ assertStoredDuration(t, svc, out, seconds)
})
}
+
+ refused := []struct {
+ seconds int64
+ constraint string
+ }{
+ {minDurationSeconds - 1, "Member must have value greater than or equal to 900"},
+ {getSessionTokenMaxDuration + 1, "Member must have value less than or equal to 129600"},
+ }
+ for _, tc := range refused {
+ t.Run(fmt.Sprintf("refuses_%d", tc.seconds), func(t *testing.T) {
+ out, err := svc.GetSessionToken(testCallerAccountID, testCallerUserName, principalTypeUser, testCallerAccessKeyID,
+ &sts.GetSessionTokenInput{DurationSeconds: aws.Int64(tc.seconds)})
+ requireAWSError(t, err, awserrors.ErrorValidationError, fmt.Sprintf(
+ "1 validation error detected: Value '%d' at 'durationSeconds' failed to satisfy constraint: %s", tc.seconds, tc.constraint))
+ assert.Nil(t, out)
+ })
+ }
+}
+
+// A session caller is refused AccessDenied, but only once its input is valid.
+func TestGetSessionToken_ConstraintsCheckedBeforeCaller(t *testing.T) {
+ svc, _ := newTestSetup(t)
+
+ _, err := svc.GetSessionToken(testCallerAccountID, testCallerUserName, principalTypeUser, "ASIAEXAMPLEAAAAAAAAA",
+ &sts.GetSessionTokenInput{DurationSeconds: aws.Int64(899), TokenCode: aws.String("")})
+ requireAWSError(t, err, awserrors.ErrorValidationError,
+ "2 validation errors detected: Value '899' at 'durationSeconds' failed to satisfy constraint: "+
+ "Member must have value greater than or equal to 900; "+
+ "Value '' at 'tokenCode' failed to satisfy constraint: Member must have length greater than or equal to 6")
}
func TestGetSessionToken_RejectsNonUserAndSessionCallers(t *testing.T) {
@@ -149,6 +168,15 @@ func TestGetSessionToken_RejectsMFAParameters(t *testing.T) {
assert.Nil(t, out)
})
}
+
+ for _, tc := range mfaConstraintCases {
+ t.Run(tc.name, func(t *testing.T) {
+ out, err := svc.GetSessionToken(testCallerAccountID, testCallerUserName, principalTypeUser, testCallerAccessKeyID,
+ &sts.GetSessionTokenInput{SerialNumber: tc.serialNumber, TokenCode: tc.tokenCode})
+ requireAWSError(t, err, awserrors.ErrorValidationError, tc.wantMessage)
+ assert.Nil(t, out)
+ })
+ }
}
func TestGetSessionToken_RejectsMissingUserName(t *testing.T) {
diff --git a/spinifex/handlers/sts/oidc_jwks_test.go b/spinifex/handlers/sts/oidc_jwks_test.go
index 827e58ee0..6549eabb6 100644
--- a/spinifex/handlers/sts/oidc_jwks_test.go
+++ b/spinifex/handlers/sts/oidc_jwks_test.go
@@ -80,7 +80,7 @@ func TestFetchClusterJWKS_RoundTrip(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1)
+ kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012")
require.NoError(t, err)
want := &JWKS{Keys: []JWK{
@@ -104,7 +104,7 @@ func TestFetchClusterJWKS_EmptyKeysIsError(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1)
+ kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012")
require.NoError(t, err)
_, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey("empty-cluster"), []byte(`{"keys":[]}`))
diff --git a/spinifex/handlers/sts/service_impl.go b/spinifex/handlers/sts/service_impl.go
index 7520d471d..5edcb91ee 100644
--- a/spinifex/handlers/sts/service_impl.go
+++ b/spinifex/handlers/sts/service_impl.go
@@ -37,10 +37,9 @@ type STSServiceImpl struct {
var _ STSService = (*STSServiceImpl)(nil)
// NewSTSServiceImpl constructs an STSServiceImpl. masterKey must be the 32-byte
-// key shared with IAMServiceImpl. clusterSize sets the JetStream replication factor;
-// pass 1 for single-node or test setups. The context bounds bucket creation and
-// the schema migration only.
-func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte, clusterSize int) (*STSServiceImpl, error) {
+// key shared with IAMServiceImpl. The context bounds bucket creation and the
+// schema migration only.
+func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte) (*STSServiceImpl, error) {
if natsConn == nil {
return nil, errors.New("nil NATS connection")
}
@@ -51,21 +50,18 @@ func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers
return nil, fmt.Errorf("master key must be %d bytes, got %d", masterKeySize, len(masterKey))
}
- replicas := max(clusterSize, 1)
-
js, err := jetstream.New(natsConn)
if err != nil {
return nil, fmt.Errorf("get JetStream context: %w", err)
}
- sessions, err := initSessionCredentialsStore(ctx, js, replicas)
+ sessions, err := initSessionCredentialsStore(ctx, js)
if err != nil {
return nil, fmt.Errorf("init session credentials bucket: %w", err)
}
slog.Info("STS service initialized",
- "sessions_bucket", KVBucketSessionCredentials,
- "replicas", replicas)
+ "sessions_bucket", KVBucketSessionCredentials)
return &STSServiceImpl{
natsConn: natsConn,
diff --git a/spinifex/handlers/sts/service_impl_test.go b/spinifex/handlers/sts/service_impl_test.go
index a8124f4da..7914033a0 100644
--- a/spinifex/handlers/sts/service_impl_test.go
+++ b/spinifex/handlers/sts/service_impl_test.go
@@ -24,17 +24,17 @@ func newTestSetup(t *testing.T) (*STSServiceImpl, *nats.Conn) {
masterKey, err := handlers_iam.GenerateMasterKey()
require.NoError(t, err)
- iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
- stsSvc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1)
+ stsSvc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey)
require.NoError(t, err)
return stsSvc, nc
}
func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) {
masterKey := bytes.Repeat([]byte{0x01}, masterKeySize)
- _, err := NewSTSServiceImpl(t.Context(), nil, nopIAMService{}, masterKey, 1)
+ _, err := NewSTSServiceImpl(t.Context(), nil, nopIAMService{}, masterKey)
require.Error(t, err)
assert.Contains(t, err.Error(), "NATS")
}
@@ -42,7 +42,7 @@ func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) {
func TestNewSTSServiceImpl_RejectsNilIAMService(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
masterKey := bytes.Repeat([]byte{0x01}, masterKeySize)
- _, err := NewSTSServiceImpl(t.Context(), nc, nil, masterKey, 1)
+ _, err := NewSTSServiceImpl(t.Context(), nc, nil, masterKey)
require.Error(t, err)
assert.Contains(t, err.Error(), "IAM")
}
@@ -51,7 +51,7 @@ func TestNewSTSServiceImpl_RejectsWrongMasterKeySize(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
for _, size := range []int{0, 1, 16, 31, 33, 64} {
t.Run("size", func(t *testing.T) {
- _, err := NewSTSServiceImpl(t.Context(), nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size), 1)
+ _, err := NewSTSServiceImpl(t.Context(), nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size))
require.Error(t, err)
assert.Contains(t, err.Error(), "master key")
})
@@ -68,10 +68,10 @@ func TestNewSTSServiceImpl_NormalisesNegativeClusterSize(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
masterKey, err := handlers_iam.GenerateMasterKey()
require.NoError(t, err)
- iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
+ iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
require.NoError(t, err)
- svc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 0)
+ svc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey)
require.NoError(t, err)
require.NotNil(t, svc)
}
diff --git a/spinifex/handlers/sts/session_credentials.go b/spinifex/handlers/sts/session_credentials.go
index a68fd3e16..2e94ef0de 100644
--- a/spinifex/handlers/sts/session_credentials.go
+++ b/spinifex/handlers/sts/session_credentials.go
@@ -65,12 +65,11 @@ type SessionCredential struct {
// sessionCredentialsConfig describes the session-credentials bucket. History is
// fixed at 1: credentials are write-once at mint and delete-once at expiry.
-func sessionCredentialsConfig(replicas int) kvstore.Config {
+func sessionCredentialsConfig() kvstore.Config {
return kvstore.Config{
- Name: KVBucketSessionCredentials,
- History: 1,
- Replicas: replicas,
- Missing: "session credentials KV bucket not initialized",
+ Name: KVBucketSessionCredentials,
+ History: 1,
+ Missing: "session credentials KV bucket not initialized",
OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
return migrate.DefaultRegistry.RunKV(ctx, KVBucketSessionCredentials, kv, KVBucketSessionCredentialsVersion)
},
@@ -78,9 +77,8 @@ func sessionCredentialsConfig(replicas int) kvstore.Config {
}
// initSessionCredentialsStore opens (or creates) the session-credentials bucket.
-func initSessionCredentialsStore(ctx context.Context, js jetstream.JetStream, replicas int) (*kvstore.Store[SessionCredential], error) {
- // kvstore clamps replicas to a minimum of 1, so a zero clusterSize still creates.
- store := kvstore.New[SessionCredential](js, sessionCredentialsConfig(replicas))
+func initSessionCredentialsStore(ctx context.Context, js jetstream.JetStream) (*kvstore.Store[SessionCredential], error) {
+ store := kvstore.New[SessionCredential](js, sessionCredentialsConfig())
// Opened eagerly: a bucket that cannot be created must fail service
// construction, not the first AssumeRole that needs to mint into it.
if _, err := store.KV(ctx); err != nil {
diff --git a/spinifex/handlers/sts/session_credentials_test.go b/spinifex/handlers/sts/session_credentials_test.go
index 1ce66839d..64af3a1a9 100644
--- a/spinifex/handlers/sts/session_credentials_test.go
+++ b/spinifex/handlers/sts/session_credentials_test.go
@@ -17,7 +17,7 @@ import (
func setupBucket(t *testing.T) *kvstore.Store[SessionCredential] {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
- store, err := initSessionCredentialsStore(t.Context(), testutil.NewJetStream(t, nc), 1)
+ store, err := initSessionCredentialsStore(t.Context(), testutil.NewJetStream(t, nc))
require.NoError(t, err)
return store
}
@@ -52,7 +52,7 @@ func TestInitSessionCredentialsBucket_StampsVersion(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- store, err := initSessionCredentialsStore(t.Context(), js, 1)
+ store, err := initSessionCredentialsStore(t.Context(), js)
require.NoError(t, err)
kv, err := store.KV(t.Context())
require.NoError(t, err)
@@ -66,11 +66,11 @@ func TestInitSessionCredentialsBucket_Idempotent(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- store1, err := initSessionCredentialsStore(t.Context(), js, 1)
+ store1, err := initSessionCredentialsStore(t.Context(), js)
require.NoError(t, err)
// Reopen — must return the same bucket without error.
- store2, err := initSessionCredentialsStore(t.Context(), js, 1)
+ store2, err := initSessionCredentialsStore(t.Context(), js)
require.NoError(t, err)
assert.Equal(t, store1.Name(), store2.Name())
}
diff --git a/spinifex/handlers/sts/validation.go b/spinifex/handlers/sts/validation.go
index 1eb9842cd..51f8b26dd 100644
--- a/spinifex/handlers/sts/validation.go
+++ b/spinifex/handlers/sts/validation.go
@@ -2,7 +2,10 @@ package handlers_sts
import (
"fmt"
+ "regexp"
+ "strconv"
"strings"
+ "unicode/utf8"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
)
@@ -14,6 +17,70 @@ type constraintViolation struct {
constraint string
}
+// stringConstraint is a model string shape: a pattern, as AWS prints it without
+// anchors, and a length range counted in characters.
+type stringConstraint struct {
+ pattern *regexp.Regexp
+ patternText string
+ minLen, maxLen int
+}
+
+var (
+ // roleSessionNameConstraint is roleSessionNameRegex without the length bound, so
+ // the charset and length constraints are reported separately, as AWS does.
+ roleSessionNameConstraint = stringConstraint{
+ regexp.MustCompile(`^[A-Za-z0-9_+=,.@-]*$`), `[\w+=,.@-]*`, minRoleSessionNameLength, maxRoleSessionNameLength,
+ }
+ serialNumberConstraint = stringConstraint{regexp.MustCompile(`^[A-Za-z0-9_+=/:,.@-]*$`), `[\w+=/:,.@-]*`, 9, 256}
+ tokenCodeConstraint = stringConstraint{regexp.MustCompile(`^[0-9]*$`), `[\d]*`, 6, 6}
+)
+
+// check reports value's pattern violation, then its length violation.
+func (c stringConstraint) check(field, value string) []constraintViolation {
+ var violations []constraintViolation
+ if !c.pattern.MatchString(value) {
+ violations = append(violations, constraintViolation{field, value,
+ "Member must satisfy regular expression pattern: " + c.patternText})
+ }
+ if n := utf8.RuneCountInString(value); n < c.minLen {
+ violations = append(violations, constraintViolation{field, value,
+ fmt.Sprintf("Member must have length greater than or equal to %d", c.minLen)})
+ } else if n > c.maxLen {
+ violations = append(violations, constraintViolation{field, value,
+ fmt.Sprintf("Member must have length less than or equal to %d", c.maxLen)})
+ }
+ return violations
+}
+
+// durationViolations range-checks a present durationSeconds against [900, maxSeconds].
+func durationViolations(d *int64, maxSeconds int64) []constraintViolation {
+ if d == nil {
+ return nil
+ }
+ value := strconv.FormatInt(*d, 10)
+ if *d < minDurationSeconds {
+ return []constraintViolation{{"durationSeconds", value,
+ fmt.Sprintf("Member must have value greater than or equal to %d", minDurationSeconds)}}
+ }
+ if *d > maxSeconds {
+ return []constraintViolation{{"durationSeconds", value,
+ fmt.Sprintf("Member must have value less than or equal to %d", maxSeconds)}}
+ }
+ return nil
+}
+
+// mfaViolations checks the MFA members when present, an empty value included.
+func mfaViolations(serialNumber, tokenCode *string) []constraintViolation {
+ var violations []constraintViolation
+ if serialNumber != nil {
+ violations = append(violations, serialNumberConstraint.check("serialNumber", *serialNumber)...)
+ }
+ if tokenCode != nil {
+ violations = append(violations, tokenCodeConstraint.check("tokenCode", *tokenCode)...)
+ }
+ return violations
+}
+
// validationError renders violations as the single ValidationError AWS returns for
// failed model constraints, in the order given, or nil when there are none.
func validationError(violations []constraintViolation) error {
diff --git a/spinifex/handlers/systemvpc/systemvpc.go b/spinifex/handlers/systemvpc/systemvpc.go
index 728637df5..7185af777 100644
--- a/spinifex/handlers/systemvpc/systemvpc.go
+++ b/spinifex/handlers/systemvpc/systemvpc.go
@@ -19,6 +19,7 @@ import (
"hash/fnv"
"log/slog"
"net/netip"
+ "slices"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -431,12 +432,9 @@ func ensureRouteTable(ctx context.Context, rtp RouteTableProvisioner, spec Spec,
// associatedWith reports whether subnetID already holds an explicit association
// on rt, as of the describe ensureRouteTable adopted it from.
func associatedWith(rt *ec2.RouteTable, subnetID string) bool {
- for _, assoc := range rt.Associations {
- if aws.StringValue(assoc.SubnetId) == subnetID && !aws.BoolValue(assoc.Main) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(rt.Associations, func(assoc *ec2.RouteTableAssociation) bool {
+ return aws.StringValue(assoc.SubnetId) == subnetID && !aws.BoolValue(assoc.Main)
+ })
}
// describeOrCreateRouteTable returns the role-tagged route table, adopting an
diff --git a/spinifex/ingress/aws/query/parser_fuzz_test.go b/spinifex/ingress/aws/query/parser_fuzz_test.go
new file mode 100644
index 000000000..7e7f01026
--- /dev/null
+++ b/spinifex/ingress/aws/query/parser_fuzz_test.go
@@ -0,0 +1,151 @@
+package query
+
+import (
+ "fmt"
+ "net/url"
+ "reflect"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/aws/aws-sdk-go/service/elbv2"
+ "github.com/aws/aws-sdk-go/service/rds"
+)
+
+// Input shapes covering each list serialisation: EC2 Prefix.N, ELBv2
+// Prefix.member.N, RDS locationNameList, and deep nesting under a struct.
+var fuzzTargets = []func() any{
+ func() any { return &ec2.DescribeInstancesInput{} },
+ func() any { return &ec2.RunInstancesInput{} },
+ func() any { return &ec2.CreateLaunchTemplateInput{} },
+ func() any { return &elbv2.RegisterTargetsInput{} },
+ func() any { return &elbv2.CreateListenerInput{} },
+ func() any { return &rds.CreateDBInstanceInput{} },
+}
+
+// queryParams builds the params map the way the gateway does from a request
+// body: url.ParseQuery, keeping the last value of a repeated key.
+func queryParams(body string) (map[string]string, bool) {
+ values, err := url.ParseQuery(body)
+ if err != nil {
+ return nil, false
+ }
+ params := make(map[string]string, len(values))
+ for key, vs := range values {
+ params[key] = vs[len(vs)-1]
+ }
+ return params, true
+}
+
+// countSliceElems tallies materialised slice elements by nesting depth,
+// failing on any slice past MaxSliceLen.
+func countSliceElems(t *testing.T, v reflect.Value, depth int, counts map[int]int) {
+ switch v.Kind() {
+ case reflect.Pointer, reflect.Interface:
+ if !v.IsNil() {
+ countSliceElems(t, v.Elem(), depth, counts)
+ }
+ case reflect.Struct:
+ for i := range v.NumField() {
+ if v.Type().Field(i).IsExported() {
+ countSliceElems(t, v.Field(i), depth, counts)
+ }
+ }
+ case reflect.Slice:
+ if v.Type().Elem().Kind() == reflect.Uint8 {
+ return
+ }
+ if v.Len() > MaxSliceLen {
+ t.Fatalf("slice of %d entries exceeds MaxSliceLen %d", v.Len(), MaxSliceLen)
+ }
+ counts[depth] += v.Len()
+ for i := range v.Len() {
+ countSliceElems(t, v.Index(i), depth+1, counts)
+ }
+ }
+}
+
+// FuzzQueryParamsToStruct pins the properties the parser owes untrusted input:
+// it never panics, and on success every list level holds no more entries than
+// the request had parameters, so allocation is bounded by the input.
+func FuzzQueryParamsToStruct(f *testing.F) {
+ for _, seed := range []string{
+ "Filter.1.Name=instance-type&Filter.1.Value.1=t2.micro&Filter.3.Name=x&Filter.3.Value.1=y",
+ "Filter.2.Name=skipped&Filter.2.Value.1=x",
+ "Filter.999999.Name=ignored",
+ "Filter.01.Name=a&Filter.+1.Value.1=b&Filter.-1.Name=c",
+ "Filter.1.Value.item.1=a&Filter.1.Value.2=b",
+ "InstanceId.1=i-1&InstanceId.2=i-2&MaxResults=5&DryRun=true",
+ "MaxResults=99999999999999999999&DryRun=maybe",
+ "ImageId=ami-1&MinCount=1&MaxCount=1&UserData=aGVsbG8=&BlockDeviceMapping.1.DeviceName=/dev/sda1&BlockDeviceMapping.1.Ebs.VolumeSize=20",
+ "TagSpecification.1.ResourceType=instance&TagSpecification.1.Tag.1.Key=k&TagSpecification.1.Tag.1.Value=v",
+ "LaunchTemplateData.NetworkInterface.1.PrivateIpAddresses.1.PrivateIpAddress=10.0.0.1&LaunchTemplateData.InstanceMarketOptions.SpotOptions.ValidUntil=2026-07-10T12:00:00Z",
+ "TargetGroupArn=arn&Targets.member.1.Id=i-1&Targets.member.1.Port=80&Targets.member.2.Id=i-2",
+ "DefaultActions.member.1.Type=forward&DefaultActions.member.1.ForwardConfig.TargetGroups.member.1.Weight=1",
+ "DBInstanceIdentifier=db&Tags.Tag.1.Key=k&Tags.Tag.1.Value=v&Tags.member.2.Key=x",
+ } {
+ f.Add(seed)
+ }
+
+ f.Fuzz(func(t *testing.T, body string) {
+ params, ok := queryParams(body)
+ if !ok {
+ return
+ }
+ for _, target := range fuzzTargets {
+ out := target()
+ if err := QueryParamsToStruct(params, out); err != nil {
+ continue
+ }
+ counts := map[int]int{}
+ countSliceElems(t, reflect.ValueOf(out), 0, counts)
+ for depth, n := range counts {
+ if n > len(params) {
+ t.Fatalf("%T: %d list entries at depth %d from %d params: %q", out, n, depth, len(params), body)
+ }
+ }
+ }
+ })
+}
+
+// FuzzQueryParamsToStruct_EntryPastGapIsIgnored pins AWS's gap rule on
+// arbitrary input: adding an entry one past the first gap in any list leaves
+// the parsed request unchanged.
+func FuzzQueryParamsToStruct_EntryPastGapIsIgnored(f *testing.F) {
+ for _, seed := range []string{
+ "Filter.1.Name=a&Filter.1.Value.1=b&Filter.3.Name=c",
+ "Filter.2.Name=a",
+ "Filter.1.Name=a&Filter.1.Value.2=b",
+ "InstanceId.1=i-1&InstanceId.3=i-3",
+ "Filter.01.Name=a&Filter.1.Value.1=b",
+ "Filter.1.Value.item.1=a",
+ } {
+ f.Add(seed)
+ }
+
+ f.Fuzz(func(t *testing.T, body string) {
+ params, ok := queryParams(body)
+ if !ok {
+ return
+ }
+ want := &ec2.DescribeInstancesInput{}
+ if err := QueryParamsToStruct(params, want); err != nil {
+ return
+ }
+
+ pastGap := len(want.Filters) + 2
+ params[fmt.Sprintf("Filter.%d.Name", pastGap)] = "past-gap"
+ params[fmt.Sprintf("Filter.%d.Value.1", pastGap)] = "past-gap"
+ params[fmt.Sprintf("InstanceId.%d", len(want.InstanceIds)+2)] = "past-gap"
+ for i, filter := range want.Filters {
+ params[fmt.Sprintf("Filter.%d.Value.%d", i+1, len(filter.Values)+2)] = "past-gap"
+ }
+
+ got := &ec2.DescribeInstancesInput{}
+ if err := QueryParamsToStruct(params, got); err != nil {
+ t.Fatalf("entry past the gap turned success into %v: %q", err, body)
+ }
+ if !reflect.DeepEqual(want, got) {
+ t.Fatalf("entry past the gap changed the parse of %q:\nwant %s\ngot %s", body, want.String(), got.String())
+ }
+ })
+}
diff --git a/spinifex/lbagent/healthcheck.go b/spinifex/lbagent/healthcheck.go
index 14424755e..4bd752283 100644
--- a/spinifex/lbagent/healthcheck.go
+++ b/spinifex/lbagent/healthcheck.go
@@ -76,7 +76,7 @@ func probeHTTP(t HealthTarget) bool {
}
url := fmt.Sprintf("%s://%s%s", scheme, t.Address, path)
- resp, err := client.Get(url) //nolint:noctx // short fixed-timeout probe
+ resp, err := client.Get(url)
if err != nil {
slog.Warn("HTTP probe failed", "url", url, "err", err)
return false
diff --git a/spinifex/migrate/version_readers.go b/spinifex/migrate/version_readers.go
index 1dc388144..748b6ad4e 100644
--- a/spinifex/migrate/version_readers.go
+++ b/spinifex/migrate/version_readers.go
@@ -42,9 +42,7 @@ func (r *TOMLVersionReader) ReadVersion(path string) (int, error) {
}
// Handle "1.0" style — take the integer part before the dot.
- if idx := strings.Index(raw, "."); idx >= 0 {
- raw = raw[:idx]
- }
+ raw, _, _ = strings.Cut(raw, ".")
v, err := strconv.Atoi(raw)
if err != nil {
diff --git a/spinifex/network/external/exonet/allocator.go b/spinifex/network/external/exonet/allocator.go
new file mode 100644
index 000000000..ad4f240d3
--- /dev/null
+++ b/spinifex/network/external/exonet/allocator.go
@@ -0,0 +1,282 @@
+// Package exonet implements external.Allocator against Exoscale Elastic IPs.
+//
+// A manual Exoscale EIP is delivered to the instance addressed to itself, and
+// the instance may use it as a source, so the address Spinifex hands out is the
+// address on the wire. That makes this the OCI allocator without its hardest
+// half: no private-IP object, no VNIC and no public-to-private mapping. What
+// remains is the part every cloud shares — the address only exists once the
+// provider has been asked for it, so allocation is an API round trip.
+//
+// One node, one Exoscale instance: Allocate creates the EIP and attaches it to
+// this node straight away, and the routed EIP path takes it from there exactly
+// as it does on bare metal.
+package exonet
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/netip"
+ "strings"
+
+ "github.com/mulgadc/spinifex/spinifex/cloud/exoscale"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/network/external"
+)
+
+// MarkerPrefix starts the description of every EIP this package creates.
+// Exoscale EIPs carry no labels, so the description is the only place to say
+// who made one, and Reconcile never touches an EIP without our marker.
+const MarkerPrefix = "spinifex:"
+
+// purposeEIP duplicates handlers/ec2/vpc.PurposeEIP so the allocator can tell
+// an Elastic IP from an auto-assigned address without importing handlers.
+const purposeEIP = "eip"
+
+// Binding is one allocated EIP and the AWS identity that claimed it, keyed in
+// the record by the address.
+type Binding struct {
+ EIPID string `json:"eip_id"`
+ Purpose string `json:"purpose,omitempty"`
+ AllocationID string `json:"allocation_id,omitempty"`
+ ENIID string `json:"eni_id,omitempty"`
+ InstanceID string `json:"instance_id,omitempty"`
+ // NodeInstanceID is the Exoscale instance the EIP is attached to. With one
+ // node it is always ours; recording it now means a second node adds a
+ // check rather than a migration.
+ NodeInstanceID string `json:"node_instance_id"`
+}
+
+// Record is the per-pool binding set.
+type Record struct {
+ Bindings map[string]Binding `json:"bindings"`
+}
+
+// Store persists bindings. JetStream KV in store.go; an interface so the
+// allocator's tests need no NATS.
+type Store interface {
+ Mutate(ctx context.Context, poolName string, fn func(*Record) (bool, error)) error
+ Get(ctx context.Context, poolName string) (Record, error)
+}
+
+// Config wires one pool to one Exoscale instance.
+type Config struct {
+ Pool external.ExternalPoolConfig
+ // InstanceID is this node's Exoscale instance, which every EIP attaches to.
+ InstanceID string
+}
+
+// PoolAllocator implements external.Allocator against Exoscale.
+type PoolAllocator struct {
+ client exoscale.Client
+ store Store
+ cfg Config
+}
+
+var _ external.Allocator = (*PoolAllocator)(nil)
+
+// New constructs an allocator bound to a single pool.
+func New(client exoscale.Client, store Store, cfg Config) (*PoolAllocator, error) {
+ if client == nil {
+ return nil, errors.New("exonet: nil Exoscale client")
+ }
+ if store == nil {
+ return nil, errors.New("exonet: nil store")
+ }
+ if cfg.InstanceID == "" {
+ return nil, fmt.Errorf("exonet: pool %q missing instance id", cfg.Pool.Name)
+ }
+ return &PoolAllocator{client: client, store: store, cfg: cfg}, nil
+}
+
+// Pool returns the pool this allocator manages.
+func (a *PoolAllocator) Pool() external.ExternalPoolConfig { return a.cfg.Pool }
+
+// Allocate creates a marked EIP, attaches it to this node, and records it.
+//
+// The EIP is created before the record is written. A crash in between leaves a
+// marked EIP with no binding, which Reconcile finds and deletes; the reverse
+// order could leave a binding naming an address that never existed.
+func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateRequest) (netip.Addr, error) {
+ if req.PoolName != "" && req.PoolName != a.cfg.Pool.Name {
+ return netip.Addr{}, fmt.Errorf("exonet: pool mismatch (got %q, bound to %q)", req.PoolName, a.cfg.Pool.Name)
+ }
+
+ eip, err := a.client.CreateElasticIP(ctx, a.marker(req))
+ if err != nil {
+ return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("exonet: create elastic IP: %w", err), err)
+ }
+
+ if err := a.client.AttachElasticIP(ctx, a.cfg.InstanceID, eip.ID); err != nil {
+ a.undo(ctx, eip, false)
+ return netip.Addr{}, fmt.Errorf("exonet: attach elastic IP %s to %s: %w", eip.ID, a.cfg.InstanceID, err)
+ }
+
+ key := eip.Address.String()
+ binding := Binding{
+ EIPID: eip.ID,
+ Purpose: req.Purpose,
+ AllocationID: req.AllocationID,
+ ENIID: req.ENIID,
+ InstanceID: req.InstanceID,
+ NodeInstanceID: a.cfg.InstanceID,
+ }
+ err = a.store.Mutate(ctx, a.cfg.Pool.Name, func(rec *Record) (bool, error) {
+ if rec.Bindings == nil {
+ rec.Bindings = map[string]Binding{}
+ }
+ rec.Bindings[key] = binding
+ return true, nil
+ })
+ if err != nil {
+ a.undo(ctx, eip, true)
+ return netip.Addr{}, fmt.Errorf("exonet: record binding for %s: %w", key, err)
+ }
+
+ slog.InfoContext(ctx, "exonet allocated external IP",
+ "pool", a.cfg.Pool.Name, "public_ip", key, "eip_id", eip.ID, "purpose", req.Purpose)
+ return eip.Address, nil
+}
+
+// Release detaches and deletes the EIP and drops the binding. ownerENIID scopes
+// the release as StaticPoolAllocator.Release does, so a teardown sweep naming a
+// stale owner is a no-op rather than a double-free.
+//
+// The record is dropped first: if the Exoscale calls then fail, the EIP is an
+// unbound marked address that Reconcile collects, not a record that lies.
+func (a *PoolAllocator) Release(ctx context.Context, poolName string, ip netip.Addr, ownerENIID string) error {
+ if poolName != "" && poolName != a.cfg.Pool.Name {
+ return fmt.Errorf("exonet: pool mismatch (got %q, bound to %q)", poolName, a.cfg.Pool.Name)
+ }
+ if !ip.IsValid() {
+ return errors.New("exonet: invalid ip")
+ }
+ key := ip.String()
+
+ var released Binding
+ var found bool
+ err := a.store.Mutate(ctx, a.cfg.Pool.Name, func(rec *Record) (bool, error) {
+ released, found = Binding{}, false
+
+ b, ok := rec.Bindings[key]
+ if !ok {
+ if ownerENIID != "" {
+ return false, nil
+ }
+ return false, fmt.Errorf("exonet: %s not allocated in pool %s", key, a.cfg.Pool.Name)
+ }
+ if ownerENIID != "" && b.ENIID != "" && b.ENIID != ownerENIID {
+ slog.InfoContext(ctx, "exonet release skip — address reassigned to a different ENI (stale release)",
+ "pool", a.cfg.Pool.Name, "public_ip", key,
+ "stale_owner_eni", ownerENIID, "current_owner_eni", b.ENIID)
+ return false, nil
+ }
+ // Only ReleaseAddress, which names no interface, may free an Elastic IP.
+ // Instance teardown must leave it allocated, as AWS does.
+ if ownerENIID != "" && (b.Purpose == purposeEIP || b.AllocationID != "") {
+ slog.InfoContext(ctx, "exonet release skip — address belongs to an Elastic IP allocation",
+ "pool", a.cfg.Pool.Name, "public_ip", key,
+ "allocation_id", b.AllocationID, "owner_eni", ownerENIID)
+ return false, nil
+ }
+ delete(rec.Bindings, key)
+ released, found = b, true
+ return true, nil
+ })
+ if err != nil || !found {
+ return err
+ }
+
+ if err := a.detachAndDelete(ctx, released.NodeInstanceID, released.EIPID); err != nil {
+ return err
+ }
+ slog.InfoContext(ctx, "exonet released external IP",
+ "pool", a.cfg.Pool.Name, "public_ip", key, "eip_id", released.EIPID)
+ return nil
+}
+
+// detachAndDelete returns an EIP to Exoscale. Not-found on either step is
+// success: the address is already in the state we wanted. A failed detach is
+// not final, because the delete may still succeed on an EIP nobody holds.
+func (a *PoolAllocator) detachAndDelete(ctx context.Context, instanceID, eipID string) error {
+ var detachErr error
+ if instanceID != "" {
+ if err := a.client.DetachElasticIP(ctx, instanceID, eipID); err != nil && !errors.Is(err, exoscale.ErrNotFound) {
+ detachErr = fmt.Errorf("exonet: detach elastic IP %s: %w", eipID, err)
+ }
+ }
+ if err := a.client.DeleteElasticIP(ctx, eipID); err != nil && !errors.Is(err, exoscale.ErrNotFound) {
+ return errors.Join(detachErr, fmt.Errorf("exonet: delete elastic IP %s: %w", eipID, err))
+ }
+ if detachErr != nil {
+ slog.WarnContext(ctx, "exonet: detach failed but the delete succeeded", "eip_id", eipID, "error", detachErr)
+ }
+ return nil
+}
+
+// undo rolls back a partial Allocate. Failures are logged, not returned: the
+// caller is already failing, and the EIP keeps its marker for Reconcile.
+func (a *PoolAllocator) undo(ctx context.Context, eip exoscale.ElasticIP, attached bool) {
+ instanceID := ""
+ if attached {
+ instanceID = a.cfg.InstanceID
+ }
+ if err := a.detachAndDelete(ctx, instanceID, eip.ID); err != nil {
+ slog.WarnContext(ctx, "exonet: could not roll back elastic IP; reconcile will collect it",
+ "eip_id", eip.ID, "public_ip", eip.Address.String(), "error", err)
+ }
+}
+
+// markerFor is the description prefix for EIPs attached to instanceID. The
+// instance is in it so two Spinifex nodes in one organisation never judge each
+// other's addresses.
+func markerFor(instanceID string) string { return MarkerPrefix + instanceID + ":" }
+
+// marker is this allocation's description: our prefix plus the AWS identity,
+// so the Exoscale portal shows what claimed the address.
+func (a *PoolAllocator) marker(req external.AllocateRequest) string {
+ id := "unclaimed"
+ switch {
+ case req.AllocationID != "":
+ id = req.AllocationID
+ case req.ENIID != "":
+ id = req.ENIID
+ case req.InstanceID != "":
+ id = req.InstanceID
+ }
+ return markerFor(a.cfg.InstanceID) + id
+}
+
+// ours reports whether an EIP carries this node's marker.
+func (a *PoolAllocator) ours(e exoscale.ElasticIP) bool {
+ return strings.HasPrefix(e.Description, markerFor(a.cfg.InstanceID))
+}
+
+// capacityErr maps the organisation quota onto the AWS code for the call. An
+// Elastic IP allocation that hits a limit is AddressLimitExceeded in AWS; an
+// auto-assigned address has no such limit there, so it reports capacity.
+func capacityErr(purpose string, wrapped, raw error) error {
+ if !errors.Is(raw, exoscale.ErrQuotaExceeded) {
+ return wrapped
+ }
+ code := awserrors.ErrorInsufficientAddressCapacity
+ if purpose == purposeEIP {
+ code = awserrors.ErrorAddressLimitExceeded
+ }
+ return fmt.Errorf("%w: %w", errors.New(code), wrapped)
+}
+
+// BindingFor returns the binding behind an allocated address.
+func (a *PoolAllocator) BindingFor(ctx context.Context, ip netip.Addr) (Binding, bool, error) {
+ rec, err := a.store.Get(ctx, a.cfg.Pool.Name)
+ if errors.Is(err, kvstore.ErrNotFound) {
+ return Binding{}, false, nil
+ }
+ if err != nil {
+ return Binding{}, false, err
+ }
+ b, ok := rec.Bindings[ip.String()]
+ return b, ok, nil
+}
diff --git a/spinifex/network/external/exonet/allocator_test.go b/spinifex/network/external/exonet/allocator_test.go
new file mode 100644
index 000000000..0f4871d09
--- /dev/null
+++ b/spinifex/network/external/exonet/allocator_test.go
@@ -0,0 +1,250 @@
+package exonet
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "maps"
+ "net/netip"
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/mulgadc/spinifex/spinifex/cloud/exoscale"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/network/external"
+)
+
+const (
+ nodeID = "11111111-1111-4111-8111-111111111111"
+ otherID = "22222222-2222-4222-8222-222222222222"
+ pool = "exo"
+)
+
+// memStore mirrors KVStore, including not-found for a pool never written.
+type memStore struct {
+ mu sync.Mutex
+ records map[string]Record
+ mutateErr error
+}
+
+func newMemStore() *memStore { return &memStore{records: map[string]Record{}} }
+
+func (m *memStore) Mutate(_ context.Context, poolName string, fn func(*Record) (bool, error)) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.mutateErr != nil {
+ return m.mutateErr
+ }
+ rec := Record{Bindings: map[string]Binding{}}
+ maps.Copy(rec.Bindings, m.records[poolName].Bindings)
+ changed, err := fn(&rec)
+ if err != nil {
+ return err
+ }
+ if changed {
+ m.records[poolName] = rec
+ }
+ return nil
+}
+
+func (m *memStore) Get(_ context.Context, poolName string) (Record, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ r, ok := m.records[poolName]
+ if !ok {
+ return Record{}, fmt.Errorf("%w: %s", kvstore.ErrNotFound, poolName)
+ }
+ rec := Record{Bindings: map[string]Binding{}}
+ maps.Copy(rec.Bindings, r.Bindings)
+ return rec, nil
+}
+
+func newTest(t *testing.T, quota int) (*PoolAllocator, *exoscale.Fake, *memStore) {
+ t.Helper()
+ fake := exoscale.NewFake("de-fra-1", quota)
+ store := newMemStore()
+ a, err := New(fake, store, Config{Pool: external.ExternalPoolConfig{Name: pool, Source: external.SourceExoscale}, InstanceID: nodeID})
+ require.NoError(t, err)
+ return a, fake, store
+}
+
+func binding(t *testing.T, s *memStore, ip netip.Addr) (Binding, bool) {
+ t.Helper()
+ rec, err := s.Get(context.Background(), pool)
+ if errors.Is(err, kvstore.ErrNotFound) {
+ return Binding{}, false
+ }
+ require.NoError(t, err)
+ b, ok := rec.Bindings[ip.String()]
+ return b, ok
+}
+
+func TestAllocate_CreatesAttachesAndRecords(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ ip, err := a.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eip", AllocationID: "eipalloc-1"})
+ require.NoError(t, err)
+
+ b, ok := binding(t, store, ip)
+ require.True(t, ok)
+ assert.Equal(t, nodeID, b.NodeInstanceID)
+ assert.Equal(t, "eipalloc-1", b.AllocationID)
+
+ inst, attached := fake.AttachedTo(b.EIPID)
+ assert.True(t, attached)
+ assert.Equal(t, nodeID, inst)
+
+ eips, err := fake.ListElasticIPs(context.Background())
+ require.NoError(t, err)
+ require.Len(t, eips, 1)
+ assert.Equal(t, "spinifex:"+nodeID+":eipalloc-1", eips[0].Description)
+}
+
+func TestAllocate_QuotaMapsToTheAWSCodeForThePurpose(t *testing.T) {
+ cases := []struct{ purpose, want string }{
+ {"eip", awserrors.ErrorAddressLimitExceeded},
+ {"eni-public", awserrors.ErrorInsufficientAddressCapacity},
+ }
+ for _, tc := range cases {
+ t.Run(tc.purpose, func(t *testing.T) {
+ a, fake, _ := newTest(t, 1)
+ fake.Seed(exoscale.ElasticIP{ID: "operator", Address: netip.MustParseAddr("89.145.162.53"), Description: "mulga1"})
+ _, err := a.Allocate(context.Background(), external.AllocateRequest{Purpose: tc.purpose})
+ require.Error(t, err)
+ code, ok := awserrors.ResolveErrorCode(err)
+ require.True(t, ok, "no AWS code in %v", err)
+ assert.Equal(t, tc.want, code)
+ })
+ }
+}
+
+func TestAllocate_AttachFailureDeletesTheNewEIP(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ fake.Fail["attach"] = errors.New("boom")
+ _, err := a.Allocate(context.Background(), external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.Error(t, err)
+ assert.Empty(t, fake.IDs())
+ _, err = store.Get(context.Background(), pool)
+ assert.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestAllocate_RecordFailureDetachesAndDeletes(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ store.mutateErr = errors.New("kv down")
+ _, err := a.Allocate(context.Background(), external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.Error(t, err)
+ assert.Empty(t, fake.IDs())
+ assert.Contains(t, fake.Calls, "detach")
+}
+
+func TestRelease_DeletesTheEIP(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ ctx := context.Background()
+ ip, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.NoError(t, err)
+
+ require.NoError(t, a.Release(ctx, pool, ip, "eni-1"))
+ assert.Empty(t, fake.IDs())
+ _, ok := binding(t, store, ip)
+ assert.False(t, ok)
+}
+
+func TestRelease_InstanceTeardownKeepsAnElasticIP(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ ctx := context.Background()
+ ip, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eip", AllocationID: "eipalloc-1"})
+ require.NoError(t, err)
+ require.NoError(t, a.Release(ctx, pool, ip, "eni-1")) // scoped: teardown of an associated instance
+ assert.Len(t, fake.IDs(), 1)
+ _, ok := binding(t, store, ip)
+ assert.True(t, ok)
+
+ require.NoError(t, a.Release(ctx, pool, ip, "")) // ReleaseAddress
+ assert.Empty(t, fake.IDs())
+}
+
+func TestRelease_StaleOwnerIsANoop(t *testing.T) {
+ a, fake, _ := newTest(t, 0)
+ ctx := context.Background()
+ ip, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-new"})
+ require.NoError(t, err)
+ require.NoError(t, a.Release(ctx, pool, ip, "eni-old"))
+ assert.Len(t, fake.IDs(), 1)
+}
+
+func TestRelease_EIPAlreadyGoneAtExoscaleSucceeds(t *testing.T) {
+ a, fake, _ := newTest(t, 0)
+ ctx := context.Background()
+ ip, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.NoError(t, err)
+ for _, id := range fake.IDs() {
+ require.NoError(t, fake.DeleteElasticIP(ctx, id))
+ }
+ require.NoError(t, a.Release(ctx, pool, ip, "eni-1"))
+}
+
+func TestRelease_DeleteFailureIsReported(t *testing.T) {
+ a, fake, _ := newTest(t, 0)
+ ctx := context.Background()
+ ip, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.NoError(t, err)
+ fake.Fail["delete"] = errors.New("api down")
+ err = a.Release(ctx, pool, ip, "eni-1")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "api down")
+}
+
+// A crash between create and record leaves a marked EIP with no binding; the
+// reconcile must collect it and must leave everything unmarked or foreign alone.
+func TestReconcile_CollectsOnlyOurLeaks(t *testing.T) {
+ a, fake, store := newTest(t, 0)
+ ctx := context.Background()
+
+ kept, err := a.Allocate(ctx, external.AllocateRequest{Purpose: "eni-public", ENIID: "eni-1"})
+ require.NoError(t, err)
+
+ leak, err := fake.CreateElasticIP(ctx, "spinifex:"+nodeID+":eni-crashed")
+ require.NoError(t, err)
+ require.NoError(t, fake.AttachElasticIP(ctx, nodeID, leak.ID))
+ fake.Seed(exoscale.ElasticIP{ID: "operator", Address: netip.MustParseAddr("89.145.162.53"), Description: "mulga1"})
+ fake.Seed(exoscale.ElasticIP{ID: "peer", Address: netip.MustParseAddr("89.145.162.54"), Description: "spinifex:" + otherID + ":eni-9"})
+
+ res, err := a.Reconcile(ctx)
+ require.NoError(t, err)
+ assert.Equal(t, []string{leak.ID}, res.Collected)
+ assert.Equal(t, 2, res.Skipped)
+ assert.NotContains(t, fake.IDs(), leak.ID)
+ assert.Contains(t, fake.IDs(), "operator")
+ assert.Contains(t, fake.IDs(), "peer")
+ _, ok := binding(t, store, kept)
+ assert.True(t, ok)
+}
+
+func TestReconcile_DropsOnlyThisNodesStaleBindings(t *testing.T) {
+ a, _, store := newTest(t, 0)
+ ctx := context.Background()
+ require.NoError(t, store.Mutate(ctx, pool, func(r *Record) (bool, error) {
+ r.Bindings["194.182.160.9"] = Binding{EIPID: "gone", NodeInstanceID: nodeID}
+ r.Bindings["194.182.160.10"] = Binding{EIPID: "gone-too", NodeInstanceID: otherID}
+ return true, nil
+ }))
+
+ res, err := a.Reconcile(ctx)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"194.182.160.9"}, res.Stale)
+ rec, err := store.Get(ctx, pool)
+ require.NoError(t, err)
+ assert.Contains(t, rec.Bindings, "194.182.160.10")
+ assert.NotContains(t, rec.Bindings, "194.182.160.9")
+}
+
+func TestReconcile_ListFailureChangesNothing(t *testing.T) {
+ a, fake, _ := newTest(t, 0)
+ fake.Fail["list"] = errors.New("api down")
+ _, err := a.Reconcile(context.Background())
+ require.Error(t, err)
+ assert.NotContains(t, fake.Calls, "delete")
+}
diff --git a/spinifex/network/external/exonet/reconcile.go b/spinifex/network/external/exonet/reconcile.go
new file mode 100644
index 000000000..de60df575
--- /dev/null
+++ b/spinifex/network/external/exonet/reconcile.go
@@ -0,0 +1,104 @@
+package exonet
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// ReconcileResult reports what a pass found.
+type ReconcileResult struct {
+ // Collected is the IDs of marked EIPs deleted as leaks.
+ Collected []string
+ // Stale is the binding keys dropped because Exoscale no longer has the EIP.
+ Stale []string
+ // Skipped counts EIPs without this node's marker, left alone.
+ Skipped int
+}
+
+// Reconcile squares the record against Exoscale in both directions:
+//
+// - A marked EIP with no binding leaked from an interrupted Allocate or
+// Release. It bills and holds one of the organisation's few EIPs, and
+// nothing else will ever find it. Delete it.
+// - A binding whose EIP Exoscale no longer has was deleted behind our back.
+// Drop it, so the address is not reported to a customer as theirs.
+//
+// It never touches an EIP without this node's marker. The organisation can hold
+// the operator's own addresses, and the marker is the whole safety property.
+// Run it before the allocator serves, since it cannot tell a leak from an
+// allocation that is between its create and its record write.
+func (a *PoolAllocator) Reconcile(ctx context.Context) (ReconcileResult, error) {
+ var res ReconcileResult
+
+ live, err := a.client.ListElasticIPs(ctx)
+ if err != nil {
+ return res, fmt.Errorf("exonet reconcile: list elastic IPs: %w", err)
+ }
+ rec, err := a.store.Get(ctx, a.cfg.Pool.Name)
+ if err != nil && !errors.Is(err, kvstore.ErrNotFound) {
+ return res, fmt.Errorf("exonet reconcile: read bindings: %w", err)
+ }
+
+ claimed := make(map[string]struct{}, len(rec.Bindings))
+ for _, b := range rec.Bindings {
+ claimed[b.EIPID] = struct{}{}
+ }
+
+ liveIDs := make(map[string]struct{}, len(live))
+ for _, e := range live {
+ liveIDs[e.ID] = struct{}{}
+ if !a.ours(e) {
+ res.Skipped++
+ continue
+ }
+ if _, ok := claimed[e.ID]; ok {
+ continue
+ }
+ if err := a.detachAndDelete(ctx, a.cfg.InstanceID, e.ID); err != nil {
+ return res, fmt.Errorf("exonet reconcile: collect leaked %s: %w", e.ID, err)
+ }
+ slog.WarnContext(ctx, "exonet reconcile collected a leaked elastic IP",
+ "pool", a.cfg.Pool.Name, "eip_id", e.ID, "public_ip", e.Address.String(), "description", e.Description)
+ res.Collected = append(res.Collected, e.ID)
+ }
+
+ // Only bindings attached to this node are this node's to judge.
+ var stale []string
+ for key, b := range rec.Bindings {
+ if _, ok := liveIDs[b.EIPID]; ok || b.NodeInstanceID != a.cfg.InstanceID {
+ continue
+ }
+ stale = append(stale, key)
+ }
+ if len(stale) == 0 {
+ return res, nil
+ }
+ err = a.store.Mutate(ctx, a.cfg.Pool.Name, func(r *Record) (bool, error) {
+ changed := false
+ for _, key := range stale {
+ b, ok := r.Bindings[key]
+ if !ok || b.NodeInstanceID != a.cfg.InstanceID {
+ continue
+ }
+ if _, still := liveIDs[b.EIPID]; still {
+ continue
+ }
+ delete(r.Bindings, key)
+ changed = true
+ }
+ return changed, nil
+ })
+ if err != nil {
+ return res, fmt.Errorf("exonet reconcile: drop stale bindings: %w", err)
+ }
+ for _, key := range stale {
+ slog.WarnContext(ctx, "exonet reconcile dropped a stale binding",
+ "pool", a.cfg.Pool.Name, "public_ip", key)
+ }
+ res.Stale = stale
+ return res, nil
+}
diff --git a/spinifex/network/external/exonet/store.go b/spinifex/network/external/exonet/store.go
new file mode 100644
index 000000000..b01509267
--- /dev/null
+++ b/spinifex/network/external/exonet/store.go
@@ -0,0 +1,66 @@
+package exonet
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/nats-io/nats.go/jetstream"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+const (
+ // KVBucketExoscaleBindings holds the address → Exoscale EIP mapping, in a
+ // bucket of its own for the same reason OCI's bindings have one.
+ KVBucketExoscaleBindings = "spinifex-exoscale-bindings"
+ KVBucketExoscaleBindingsHistory = 5
+ kvCASRetries = 10
+)
+
+func bucketConfig() kvstore.Config {
+ return kvstore.Config{
+ Name: KVBucketExoscaleBindings,
+ History: KVBucketExoscaleBindingsHistory,
+ Attempts: kvCASRetries,
+ Missing: "exonet: no JetStream client configured",
+ Exhausted: func(key string, attempts int) error {
+ return fmt.Errorf("exonet: pool %s contended after %d attempts", key, attempts)
+ },
+ }
+}
+
+// KVStore is the JetStream-backed Store.
+type KVStore struct {
+ store *kvstore.Store[Record]
+}
+
+var _ Store = (*KVStore)(nil)
+
+// NewKVStore creates the bindings bucket if it is missing.
+func NewKVStore(js jetstream.JetStream) *KVStore {
+ return &KVStore{store: kvstore.New[Record](js, bucketConfig())}
+}
+
+// Mutate implements Store. Upsert, because a pool's first allocation is what
+// creates its record, and that allocation has already made a billable EIP.
+func (s *KVStore) Mutate(ctx context.Context, poolName string, fn func(*Record) (bool, error)) error {
+ return s.store.Upsert(ctx, poolName, func(rec *Record) (bool, error) {
+ if rec.Bindings == nil {
+ rec.Bindings = map[string]Binding{}
+ }
+ return fn(rec)
+ })
+}
+
+// Get implements Store. A pool with no record reports kvstore.ErrNotFound,
+// which callers read as an empty binding set.
+func (s *KVStore) Get(ctx context.Context, poolName string) (Record, error) {
+ rec, _, err := s.store.Get(ctx, poolName)
+ if err != nil {
+ return Record{}, err
+ }
+ if rec == nil || rec.Bindings == nil {
+ return Record{Bindings: map[string]Binding{}}, nil
+ }
+ return *rec, nil
+}
diff --git a/spinifex/network/external/exonet/store_test.go b/spinifex/network/external/exonet/store_test.go
new file mode 100644
index 000000000..a005bcc34
--- /dev/null
+++ b/spinifex/network/external/exonet/store_test.go
@@ -0,0 +1,63 @@
+package exonet
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/mulgadc/spinifex/internal/testkit"
+ "github.com/mulgadc/spinifex/spinifex/cloud/exoscale"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/network/external"
+)
+
+func TestKVStore_UnwrittenPoolIsNotFound(t *testing.T) {
+ _, _, js := testutil.StartTestJetStream(t)
+ _, err := NewKVStore(js).Get(context.Background(), pool)
+ assert.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A daemon restart builds a new allocator over the same bucket. Its reconcile
+// must find the live binding and leave the EIP alone, not collect it as a leak.
+func TestKVStore_BindingSurvivesARestartReconcile(t *testing.T) {
+ _, _, js := testutil.StartTestJetStream(t)
+ ctx := context.Background()
+ fake := exoscale.NewFake("de-fra-1", 0)
+ cfg := Config{Pool: external.ExternalPoolConfig{Name: pool, Source: external.SourceExoscale}, InstanceID: nodeID}
+
+ first, err := New(fake, NewKVStore(js), cfg)
+ require.NoError(t, err)
+ ip, err := first.Allocate(ctx, external.AllocateRequest{PoolName: pool, Purpose: "eni-public", ENIID: "eni-1"})
+ require.NoError(t, err)
+
+ second, err := New(fake, NewKVStore(js), cfg)
+ require.NoError(t, err)
+ res, err := second.Reconcile(ctx)
+ require.NoError(t, err)
+ assert.Empty(t, res.Collected)
+ assert.Empty(t, res.Stale)
+ assert.Len(t, fake.IDs(), 1)
+
+ require.NoError(t, second.Release(ctx, pool, ip, "eni-1"))
+ assert.Empty(t, fake.IDs())
+ rec, err := NewKVStore(js).Get(ctx, pool)
+ require.NoError(t, err)
+ assert.NotContains(t, rec.Bindings, ip.String())
+}
+
+func TestFromPoolConfig_RefusesAnUnusableBinary(t *testing.T) {
+ _, _, js := testutil.StartTestJetStream(t)
+ _, err := FromPoolConfig(context.Background(), js, external.ExternalPoolConfig{
+ Name: pool, Source: external.SourceExoscale, ExoscaleZone: "de-fra-1",
+ ExoscaleInstanceID: nodeID, ExoscaleBinary: "/bin/false",
+ })
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "exo CLI is not usable")
+}
+
+func TestFromPoolConfig_RefusesAnotherSource(t *testing.T) {
+ _, err := FromPoolConfig(context.Background(), nil, external.ExternalPoolConfig{Name: pool, Source: "static"})
+ require.Error(t, err)
+}
diff --git a/spinifex/network/external/exonet/wiring.go b/spinifex/network/external/exonet/wiring.go
new file mode 100644
index 000000000..0e39e241a
--- /dev/null
+++ b/spinifex/network/external/exonet/wiring.go
@@ -0,0 +1,37 @@
+package exonet
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/nats-io/nats.go/jetstream"
+
+ "github.com/mulgadc/spinifex/spinifex/cloud/exoscale"
+ "github.com/mulgadc/spinifex/spinifex/network/external"
+)
+
+// FromPoolConfig builds a live allocator for one source="exoscale" pool and
+// proves the exo binary runs before anything depends on it. It does not
+// reconcile; the caller runs that pass before the allocator serves.
+func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig) (*PoolAllocator, error) {
+ if !pool.IsExoscale() {
+ return nil, fmt.Errorf("exonet: pool %q has source %q, not %q", pool.Name, pool.Source, external.SourceExoscale)
+ }
+ client, err := exoscale.NewCLIClient(exoscale.CLIConfig{
+ Binary: pool.ExoscaleBinary,
+ ConfigFile: pool.ExoscaleConfigFile,
+ Account: pool.ExoscaleAccount,
+ Zone: pool.ExoscaleZone,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("exonet: pool %q: %w", pool.Name, err)
+ }
+ version, err := client.Version(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("exonet: pool %q: exo CLI is not usable: %w", pool.Name, err)
+ }
+ slog.InfoContext(ctx, "exonet using exo CLI", "pool", pool.Name, "version", version,
+ "zone", pool.ExoscaleZone, "instance_id", pool.ExoscaleInstanceID)
+ return New(client, NewKVStore(js), Config{Pool: pool, InstanceID: pool.ExoscaleInstanceID})
+}
diff --git a/spinifex/network/external/types.go b/spinifex/network/external/types.go
index 797b4aa06..edf86b06a 100644
--- a/spinifex/network/external/types.go
+++ b/spinifex/network/external/types.go
@@ -37,6 +37,13 @@ type ExternalPoolConfig struct {
// principal: no dynamic group, no IAM policy, and no metadata read.
OCIConfigFile string
OCIConfigProfile string
+ // Exoscale* configure a source="exoscale" pool: the zone the EIPs live in,
+ // the instance they attach to, and how the exo CLI is invoked.
+ ExoscaleZone string
+ ExoscaleInstanceID string
+ ExoscaleConfigFile string
+ ExoscaleAccount string
+ ExoscaleBinary string
}
// IsDHCP reports whether the pool sources IPs from an upstream DHCP server.
@@ -49,6 +56,11 @@ func (p *ExternalPoolConfig) IsOCI() bool {
return p != nil && p.Source == SourceOCI
}
+// IsExoscale reports whether the pool sources IPs from Exoscale Elastic IPs.
+func (p *ExternalPoolConfig) IsExoscale() bool {
+ return p != nil && p.Source == SourceExoscale
+}
+
const (
// SourceStatic is the default pool source (inline range math, KV-backed).
SourceStatic = "static"
@@ -59,6 +71,10 @@ const (
// it, so the addresses have to be created through the provider rather than
// computed from a range.
SourceOCI = "oci"
+ // SourceExoscale delegates allocation to Exoscale Elastic IPs, created and
+ // attached through the exo CLI. A manual EIP arrives addressed to itself,
+ // so unlike OCI there is no private half to map.
+ SourceExoscale = "exoscale"
// DHCPMACDerived leases with deterministic per-client-id 02:xx MACs.
DHCPMACDerived = "derived"
diff --git a/spinifex/network/host/ipsec.go b/spinifex/network/host/ipsec.go
index d13f02cab..4a61a254f 100644
--- a/spinifex/network/host/ipsec.go
+++ b/spinifex/network/host/ipsec.go
@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"fmt"
+ "slices"
"strings"
"github.com/mulgadc/spinifex/spinifex/utils"
@@ -88,12 +89,9 @@ func GetNBGlobalIPSec(nbAddr string) (bool, error) {
func isNBUnreachable(msg string) bool {
lower := strings.ToLower(msg)
- for _, pattern := range nbUnreachablePatterns {
- if strings.Contains(lower, pattern) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(nbUnreachablePatterns, func(pattern string) bool {
+ return strings.Contains(lower, pattern)
+ })
}
// SetNBGlobalIPSec writes NB_Global.ipsec, triggering ovn-controller to add
diff --git a/spinifex/network/ovn/live.go b/spinifex/network/ovn/live.go
index 3e815aaf7..177c09f14 100644
--- a/spinifex/network/ovn/live.go
+++ b/spinifex/network/ovn/live.go
@@ -349,7 +349,7 @@ func (c *LiveClient) CreateLogicalSwitchPortInGroups(ctx context.Context, switch
ops := append(createOps, switchMutateOps...)
for _, pgName := range portGroupNames {
- pg, err := c.getPortGroup(ctx, pgName)
+ pg, err := c.GetPortGroup(ctx, pgName)
if err != nil {
return fmt.Errorf("get port group %s for port add: %w", pgName, err)
}
@@ -1370,7 +1370,7 @@ func (c *LiveClient) CreatePortGroup(ctx context.Context, name string, ports []s
}
func (c *LiveClient) EnsurePortGroup(ctx context.Context, name string, ports []string) (*nbdb.PortGroup, bool, error) {
- if existing, err := c.getPortGroup(ctx, name); err == nil {
+ if existing, err := c.GetPortGroup(ctx, name); err == nil {
return existing, false, nil
}
pg := &nbdb.PortGroup{
@@ -1404,7 +1404,7 @@ func (c *LiveClient) waitForCachedPortGroup(ctx context.Context, name string) (*
deadline := time.Now().Add(ensureRefetchTimeout)
var lastErr error
for {
- v, err := c.getPortGroup(ctx, name)
+ v, err := c.GetPortGroup(ctx, name)
if err == nil {
return v, nil
}
@@ -1421,7 +1421,7 @@ func (c *LiveClient) waitForCachedPortGroup(ctx context.Context, name string) (*
}
func (c *LiveClient) DeletePortGroup(ctx context.Context, name string) error {
- pg, err := c.getPortGroup(ctx, name)
+ pg, err := c.GetPortGroup(ctx, name)
if err != nil {
return fmt.Errorf("delete port group lookup: %w", err)
}
@@ -1446,7 +1446,7 @@ func (c *LiveClient) UpdatePortGroupMemberships(ctx context.Context, lspName str
var ops []ovsdb.Operation
appendPGOps := func(pgName, mutator string) error {
- pg, err := c.getPortGroup(ctx, pgName)
+ pg, err := c.GetPortGroup(ctx, pgName)
if err != nil {
return fmt.Errorf("port group %s lookup: %w", pgName, err)
}
@@ -1479,7 +1479,8 @@ func (c *LiveClient) UpdatePortGroupMemberships(ctx context.Context, lspName str
return nil
}
-func (c *LiveClient) getPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) {
+// GetPortGroup returns the named port group.
+func (c *LiveClient) GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) {
var pgs []nbdb.PortGroup
err := c.client.WhereCache(func(pg *nbdb.PortGroup) bool {
return pg.Name == name
@@ -1493,11 +1494,6 @@ func (c *LiveClient) getPortGroup(ctx context.Context, name string) (*nbdb.PortG
return &pgs[0], nil
}
-// GetPortGroup returns the named port group.
-func (c *LiveClient) GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) {
- return c.getPortGroup(ctx, name)
-}
-
func (c *LiveClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) {
var pgs []nbdb.PortGroup
if err := c.client.List(ctx, &pgs); err != nil {
@@ -1510,7 +1506,7 @@ func (c *LiveClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, erro
// addresses on the existing row. Wait-op serialises concurrent writers (NB has
// no unique-Name constraint). Returns the persisted row UUID.
func (c *LiveClient) EnsureAddressSet(ctx context.Context, name string, addresses []string) (string, error) {
- if existing, err := c.getAddressSet(ctx, name); err == nil {
+ if existing, err := c.GetAddressSet(ctx, name); err == nil {
return existing.UUID, c.convergeAddressSet(ctx, existing, addresses)
}
as := &nbdb.AddressSet{
@@ -1562,7 +1558,7 @@ func (c *LiveClient) waitForCachedAddressSet(ctx context.Context, name string) (
deadline := time.Now().Add(ensureRefetchTimeout)
var lastErr error
for {
- v, err := c.getAddressSet(ctx, name)
+ v, err := c.GetAddressSet(ctx, name)
if err == nil {
return v, nil
}
@@ -1580,10 +1576,6 @@ func (c *LiveClient) waitForCachedAddressSet(ctx context.Context, name string) (
// GetAddressSet returns the named address set.
func (c *LiveClient) GetAddressSet(ctx context.Context, name string) (*nbdb.AddressSet, error) {
- return c.getAddressSet(ctx, name)
-}
-
-func (c *LiveClient) getAddressSet(ctx context.Context, name string) (*nbdb.AddressSet, error) {
var sets []nbdb.AddressSet
err := c.client.WhereCache(func(as *nbdb.AddressSet) bool {
return as.Name == name
@@ -1622,7 +1614,7 @@ func (c *LiveClient) AddACLs(ctx context.Context, portGroupName string, specs []
if len(specs) == 0 {
return nil
}
- pg, err := c.getPortGroup(ctx, portGroupName)
+ pg, err := c.GetPortGroup(ctx, portGroupName)
if err != nil {
return fmt.Errorf("add ACLs port group lookup: %w", err)
}
@@ -1675,7 +1667,7 @@ func (c *LiveClient) AddACLs(ctx context.Context, portGroupName string, specs []
// ClearACLs removes every ACL row referenced by the port group and detaches
// them from the port group's ACLs set in one transaction.
func (c *LiveClient) ClearACLs(ctx context.Context, portGroupName string) error {
- pg, err := c.getPortGroup(ctx, portGroupName)
+ pg, err := c.GetPortGroup(ctx, portGroupName)
if err != nil {
return fmt.Errorf("clear ACLs port group lookup: %w", err)
}
@@ -1730,7 +1722,7 @@ func (c *LiveClient) aclSetUnchanged(ctx context.Context, pg *nbdb.PortGroup, sp
// No-ops when the current ACL set already matches specs, so an unchanged SG does
// not churn NB rows (and force ovn-northd flow recompute) on every drift tick.
func (c *LiveClient) ReplaceACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error {
- pg, err := c.getPortGroup(ctx, portGroupName)
+ pg, err := c.GetPortGroup(ctx, portGroupName)
if err != nil {
return fmt.Errorf("replace ACLs port group lookup: %w", err)
}
diff --git a/spinifex/network/reconcile/intent.go b/spinifex/network/reconcile/intent.go
index d00c2b93d..99fa6cf35 100644
--- a/spinifex/network/reconcile/intent.go
+++ b/spinifex/network/reconcile/intent.go
@@ -8,6 +8,7 @@ import (
"log/slog"
"net"
"net/netip"
+ "slices"
"strings"
handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
@@ -657,14 +658,9 @@ func loadNATGWs(
if rt.VpcId != rec.VpcId {
continue
}
- hasNatRoute := false
- for _, r := range rt.Routes {
- if r.NatGatewayId == rec.NatGatewayId {
- hasNatRoute = true
- break
- }
- }
- if !hasNatRoute {
+ if !slices.ContainsFunc(rt.Routes, func(r handlers_ec2_routetable.RouteRecord) bool {
+ return r.NatGatewayId == rec.NatGatewayId
+ }) {
continue
}
for _, assoc := range rt.Associations {
diff --git a/spinifex/operator/host/systemd/reconcile.go b/spinifex/operator/host/systemd/reconcile.go
index db2cfcf0d..192a8e4a5 100644
--- a/spinifex/operator/host/systemd/reconcile.go
+++ b/spinifex/operator/host/systemd/reconcile.go
@@ -9,11 +9,12 @@ package systemd
import (
"errors"
"fmt"
+ "maps"
"os"
"os/exec"
"path/filepath"
"regexp"
- "sort"
+ "slices"
"strconv"
"strings"
"time"
@@ -54,23 +55,17 @@ type Result struct {
// HasChanges reports whether any unit is missing or stale — i.e. whether
// applying (DryRun: false) would write anything to disk.
func (r Result) HasChanges() bool {
- for _, s := range r.Statuses {
- if s.Action == ActionInstall || s.Action == ActionReplace {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(r.Statuses, func(s UnitStatus) bool {
+ return s.Action == ActionInstall || s.Action == ActionReplace
+ })
}
// HasConflicts reports whether any unit is at the current version but was
// modified on disk — drift Reconcile deliberately refuses to overwrite.
func (r Result) HasConflicts() bool {
- for _, s := range r.Statuses {
- if s.Action == ActionConflict {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(r.Statuses, func(s UnitStatus) bool {
+ return s.Action == ActionConflict
+ })
}
// ErrRootRequired is returned when Reconcile has pending changes but root
@@ -103,12 +98,7 @@ var systemctlDaemonReload = func() error {
}
func sortedUnitNames() []string {
- names := make([]string, 0, len(Units))
- for n := range Units {
- names = append(names, n)
- }
- sort.Strings(names)
- return names
+ return slices.Sorted(maps.Keys(Units))
}
// Reconcile compares each embedded unit (Units, generated from
diff --git a/spinifex/paging/ec2.go b/spinifex/paging/ec2.go
new file mode 100644
index 000000000..b6fd0b01b
--- /dev/null
+++ b/spinifex/paging/ec2.go
@@ -0,0 +1,64 @@
+package paging
+
+import (
+ "errors"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// ec2MinResults is the smallest MaxResults the network Describe calls accept.
+const ec2MinResults = 5
+
+// EC2 is one EC2 Describe operation's paging contract. The messages are AWS's
+// own, which differ in wording and casing between operations.
+type EC2 struct {
+ // MaxResults is the largest MaxResults the operation accepts.
+ MaxResults int64
+ // TooLarge and TooSmall answer a MaxResults outside the accepted range;
+ // %d is the requested value.
+ TooLarge string
+ TooSmall string
+ // WithIDs answers MaxResults sent together with explicit resource IDs.
+ WithIDs string
+ // PaginationTokenError selects the InvalidPaginationToken answer to a bad
+ // token that some operations give instead of InvalidParameterValue.
+ PaginationTokenError bool
+}
+
+// Parse validates MaxResults, its combination with ids explicit resource IDs,
+// and NextToken, and returns the page they request.
+func (p EC2) Parse(maxResults *int64, nextToken *string, ids int) (Request, error) {
+ if maxResults != nil {
+ n := *maxResults
+ if n > p.MaxResults {
+ return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, p.TooLarge, n)
+ }
+ if n < ec2MinResults {
+ return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, p.TooSmall, n)
+ }
+ if ids > 0 {
+ return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterCombination, "%s", p.WithIDs)
+ }
+ }
+
+ token := aws.StringValue(nextToken)
+ from, err := DecodeToken(token)
+ if errors.Is(err, ErrInvalidToken) {
+ if p.PaginationTokenError {
+ return Request{}, awserrors.Errorf(awserrors.ErrorInvalidPaginationToken, "Next token '%s' is invalid", token)
+ }
+ return Request{}, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+ "Value ( %s ) for parameter NextToken is invalid. The token is invalid.", token)
+ }
+ return Request{Limit: int(aws.Int64Value(maxResults)), From: from}, nil
+}
+
+// EC2Page is Page with the next token in the SDK's form: nil on the last page.
+func EC2Page[T any](items []T, key func(T) string, req Request) ([]T, *string) {
+ page, next := Page(items, key, req)
+ if next == "" {
+ return page, nil
+ }
+ return page, aws.String(next)
+}
diff --git a/spinifex/paging/paging.go b/spinifex/paging/paging.go
new file mode 100644
index 000000000..5b451da37
--- /dev/null
+++ b/spinifex/paging/paging.go
@@ -0,0 +1,64 @@
+// Package paging pages a listing with opaque resume tokens. Each API keeps its
+// own limits and error text; this package only owns the token and the slicing.
+package paging
+
+import (
+ "encoding/base64"
+ "errors"
+ "slices"
+ "strings"
+)
+
+// ErrInvalidToken reports a token this package did not issue.
+var ErrInvalidToken = errors.New("paging: token was not issued by this service")
+
+// Marks a token as ours, so arbitrary base64 is rejected rather than resumed.
+const tokenPrefix = "spx-page-v1:" //nolint:gosec // a format marker, not a credential
+
+// Request is a validated page request. The zero value asks for everything.
+type Request struct {
+ // Limit caps the page; zero or less means no cap.
+ Limit int
+ // From is the key to resume at, decoded from the caller's token.
+ From string
+}
+
+// EncodeToken returns the opaque token that resumes a listing at key.
+func EncodeToken(key string) string {
+ return base64.RawURLEncoding.EncodeToString([]byte(tokenPrefix + key))
+}
+
+// DecodeToken returns the key a token resumes at. An empty token is the start.
+func DecodeToken(token string) (string, error) {
+ if token == "" {
+ return "", nil
+ }
+ decoded, err := base64.RawURLEncoding.DecodeString(token)
+ if err != nil {
+ return "", ErrInvalidToken
+ }
+ key, ok := strings.CutPrefix(string(decoded), tokenPrefix)
+ if !ok || key == "" {
+ return "", ErrInvalidToken
+ }
+ return key, nil
+}
+
+// Page sorts items by key in place and returns the page req selects, plus the
+// token for the next page, or "" on the last page. A resume starts at the first
+// key not below From, so deleting the item a token names skips nothing.
+func Page[T any](items []T, key func(T) string, req Request) ([]T, string) {
+ slices.SortStableFunc(items, func(a, b T) int { return strings.Compare(key(a), key(b)) })
+
+ start := 0
+ if req.From != "" {
+ start, _ = slices.BinarySearchFunc(items, req.From, func(item T, target string) int {
+ return strings.Compare(key(item), target)
+ })
+ }
+ rest := items[start:]
+ if req.Limit <= 0 || len(rest) <= req.Limit {
+ return rest, ""
+ }
+ return rest[:req.Limit], EncodeToken(key(rest[req.Limit]))
+}
diff --git a/spinifex/paging/paging_test.go b/spinifex/paging/paging_test.go
new file mode 100644
index 000000000..82abce477
--- /dev/null
+++ b/spinifex/paging/paging_test.go
@@ -0,0 +1,111 @@
+package paging_test
+
+import (
+ "encoding/base64"
+ "fmt"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+ "github.com/mulgadc/spinifex/spinifex/paging"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func identity(s string) string { return s }
+
+func TestPage_WalksEveryItemOnceInKeyOrder(t *testing.T) {
+ items := []string{"d", "a", "e", "c", "b"}
+ var seen []string
+ req := paging.Request{Limit: 2}
+ for range len(items) {
+ page, next := paging.Page(items, identity, req)
+ seen = append(seen, page...)
+ if next == "" {
+ break
+ }
+ from, err := paging.DecodeToken(next)
+ require.NoError(t, err)
+ req.From = from
+ }
+ assert.Equal(t, []string{"a", "b", "c", "d", "e"}, seen)
+}
+
+func TestPage_ResumesPastADeletedItem(t *testing.T) {
+ _, next := paging.Page([]string{"a", "b", "c", "d"}, identity, paging.Request{Limit: 2})
+ from, err := paging.DecodeToken(next)
+ require.NoError(t, err)
+ require.Equal(t, "c", from)
+
+ page, _ := paging.Page([]string{"a", "b", "d"}, identity, paging.Request{Limit: 2, From: from})
+ assert.Equal(t, []string{"d"}, page)
+}
+
+func TestPage_NoLimitReturnsEverythingWithoutToken(t *testing.T) {
+ page, next := paging.Page([]string{"b", "a"}, identity, paging.Request{})
+ assert.Equal(t, []string{"a", "b"}, page)
+ assert.Empty(t, next)
+
+ page, next = paging.Page([]string{"a", "b"}, identity, paging.Request{Limit: 2})
+ assert.Len(t, page, 2)
+ assert.Empty(t, next, "an exactly full last page carries no token")
+}
+
+func TestDecodeToken_RejectsTokensItDidNotIssue(t *testing.T) {
+ for _, token := range []string{
+ "garbage!",
+ base64.RawURLEncoding.EncodeToString([]byte("subnet-123")),
+ paging.EncodeToken(""),
+ } {
+ _, err := paging.DecodeToken(token)
+ assert.ErrorIs(t, err, paging.ErrInvalidToken, token)
+ }
+}
+
+func requireAWSError(t *testing.T, err error, code, message string) {
+ t.Helper()
+ got, msg, ok := awserrors.ResolveErrorDetail(err)
+ require.True(t, ok, "error %v carries no AWS code", err)
+ assert.Equal(t, code, got)
+ assert.Equal(t, message, msg)
+}
+
+func TestEC2Parse(t *testing.T) {
+ p := paging.EC2{
+ MaxResults: 1000,
+ TooLarge: "too large %d",
+ TooSmall: "too small %d",
+ WithIDs: "ids and max",
+ }
+
+ req, err := p.Parse(aws.Int64(1000), aws.String(paging.EncodeToken("subnet-b")), 0)
+ require.NoError(t, err)
+ assert.Equal(t, paging.Request{Limit: 1000, From: "subnet-b"}, req)
+
+ req, err = p.Parse(nil, nil, 2)
+ require.NoError(t, err, "IDs without MaxResults are fine")
+ assert.Equal(t, paging.Request{}, req)
+
+ _, err = p.Parse(aws.Int64(5), nil, 0)
+ require.NoError(t, err)
+
+ _, err = p.Parse(aws.Int64(1001), nil, 0)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "too large 1001")
+ for _, n := range []int64{4, 0, -1} {
+ _, err = p.Parse(aws.Int64(n), nil, 0)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, fmt.Sprintf("too small %d", n))
+ }
+
+ _, err = p.Parse(aws.Int64(2000), nil, 1)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "too large 2000")
+ _, err = p.Parse(aws.Int64(5), nil, 1)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterCombination, "ids and max")
+
+ _, err = p.Parse(nil, aws.String("garbage"), 0)
+ requireAWSError(t, err, awserrors.ErrorInvalidParameterValue,
+ "Value ( garbage ) for parameter NextToken is invalid. The token is invalid.")
+
+ p.PaginationTokenError = true
+ _, err = p.Parse(nil, aws.String("garbage"), 0)
+ requireAWSError(t, err, awserrors.ErrorInvalidPaginationToken, "Next token 'garbage' is invalid")
+}
diff --git a/spinifex/providers/cloud/oci/apiclient_test.go b/spinifex/providers/cloud/oci/apiclient_test.go
index b5e9e1981..2be86031f 100644
--- a/spinifex/providers/cloud/oci/apiclient_test.go
+++ b/spinifex/providers/cloud/oci/apiclient_test.go
@@ -70,10 +70,10 @@ func TestListPrivateIPsFollowsEveryPage(t *testing.T) {
c, seen := testClient(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "":
- w.Header().Set("opc-next-page", "page2")
+ w.Header().Set("Opc-Next-Page", "page2")
_, _ = w.Write([]byte(`[{"id":"p1","ipAddress":"10.200.0.100","vnicId":"v1"}]`))
case "page2":
- w.Header().Set("opc-next-page", "page3")
+ w.Header().Set("Opc-Next-Page", "page3")
_, _ = w.Write([]byte(`[{"id":"p2","ipAddress":"10.200.0.101","vnicId":"v1"}]`))
default:
_, _ = w.Write([]byte(`[{"id":"p3","ipAddress":"10.200.0.102","vnicId":"v1"}]`))
diff --git a/spinifex/providers/cloud/oci/fake.go b/spinifex/providers/cloud/oci/fake.go
index ddd790120..efefc15b9 100644
--- a/spinifex/providers/cloud/oci/fake.go
+++ b/spinifex/providers/cloud/oci/fake.go
@@ -3,7 +3,9 @@ package oci
import (
"context"
"fmt"
+ "maps"
"net/netip"
+ "slices"
"sync"
)
@@ -71,22 +73,14 @@ func (f *Fake) SeedPublicIP(p PublicIP) {
func (f *Fake) PrivateIPs() []PrivateIP {
f.mu.Lock()
defer f.mu.Unlock()
- out := make([]PrivateIP, 0, len(f.privateIPs))
- for _, p := range f.privateIPs {
- out = append(out, p)
- }
- return out
+ return slices.Collect(maps.Values(f.privateIPs))
}
// PublicIPs returns a snapshot.
func (f *Fake) PublicIPs() []PublicIP {
f.mu.Lock()
defer f.mu.Unlock()
- out := make([]PublicIP, 0, len(f.publicIPs))
- for _, p := range f.publicIPs {
- out = append(out, p)
- }
- return out
+ return slices.Collect(maps.Values(f.publicIPs))
}
// FailOp stages err for the next call to op. Locked, so a test may stage one
diff --git a/spinifex/providers/ebs/conformance/bench.go b/spinifex/providers/ebs/conformance/bench.go
index 39a37826a..eb11e582b 100644
--- a/spinifex/providers/ebs/conformance/bench.go
+++ b/spinifex/providers/ebs/conformance/bench.go
@@ -181,5 +181,8 @@ func benchCapabilities(b *testing.B, provider ebsprovider.EBSProvider) ebsprovid
if err != nil {
b.Fatalf("get capabilities: %v", err)
}
+ if resp == nil {
+ b.Fatal("get capabilities: provider returned a nil response with a nil error")
+ }
return resp.Capabilities
}
diff --git a/spinifex/providers/ebs/conformance/nbdclient.go b/spinifex/providers/ebs/conformance/nbdclient.go
index 0dee84eb0..92e371066 100644
--- a/spinifex/providers/ebs/conformance/nbdclient.go
+++ b/spinifex/providers/ebs/conformance/nbdclient.go
@@ -11,6 +11,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "slices"
"strings"
"testing"
"time"
@@ -302,13 +303,9 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
capabilities := capabilitiesOf(t, provider)
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false)
- holes := false
- for _, extent := range nbdMap(t, dialURI(t, pub.NBDURI)) {
- if strings.Contains(extent.Description, "hole") {
- holes = true
- break
- }
- }
+ holes := slices.ContainsFunc(nbdMap(t, dialURI(t, pub.NBDURI)), func(extent nbdExtent) bool {
+ return strings.Contains(extent.Description, "hole")
+ })
assert.Equalf(t, capabilities.SparseExtentReporting, holes,
"SparseExtentReporting=%v but a volume with nothing written to it reports holes=%v",
capabilities.SparseExtentReporting, holes)
diff --git a/spinifex/providers/ebs/natsserve/handlers_test.go b/spinifex/providers/ebs/natsserve/handlers_test.go
index 58b9305eb..c2eb63fc0 100644
--- a/spinifex/providers/ebs/natsserve/handlers_test.go
+++ b/spinifex/providers/ebs/natsserve/handlers_test.go
@@ -432,11 +432,43 @@ func (p *countingProvider) PublishVolume(ctx context.Context, req ebsprovider.Pu
return p.EBSProvider.PublishVolume(ctx, req)
}
+// handlerSettle is how long a handler count is watched after it reaches the
+// expected value, to catch a delivery that should not have happened arriving
+// late. Generous, because a false pass here is a silently broken queue group.
+const handlerSettle = 250 * time.Millisecond
+
+// requireCapabilitiesHandlers waits for the capabilities handlers to have run
+// the expected number of times, then confirms the count does not move.
+//
+// Both halves are needed. How many handlers a message reaches is the server's
+// decision — a queue group picks one member, a plain subscription reaches
+// every member — so the count converges on the answer from below and cannot
+// overshoot it. But a broken queue group that delivered to both members would
+// pass through the correct value on its way to the wrong one, so converging
+// alone would accept it.
+func requireCapabilitiesHandlers(t *testing.T, workers []*countingProvider, want int32) {
+ t.Helper()
+ total := func() int32 {
+ var sum int32
+ for _, worker := range workers {
+ sum += worker.capabilities.Load()
+ }
+ return sum
+ }
+ require.Eventuallyf(t, func() bool { return total() == want }, requestTimeout, 5*time.Millisecond,
+ "capabilities never reached %d handlers", want)
+ time.Sleep(handlerSettle)
+ require.Equal(t, want, total(), "capabilities handler count moved after it had settled")
+}
+
// TestServeQueueGroupBalancesSharedSubjects is the difference between two
-// workers sharing a workload and both doing all of it. The mount subject is
-// never queue-grouped, so once both servers have handled a mount, message
-// ordering on the shared connection guarantees the earlier capabilities
-// request has already been delivered to everyone that was going to get it.
+// workers sharing a workload and both doing all of it.
+//
+// The handler counts are waited for rather than read once. Each subscription
+// dispatches on its own goroutine, so there is no ordering between a worker's
+// mount handler and its capabilities handler: the mount having landed on both
+// workers says nothing about whether the earlier capabilities message has been
+// counted yet.
func TestServeQueueGroupBalancesSharedSubjects(t *testing.T) {
for _, tc := range []struct {
name string
@@ -470,7 +502,7 @@ func TestServeQueueGroupBalancesSharedSubjects(t *testing.T) {
return workers[0].publishes.Load()+workers[1].publishes.Load() == 2
}, requestTimeout, 5*time.Millisecond, "mount is never queue-grouped, so both servers must see it")
- assert.Equal(t, tc.wantHandlers, workers[0].capabilities.Load()+workers[1].capabilities.Load())
+ requireCapabilitiesHandlers(t, workers, tc.wantHandlers)
})
}
}
diff --git a/spinifex/runtime/compute/cache/cache.go b/spinifex/runtime/compute/cache/cache.go
index 08288a052..fb00e2a38 100644
--- a/spinifex/runtime/compute/cache/cache.go
+++ b/spinifex/runtime/compute/cache/cache.go
@@ -260,6 +260,9 @@ func (c *Cache) setActive(lw *liveWatcher) {
c.watcherMu.Unlock()
}
+// markResynced is called with c.mu held, so the mark is published by the same
+// unlock that publishes readiness. A reader that takes the lock and sees a
+// ready cache cannot then be told the cache has never resynced.
func (c *Cache) markResynced() {
c.lastResync.Store(time.Now().UnixNano())
c.degraded.Store(false)
@@ -455,6 +458,7 @@ func (c *Cache) installSync(lw *liveWatcher, entries map[string]*vm.VM, index ma
c.mu.Lock()
c.entries, c.index = entries, index
c.ready = true
+ c.markResynced()
c.mu.Unlock()
c.setActive(lw)
go c.drain(lw)
@@ -481,7 +485,6 @@ func (c *Cache) resyncUntilSuccess(ctx context.Context, oldLw *liveWatcher) *liv
continue
}
c.installSync(lw, entries, index, oldLw)
- c.markResynced()
return lw
}
}
@@ -509,7 +512,6 @@ func (c *Cache) periodicResync(ctx context.Context) {
return
}
c.installSync(lw, entries, index, oldLw)
- c.markResynced()
}
// replaceWatcher retires a dead watcher and installs a fresh one under the
diff --git a/spinifex/runtime/compute/cache/cache_test.go b/spinifex/runtime/compute/cache/cache_test.go
index 31f8d93b0..d07dc337b 100644
--- a/spinifex/runtime/compute/cache/cache_test.go
+++ b/spinifex/runtime/compute/cache/cache_test.go
@@ -45,7 +45,7 @@ func newTestCache(t *testing.T, mutate ...func(*Config)) (*Cache, jetstream.KeyV
require.NoError(t, err)
cfg := Config{
- Bucket: kvstore.Config{Name: bucket, History: 1, Replicas: 1},
+ Bucket: kvstore.Config{Name: bucket, History: 1},
Prefix: testPrefix,
VisibleToCaller: instance.IsInstanceVisibleToCaller,
FallbackAccountID: utils.GlobalAccountID,
@@ -256,7 +256,7 @@ func TestPeriodicResync_TTLExpiredRecordRemovedWithNoWatchEvent(t *testing.T) {
require.NoError(t, err)
c := New(js, Config{
- Bucket: kvstore.Config{Name: bucket, History: 1, Replicas: 1},
+ Bucket: kvstore.Config{Name: bucket, History: 1},
Prefix: testPrefix,
VisibleToCaller: instance.IsInstanceVisibleToCaller,
FallbackAccountID: utils.GlobalAccountID,
diff --git a/spinifex/runtime/compute/cache/liveness_test.go b/spinifex/runtime/compute/cache/liveness_test.go
index 7491885ac..0c1e5c450 100644
--- a/spinifex/runtime/compute/cache/liveness_test.go
+++ b/spinifex/runtime/compute/cache/liveness_test.go
@@ -23,7 +23,7 @@ func newTestLiveness(t *testing.T) (*instancecache.Liveness, jetstream.KeyValue)
kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 1})
require.NoError(t, err)
- l := instancecache.NewLiveness(js, kvstore.Config{Name: bucket, History: 1, Replicas: 1})
+ l := instancecache.NewLiveness(js, kvstore.Config{Name: bucket, History: 1})
return l, kv
}
diff --git a/spinifex/runtime/compute/cache/resume_from_revision_test.go b/spinifex/runtime/compute/cache/resume_from_revision_test.go
index 5bf2bad41..89073a9a4 100644
--- a/spinifex/runtime/compute/cache/resume_from_revision_test.go
+++ b/spinifex/runtime/compute/cache/resume_from_revision_test.go
@@ -25,7 +25,7 @@ func TestResumeFromRevision_ReplaysWhatTheSnapshotMissed(t *testing.T) {
_, _, js := testutil.StartTestJetStream(t)
ctx := context.Background()
- store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{Name: "resume-bucket", History: 1, Replicas: 1})
+ store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{Name: "resume-bucket", History: 1})
rec := func(id string) *vm.InstanceRecord {
r := &vm.InstanceRecord{}
diff --git a/spinifex/runtime/compute/vm/eni_hotplug.go b/spinifex/runtime/compute/vm/eni_hotplug.go
index 790921e51..f1441ba4d 100644
--- a/spinifex/runtime/compute/vm/eni_hotplug.go
+++ b/spinifex/runtime/compute/vm/eni_hotplug.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
+ "slices"
"time"
)
@@ -233,13 +234,9 @@ func waitForPCIDevice(dc DeviceController, deviceID string, wantPresent bool) er
if err != nil {
return fmt.Errorf("query-pci attempt %d: %w", attempt, err)
}
- found := false
- for _, d := range devs {
- if d.QDevID == deviceID {
- found = true
- break
- }
- }
+ found := slices.ContainsFunc(devs, func(d PCIDevice) bool {
+ return d.QDevID == deviceID
+ })
if found == wantPresent {
return nil
}
diff --git a/spinifex/runtime/compute/vm/lifecycle.go b/spinifex/runtime/compute/vm/lifecycle.go
index 389e7a764..c64bc6a38 100644
--- a/spinifex/runtime/compute/vm/lifecycle.go
+++ b/spinifex/runtime/compute/vm/lifecycle.go
@@ -12,6 +12,7 @@ import (
"net"
"os"
"path/filepath"
+ "slices"
"strconv"
"strings"
"syscall"
@@ -900,12 +901,9 @@ func qmpGreetingTimeout(v *VM) time.Duration {
// monitor answers. Every production guest boots from one; a VM without is a
// unit test.
func hasNetworkBackedDrive(v *VM) bool {
- for _, req := range v.EBSRequests.Requests {
- if req.NBDURI != "" {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(v.EBSRequests.Requests, func(req types.EBSRequest) bool {
+ return req.NBDURI != ""
+ })
}
// removeStaleQMPSocket unlinks a leftover QMP socket inode from a prior QEMU so
diff --git a/spinifex/runtime/compute/vm/manager.go b/spinifex/runtime/compute/vm/manager.go
index 2251f0de6..6e96267b3 100644
--- a/spinifex/runtime/compute/vm/manager.go
+++ b/spinifex/runtime/compute/vm/manager.go
@@ -2,6 +2,7 @@ package vm
import (
"maps"
+ "slices"
"sync"
"time"
)
@@ -191,11 +192,7 @@ func (m *Manager) ForEach(fn func(*VM)) {
func (m *Manager) Snapshot() []*VM {
m.mu.Lock()
defer m.mu.Unlock()
- out := make([]*VM, 0, len(m.vms))
- for _, v := range m.vms {
- out = append(out, v)
- }
- return out
+ return slices.Collect(maps.Values(m.vms))
}
// SnapshotMap returns a copy of the id→VM map for serialization outside the lock.
diff --git a/spinifex/runtime/compute/vm/manager_deps_test.go b/spinifex/runtime/compute/vm/manager_deps_test.go
index c18709a5c..b5e116eb0 100644
--- a/spinifex/runtime/compute/vm/manager_deps_test.go
+++ b/spinifex/runtime/compute/vm/manager_deps_test.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"maps"
+ "slices"
"sync"
"github.com/mulgadc/spinifex/spinifex/types"
@@ -76,11 +77,7 @@ func (f *fakeStateStore) DeleteStoppedInstance(id string) error {
func (f *fakeStateStore) ListStoppedInstances() ([]*VM, error) {
f.mu.Lock()
defer f.mu.Unlock()
- out := make([]*VM, 0, len(f.stopped))
- for _, v := range f.stopped {
- out = append(out, v)
- }
- return out, nil
+ return slices.Collect(maps.Values(f.stopped)), nil
}
// ClaimStoppedInstance mimics the real atomic-delete claim for tests: under
@@ -107,11 +104,7 @@ func (f *fakeStateStore) WriteTerminatedInstance(id string, v *VM) error {
func (f *fakeStateStore) ListTerminatedInstances() ([]*VM, error) {
f.mu.Lock()
defer f.mu.Unlock()
- out := make([]*VM, 0, len(f.terminated))
- for _, v := range f.terminated {
- out = append(out, v)
- }
- return out, nil
+ return slices.Collect(maps.Values(f.terminated)), nil
}
func (f *fakeStateStore) DeleteTerminatedInstance(id string) error {
diff --git a/spinifex/runtime/compute/vm/mock/mock.go b/spinifex/runtime/compute/vm/mock/mock.go
index 56a055d93..86c7b8937 100644
--- a/spinifex/runtime/compute/vm/mock/mock.go
+++ b/spinifex/runtime/compute/vm/mock/mock.go
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"maps"
+ "slices"
"sync"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
@@ -170,11 +171,7 @@ func (s *StateStore) ListStoppedInstances() ([]*vm.VM, error) {
if s.ListStoppedErr != nil {
return nil, s.ListStoppedErr
}
- out := make([]*vm.VM, 0, len(s.Stopped))
- for _, v := range s.Stopped {
- out = append(out, v)
- }
- return out, nil
+ return slices.Collect(maps.Values(s.Stopped)), nil
}
// ClaimStoppedInstance mimics the real atomic delete-as-claim: under the
@@ -237,11 +234,7 @@ func (s *StateStore) ListTerminatedInstances() ([]*vm.VM, error) {
if s.ListTerminatedErr != nil {
return nil, s.ListTerminatedErr
}
- out := make([]*vm.VM, 0, len(s.Terminated))
- for _, v := range s.Terminated {
- out = append(out, v)
- }
- return out, nil
+ return slices.Collect(maps.Values(s.Terminated)), nil
}
func (s *StateStore) DeleteTerminatedInstance(id string) error {
diff --git a/spinifex/runtime/roles/nats/nats.go b/spinifex/runtime/roles/nats/nats.go
index 14b17e3a1..2acd5abe9 100644
--- a/spinifex/runtime/roles/nats/nats.go
+++ b/spinifex/runtime/roles/nats/nats.go
@@ -47,22 +47,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-func (svc *Service) Stop() (err error) {
- return utils.StopProcessAt(svc.Config.DataDir, serviceName)
-}
-
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.DataDir, serviceName)
-}
-
-func (svc *Service) Shutdown() (err error) {
- return svc.Stop()
-}
-
-func (svc *Service) Reload() (err error) {
- return nil
-}
-
func launchService(config *Config) (err error) {
// Create proper server options
var opts *server.Options
diff --git a/spinifex/runtime/roles/nats/servername.go b/spinifex/runtime/roles/nats/servername.go
new file mode 100644
index 000000000..647443cbb
--- /dev/null
+++ b/spinifex/runtime/roles/nats/servername.go
@@ -0,0 +1,27 @@
+package nats
+
+import "strings"
+
+// ServerNamePrefix is what nats.conf prefixes a node name with to build
+// server_name. JetStream reports stream peers by server name, so anything
+// mapping a replica back to the node holding it has to go through here rather
+// than assume the two are the same string — they are not.
+const ServerNamePrefix = "spinifex-nats-"
+
+// ServerName is the NATS server_name for a node in this cluster.
+func ServerName(node string) string { return ServerNamePrefix + node }
+
+// NodeFromServerName is ServerName's inverse. The second result is false for a
+// name this cluster did not generate, so a caller can tell "not our node" from
+// "a node called the empty string".
+//
+// The bare prefix is rejected too. It is a name we could not have generated, and
+// mapping it to the empty node would silently create a phantom peer in anything
+// counting nodes.
+func NodeFromServerName(server string) (string, bool) {
+ node, ok := strings.CutPrefix(server, ServerNamePrefix)
+ if !ok || node == "" {
+ return "", false
+ }
+ return node, true
+}
diff --git a/spinifex/runtime/roles/qemunbdd/service.go b/spinifex/runtime/roles/qemunbdd/service.go
index 1951b6782..885ad60f4 100644
--- a/spinifex/runtime/roles/qemunbdd/service.go
+++ b/spinifex/runtime/roles/qemunbdd/service.go
@@ -54,22 +54,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-func (svc *Service) Stop() (err error) {
- return utils.StopProcessAt(svc.Config.BaseDir, serviceName)
-}
-
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BaseDir, serviceName)
-}
-
-func (svc *Service) Shutdown() (err error) {
- return svc.Stop()
-}
-
-func (svc *Service) Reload() (err error) {
- return nil
-}
-
// launchService connects to NATS, roots a qcow2 provider at cfg.BaseDir and
// serves ebs.provider.v1.* until SIGINT/SIGTERM, then unsubscribes. It blocks
// for the life of the process.
diff --git a/spinifex/runtime/roles/qmpcollector/qmpcollector.go b/spinifex/runtime/roles/qmpcollector/qmpcollector.go
index 44b2c88d4..318c0939c 100644
--- a/spinifex/runtime/roles/qmpcollector/qmpcollector.go
+++ b/spinifex/runtime/roles/qmpcollector/qmpcollector.go
@@ -72,26 +72,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-// Stop stops the qmp-collector service.
-func (svc *Service) Stop() error {
- return utils.StopProcessAt(svc.Config.BaseDir, serviceName)
-}
-
-// Status returns the qmp-collector service status.
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BaseDir, serviceName)
-}
-
-// Shutdown gracefully shuts down the qmp-collector service.
-func (svc *Service) Shutdown() error {
- return svc.Stop()
-}
-
-// Reload reloads the qmp-collector service configuration.
-func (svc *Service) Reload() error {
- return nil
-}
-
func launchService(cfg *Config) error {
if cfg.RuntimeDir == "" {
cfg.RuntimeDir = utils.RuntimeDir()
diff --git a/spinifex/runtime/roles/spinifex/spinifex.go b/spinifex/runtime/roles/spinifex/spinifex.go
index 009f92cad..5599a27ec 100644
--- a/spinifex/runtime/roles/spinifex/spinifex.go
+++ b/spinifex/runtime/roles/spinifex/spinifex.go
@@ -44,22 +44,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-func (svc *Service) Stop() (err error) {
- return utils.StopProcessAt(svc.Config.NodeBaseDir(), serviceName)
-}
-
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.NodeBaseDir(), serviceName)
-}
-
-func (svc *Service) Shutdown() (err error) {
- return svc.Stop()
-}
-
-func (svc *Service) Reload() (err error) {
- return nil
-}
-
func launchService(config *config.ClusterConfig, configPath string) (err error) {
d, err := daemon.NewDaemon(config)
if err != nil {
diff --git a/spinifex/runtime/roles/spinifexui/spinifexui.go b/spinifex/runtime/roles/spinifexui/spinifexui.go
index f1aa63105..39628cad7 100644
--- a/spinifex/runtime/roles/spinifexui/spinifexui.go
+++ b/spinifex/runtime/roles/spinifexui/spinifexui.go
@@ -15,7 +15,6 @@ import (
"os/signal"
"path/filepath"
"strings"
- "sync"
"syscall"
"time"
@@ -83,8 +82,6 @@ func clusterConfigHandler(region string, ochreEnabled bool) http.HandlerFunc {
// Service represents the spinifex-ui service.
type Service struct {
Config *Config
- server *http.Server
- mu sync.Mutex
}
// New creates a new spinifex-ui service.
@@ -141,35 +138,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-// Stop stops the spinifex-ui service.
-func (svc *Service) Stop() error {
- return utils.StopProcessAt(svc.Config.BaseDir, serviceName)
-}
-
-// Status returns the status of the spinifex-ui service.
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BaseDir, serviceName)
-}
-
-// Shutdown gracefully shuts down the spinifex-ui service.
-func (svc *Service) Shutdown() error {
- svc.mu.Lock()
- server := svc.server
- svc.mu.Unlock()
-
- if server != nil {
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
- return server.Shutdown(ctx)
- }
- return svc.Stop()
-}
-
-// Reload reloads the spinifex-ui service configuration.
-func (svc *Service) Reload() error {
- return nil
-}
-
// launchService starts the HTTP server.
// newSPAHandler serves the embedded build, falling back to index.html so the
// router can resolve client-side paths.
@@ -296,10 +264,6 @@ func (svc *Service) launchService() error {
ReadHeaderTimeout: 5 * time.Second,
}
- svc.mu.Lock()
- svc.server = server
- svc.mu.Unlock()
-
// Setup graceful shutdown
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
diff --git a/spinifex/runtime/roles/spinifexui/spinifexui_test.go b/spinifex/runtime/roles/spinifexui/spinifexui_test.go
index fcb948260..3ae20fc5e 100644
--- a/spinifex/runtime/roles/spinifexui/spinifexui_test.go
+++ b/spinifex/runtime/roles/spinifexui/spinifexui_test.go
@@ -19,6 +19,7 @@ import (
"path/filepath"
"regexp"
"strconv"
+ "syscall"
"testing"
"time"
@@ -114,27 +115,6 @@ func TestNew_InvalidConfigTypeInt(t *testing.T) {
assert.Error(t, err)
}
-func TestStatus_ReturnsValidState(t *testing.T) {
- svc := &Service{
- Config: &Config{},
- }
-
- status, err := svc.Status()
-
- assert.NoError(t, err)
- // On a dev machine the spinifex-ui PID file may exist, so accept either outcome
- assert.True(t, status == "stopped" || len(status) > 0, "status should be non-empty")
-}
-
-func TestReload_ReturnsNil(t *testing.T) {
- svc := &Service{
- Config: &Config{},
- }
-
- err := svc.Reload()
- assert.NoError(t, err)
-}
-
func TestServiceName(t *testing.T) {
assert.Equal(t, "spinifex-ui", serviceName)
}
@@ -459,28 +439,6 @@ func TestStart_WritesPidFileToNestedBaseDir(t *testing.T) {
assert.Equal(t, strconv.Itoa(os.Getpid()), string(data))
}
-func TestShutdown_WithServer(t *testing.T) {
- // Create a real server on a random port so Shutdown exercises the non-nil path
- mux := http.NewServeMux()
- mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusOK)
- })
-
- srv := &http.Server{Handler: mux}
- ln, err := net.Listen("tcp", "127.0.0.1:0")
- require.NoError(t, err)
-
- go srv.Serve(ln)
-
- svc := &Service{
- Config: &Config{},
- server: srv,
- }
-
- err = svc.Shutdown()
- assert.NoError(t, err)
-}
-
// writeTestTLSFiles generates a self-signed cert/key pair on disk for a
// launchService test, which needs real file paths (tls.LoadX509KeyPair),
// unlike loadTLSCert's in-memory tls.Certificate used by the split-listener tests.
@@ -529,15 +487,16 @@ func freePort(t *testing.T) int {
}
// TestStart_GracefulShutdownIsNotAnError guards against Start propagating
-// http.ErrServerClosed -- Serve's documented return value after a deliberate
-// Shutdown -- as a failure. Before the fix this made a clean stop look like
+// http.ErrServerClosed -- Serve's documented return value after the SIGTERM
+// handler's Shutdown -- as a failure. Before the fix this made a clean stop look like
// a crash (exit code 1) to systemd.
func TestStart_GracefulShutdownIsNotAnError(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
certPath, keyPath := writeTestTLSFiles(t)
+ port := freePort(t)
svc, err := New(&Config{
- Port: freePort(t),
+ Port: port,
Host: "127.0.0.1",
TLSCert: certPath,
TLSKey: keyPath,
@@ -550,19 +509,25 @@ func TestStart_GracefulShutdownIsNotAnError(t *testing.T) {
startErrCh <- startErr
}()
+ // Start registers for SIGTERM before it listens, so a successful dial means
+ // the signal will not kill the test binary.
+ addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
require.Eventually(t, func() bool {
- svc.mu.Lock()
- defer svc.mu.Unlock()
- return svc.server != nil
- }, 2*time.Second, 10*time.Millisecond, "server never registered before shutdown")
+ conn, err := net.DialTimeout("tcp", addr, time.Second)
+ if err != nil {
+ return false
+ }
+ conn.Close()
+ return true
+ }, 2*time.Second, 10*time.Millisecond, "server never listened on %s", addr)
- require.NoError(t, svc.Shutdown())
+ require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGTERM))
select {
case startErr := <-startErrCh:
- assert.NoError(t, startErr, "Start must treat a deliberate Shutdown's ErrServerClosed as success")
+ assert.NoError(t, startErr, "Start must treat a graceful shutdown's ErrServerClosed as success")
case <-time.After(2 * time.Second):
- t.Fatal("Start did not return after Shutdown")
+ t.Fatal("Start did not return after SIGTERM")
}
}
diff --git a/spinifex/runtime/service/service.go b/spinifex/runtime/service/service.go
index 325047626..9ba713b0b 100644
--- a/spinifex/runtime/service/service.go
+++ b/spinifex/runtime/service/service.go
@@ -17,10 +17,6 @@ import (
type Service interface {
Start() (int, error)
- Stop() error
- Status() (string, error)
- Shutdown() error
- Reload() error
}
var (
diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go
index 4941e5d6b..17e914a95 100644
--- a/spinifex/services/awsgw/awsgw.go
+++ b/spinifex/services/awsgw/awsgw.go
@@ -4,14 +4,15 @@ import (
"context"
"crypto/tls"
"crypto/x509"
- "errors"
"fmt"
"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
"log/slog"
+ "maps"
"net"
"net/http"
"os"
"path/filepath"
+ "slices"
"strings"
"time"
@@ -23,6 +24,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/daemon"
"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/gateway"
gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance"
@@ -88,22 +90,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-func (svc *Service) Stop() (err error) {
- return utils.StopProcessAt(svc.Config.NodeBaseDir(), serviceName)
-}
-
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.NodeBaseDir(), serviceName)
-}
-
-func (svc *Service) Shutdown() (err error) {
- return svc.Stop()
-}
-
-func (svc *Service) Reload() (err error) {
- return nil
-}
-
// awsgwTOML is the top-level structure of awsgw.toml used to extract the
// ratelimit and quota sections. Other fields are parsed elsewhere (e.g. region,
// debug).
@@ -152,53 +138,15 @@ func loadAWSGWConfig(path string) (awsgwTOML, error) {
// openAccountUsageBucket opens (or idempotently creates) the gateway-owned
// per-account vCPU usage bucket. History is 1: each account key holds a single
// CAS-updated integer counter, so no revision beyond the latest is worth keeping.
-func openAccountUsageBucket(ctx context.Context, js jetstream.KeyValueManager, replicas int) (jetstream.KeyValue, error) {
- kv, err := js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage)
- if err == nil {
- return kv, nil
- }
- if !errors.Is(err, jetstream.ErrBucketNotFound) {
- return nil, fmt.Errorf("open account usage bucket: %w", err)
- }
-
- kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: handlers_quota.KVBucketAccountUsage,
- History: 1,
- Replicas: max(replicas, 1),
- })
- if errors.Is(err, jetstream.ErrBucketExists) {
- return js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage)
- }
- if err != nil {
- return nil, fmt.Errorf("create account usage bucket: %w", err)
- }
- return kv, nil
+func openAccountUsageBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
+ return kvutil.GetOrCreateBucket(ctx, js, handlers_quota.KVBucketAccountUsage, 1)
}
// openAccountQuotaBucket opens (or idempotently creates) the per-account quota
// override bucket. History is 1: each key holds the current override set, and
// no earlier revision of a limit is worth keeping.
-func openAccountQuotaBucket(ctx context.Context, js jetstream.KeyValueManager, replicas int) (jetstream.KeyValue, error) {
- kv, err := js.KeyValue(ctx, handlers_quota.KVBucketAccountQuota)
- if err == nil {
- return kv, nil
- }
- if !errors.Is(err, jetstream.ErrBucketNotFound) {
- return nil, fmt.Errorf("open account quota bucket: %w", err)
- }
-
- kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: handlers_quota.KVBucketAccountQuota,
- History: 1,
- Replicas: max(replicas, 1),
- })
- if errors.Is(err, jetstream.ErrBucketExists) {
- return js.KeyValue(ctx, handlers_quota.KVBucketAccountQuota)
- }
- if err != nil {
- return nil, fmt.Errorf("create account quota bucket: %w", err)
- }
- return kv, nil
+func openAccountQuotaBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
+ return kvutil.GetOrCreateBucket(ctx, js, handlers_quota.KVBucketAccountQuota, 1)
}
func launchService(config *config.ClusterConfig) error {
@@ -241,14 +189,14 @@ func launchService(config *config.ClusterConfig) error {
// Initialize IAM service with NATS KV backend (required for auth).
// On multi-node clusters, JetStream KV requires cluster quorum which may
// not be available yet if nodes start concurrently. Retry with backoff.
- iamService, err := handlers_iam.NewIAMServiceWithRetry(janitorCtx, natsConn, masterKey, len(config.Nodes))
+ iamService, err := handlers_iam.NewIAMServiceWithRetry(janitorCtx, natsConn, masterKey)
if err != nil {
return fmt.Errorf("initialize IAM service: %w", err)
}
// STS service shares the IAM master key (single envelope for at-rest
// secrets + session-token HMACs) and resolves roles via IAMService.
- stsService, err := handlers_sts.NewSTSServiceImpl(janitorCtx, natsConn, iamService, masterKey, len(config.Nodes))
+ stsService, err := handlers_sts.NewSTSServiceImpl(janitorCtx, natsConn, iamService, masterKey)
if err != nil {
return fmt.Errorf("initialize STS service: %w", err)
}
@@ -340,7 +288,7 @@ func launchService(config *config.ClusterConfig) error {
if err != nil {
return fmt.Errorf("jetstream client: %w", err)
}
- signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(janitorCtx, js, masterKey, len(config.Nodes))
+ signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(janitorCtx, js, masterKey)
if err != nil {
return fmt.Errorf("ECR auth bridge: load signing key: %w", err)
}
@@ -370,19 +318,19 @@ func launchService(config *config.ClusterConfig) error {
if key := os.Getenv("OCHRE_ANTHROPIC_API_KEY"); key != "" {
bedrockPlatformDefaults["anthropic"] = key
}
- bedrockCredentials := gateway_bedrock.NewCredentialStore(js, masterKey, len(config.Nodes), bedrockPlatformDefaults)
+ bedrockCredentials := gateway_bedrock.NewCredentialStore(js, masterKey, bedrockPlatformDefaults)
// Bedrock self-host weights: a model's serving spec (VRAM, instance type,
// vLLM args) ships in-tree, but which staged snapshot serves it is
// deployment-local state. tieredCatalog/GetFoundationModel read this
// through the package-level resolver rather than a parameter, since they
// are called from gateway/bedrock.go's fixed-arity route table.
- gateway_bedrock.SetWeightsResolver(gateway_bedrock.NewWeightsStore(js, len(config.Nodes)))
+ gateway_bedrock.SetWeightsResolver(gateway_bedrock.NewWeightsStore(js))
// Bedrock model access: grants live in the bedrock-model-access KV bucket
// and are deny-by-default, so a fresh deployment serves no models until an
// operator grants them (spx admin ochre access grant).
- bedrockAccess := gateway_bedrock.NewModelAccessStore(js, len(config.Nodes))
+ bedrockAccess := gateway_bedrock.NewModelAccessStore(js)
// Deny-by-default would otherwise leave a fresh install with a catalog
// nobody can see, so seed the platform admin account — the operator's own
@@ -434,13 +382,13 @@ func launchService(config *config.ClusterConfig) error {
// gateway_bedrock's narrow EndpointProvisioner (see provisioned_adapter.go
// for why the adapter, not handlers_bedrock.EndpointService, is what
// gateway_bedrock depends on).
- bedrockProvisioned := gateway_bedrock.NewProvisionedStore(js, len(config.Nodes), nodeConfig.Region,
+ bedrockProvisioned := gateway_bedrock.NewProvisionedStore(js, nodeConfig.Region,
handlers_bedrock.NewProvisionedEndpointAdapter(bedrockEndpointSvc))
// Bedrock guardrails: control-plane CRUD only at this stage — the record
// stores the full policy config so a later stage's filter engine and
// inference enforcement can read it back, but nothing enforces it yet.
- bedrockGuardrails := gateway_bedrock.NewGuardrailStore(js, len(config.Nodes), nodeConfig.Region)
+ bedrockGuardrails := gateway_bedrock.NewGuardrailStore(js, nodeConfig.Region)
// bedrock-agent knowledge-base + data-source resource metadata: gateway-
// owned (D-arch), opened directly against JetStream the same way the
@@ -456,8 +404,8 @@ func launchService(config *config.ClusterConfig) error {
// deliveryConsumer to any account with a configured destination bucket
// and a metadata-only log line. bedrockLoggingConfig separately gates
// whether the record written to that bucket includes body text.
- bedrockLoggingConfig := gateway_bedrock.NewLoggingConfigStore(js, len(config.Nodes))
- if _, err := gateway_bedrock.EnsureInvocationStream(janitorCtx, js, len(config.Nodes)); err != nil {
+ bedrockLoggingConfig := gateway_bedrock.NewLoggingConfigStore(js)
+ if _, err := gateway_bedrock.EnsureInvocationStream(janitorCtx, js); err != nil {
return fmt.Errorf("bedrock: ensure invocation stream: %w", err)
}
deliveryConsumer, err := gateway_bedrock.EnsureDeliveryConsumer(janitorCtx, js)
@@ -472,8 +420,8 @@ func launchService(config *config.ClusterConfig) error {
// updates per-account/model/period counters, deduping on RequestID so
// at-least-once redelivery never double-counts. bedrockPrices resolves
// KV price overrides over the catalog's in-tree defaults.
- bedrockUsage := gateway_bedrock.NewUsageStore(js, len(config.Nodes))
- bedrockPrices := gateway_bedrock.NewPriceStore(js, len(config.Nodes))
+ bedrockUsage := gateway_bedrock.NewUsageStore(js)
+ bedrockPrices := gateway_bedrock.NewPriceStore(js)
usageConsumer, err := gateway_bedrock.EnsureUsageConsumer(janitorCtx, js)
if err != nil {
return fmt.Errorf("bedrock: ensure usage metering consumer: %w", err)
@@ -483,10 +431,7 @@ func launchService(config *config.ClusterConfig) error {
// nodeIDs is the configured cluster node set, in the same namespace as the
// daemon reply header, so a fan-out's completeness can be judged by
// responder identity rather than by count.
- nodeIDs := make([]string, 0, len(config.Nodes))
- for nodeID := range config.Nodes {
- nodeIDs = append(nodeIDs, nodeID)
- }
+ nodeIDs := slices.Collect(maps.Keys(config.Nodes))
gw := gateway.GatewayConfig{
Debug: nodeConfig.AWSGW.Debug,
@@ -528,7 +473,7 @@ func launchService(config *config.ClusterConfig) error {
// Rotate the ECR signing key on a 30-day cadence, retaining the previous keys
// until their tokens expire. The rotator keeps the issuer/verifier current as
// keys roll. Bound to the same lifetime context as the STS janitor.
- keyRotator, err := gateway_ecrauth.NewRotator(janitorCtx, js, masterKey, len(config.Nodes), gw.ECRTokenIssuer, gw.ECRTokenVerifier)
+ keyRotator, err := gateway_ecrauth.NewRotator(janitorCtx, js, masterKey, gw.ECRTokenIssuer, gw.ECRTokenVerifier)
if err != nil {
return fmt.Errorf("ECR auth bridge: signing-key rotator: %w", err)
}
@@ -544,11 +489,11 @@ func launchService(config *config.ClusterConfig) error {
// config builds a no-op Service whose Exempt short-circuits every check.
var usageBucket, quotaOverrides jetstream.KeyValue
if quotaCfg.Enabled {
- usageBucket, err = openAccountUsageBucket(janitorCtx, js, len(config.Nodes))
+ usageBucket, err = openAccountUsageBucket(janitorCtx, js)
if err != nil {
return fmt.Errorf("init account usage bucket: %w", err)
}
- quotaOverrides, err = openAccountQuotaBucket(janitorCtx, js, len(config.Nodes))
+ quotaOverrides, err = openAccountQuotaBucket(janitorCtx, js)
if err != nil {
return fmt.Errorf("init account quota bucket: %w", err)
}
@@ -572,13 +517,13 @@ func launchService(config *config.ClusterConfig) error {
// quotaCfg.Enabled); this loop only pushes a periodic observability
// snapshot to KV and is itself a no-op when the RPM dimension is
// disabled.
- go gw.Quota.RunBedrockRPMSync(janitorCtx, js, len(config.Nodes))
+ go gw.Quota.RunBedrockRPMSync(janitorCtx, js)
// Instance cache: a read-only informer over the live instance record space,
// started now so it is warm well before anything reads it. The describe
// path still serves from the fan-out and KV; only status synthesis reads it.
instanceCache := instancecache.New(js, instancecache.Config{
- Bucket: kvstore.Config{Name: daemon.InstanceStateBucket, History: 1, Replicas: len(config.Nodes)},
+ Bucket: kvstore.Config{Name: daemon.InstanceStateBucket, History: 1},
Prefix: daemon.InstanceRecordPrefix,
VisibleToCaller: handlers_ec2_instance.IsInstanceVisibleToCaller,
FallbackAccountID: utils.GlobalAccountID,
@@ -590,7 +535,7 @@ func launchService(config *config.ClusterConfig) error {
gw.InstanceStatus = gateway_ec2_instance.StatusSynthesis{
Records: instanceCache,
Liveness: instancecache.NewLiveness(js, kvstore.Config{
- Name: daemon.ClusterStateBucket, History: 1, Replicas: len(config.Nodes),
+ Name: daemon.ClusterStateBucket, History: 1,
}),
}
@@ -636,7 +581,7 @@ func launchService(config *config.ClusterConfig) error {
func runQuotaReconcile(ctx context.Context, quota *handlers_quota.Service, natsConn *nats.Conn,
js jetstream.JetStream, accounts handlers_quota.AccountLister, replicas int) {
holder, _ := os.Hostname()
- cfg := kvstore.Config{Name: daemon.InstanceStateBucket, History: 1, Replicas: replicas}
+ cfg := kvstore.Config{Name: daemon.InstanceStateBucket, History: 1}
list := handlers_quota.RecordVCPULister(
kvstore.New[vm.InstanceRecord](js, cfg), daemon.InstanceRecordPrefix)
diff --git a/spinifex/services/awsgw/quota_reconcile_test.go b/spinifex/services/awsgw/quota_reconcile_test.go
index 766c8a48c..c9158d1ef 100644
--- a/spinifex/services/awsgw/quota_reconcile_test.go
+++ b/spinifex/services/awsgw/quota_reconcile_test.go
@@ -24,7 +24,7 @@ func TestOpenAccountUsageBucketPreservesExistingConfiguration(t *testing.T) {
})
require.NoError(t, err)
- bucket, err := openAccountUsageBucket(t.Context(), js, 1)
+ bucket, err := openAccountUsageBucket(t.Context(), js)
require.NoError(t, err)
status, err := bucket.Status(t.Context())
require.NoError(t, err)
@@ -63,7 +63,7 @@ func instanceRecordBucket(t *testing.T, js jetstream.JetStream) jetstream.KeyVal
func TestRunQuotaReconcileCountsFromTheRecordSpace(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- bucket, err := openAccountUsageBucket(t.Context(), js, 1)
+ bucket, err := openAccountUsageBucket(t.Context(), js)
require.NoError(t, err)
quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket)
@@ -99,7 +99,7 @@ func TestRunQuotaReconcileCountsFromTheRecordSpace(t *testing.T) {
func TestRunQuotaReconcileFollowsARecordChange(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- bucket, err := openAccountUsageBucket(t.Context(), js, 1)
+ bucket, err := openAccountUsageBucket(t.Context(), js)
require.NoError(t, err)
quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket)
@@ -135,7 +135,7 @@ func TestRunQuotaReconcileFollowsARecordChange(t *testing.T) {
func TestRunQuotaReconcileRecomputesOnlyTheChangedAccount(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
js := testutil.NewJetStream(t, nc)
- bucket, err := openAccountUsageBucket(t.Context(), js, 1)
+ bucket, err := openAccountUsageBucket(t.Context(), js)
require.NoError(t, err)
quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket)
diff --git a/spinifex/services/northstar/bootstrap.go b/spinifex/services/northstar/bootstrap.go
index 756ae7c35..d70ba99b7 100644
--- a/spinifex/services/northstar/bootstrap.go
+++ b/spinifex/services/northstar/bootstrap.go
@@ -3,7 +3,8 @@ package northstar
import (
"fmt"
"log/slog"
- "sort"
+ "maps"
+ "slices"
"strings"
"time"
@@ -150,10 +151,5 @@ func ensureZone(s3cfg *nsconfig.S3Config, seed nsconfig.BaseZoneSeed) error {
// nodeNames returns the sorted node keys of the cluster, for diagnostics.
func nodeNames(cluster *config.ClusterConfig) []string {
- names := make([]string, 0, len(cluster.Nodes))
- for name := range cluster.Nodes {
- names = append(names, name)
- }
- sort.Strings(names)
- return names
+ return slices.Sorted(maps.Keys(cluster.Nodes))
}
diff --git a/spinifex/services/northstar/northstar.go b/spinifex/services/northstar/northstar.go
index f0dfe8511..4d866ceac 100644
--- a/spinifex/services/northstar/northstar.go
+++ b/spinifex/services/northstar/northstar.go
@@ -126,31 +126,3 @@ func (svc *Service) subscribeReload() {
}
slog.Info("northstar: subscribed to live zone reload", "subject", handlers_dns.SubjectZoneReload)
}
-
-// Stop signals a running northstar service via its PID file.
-func (svc *Service) Stop() error {
- return utils.StopProcessAt(svc.Config.BasePath, serviceName)
-}
-
-// Status returns the status of the northstar service.
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BasePath, serviceName)
-}
-
-// Shutdown gracefully shuts down the in-process server, falling back to Stop.
-func (svc *Service) Shutdown() error {
- if svc.server != nil {
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- return svc.server.Shutdown(ctx)
- }
- return svc.Stop()
-}
-
-// Reload re-reads the zone database without restarting the listeners.
-func (svc *Service) Reload() error {
- if svc.server != nil {
- return svc.server.Reload()
- }
- return nil
-}
diff --git a/spinifex/services/northstar/northstar_test.go b/spinifex/services/northstar/northstar_test.go
index e6dd371f4..ca8c72c60 100644
--- a/spinifex/services/northstar/northstar_test.go
+++ b/spinifex/services/northstar/northstar_test.go
@@ -31,9 +31,3 @@ func TestStartRequiresConfigPath(t *testing.T) {
_, err = svc.Start()
require.Error(t, err)
}
-
-func TestReloadNilServerIsNoop(t *testing.T) {
- svc, err := New(&Config{})
- require.NoError(t, err)
- require.NoError(t, svc.Reload())
-}
diff --git a/spinifex/services/predastore/predastore.go b/spinifex/services/predastore/predastore.go
index 6b8dd7832..2f2d4663f 100644
--- a/spinifex/services/predastore/predastore.go
+++ b/spinifex/services/predastore/predastore.go
@@ -7,7 +7,6 @@ import (
"log/slog"
"os"
"os/signal"
- "sync"
"syscall"
"github.com/mulgadc/bluebottle/pkg/masterkey"
@@ -30,8 +29,7 @@ type Config struct {
TlsKey string
// BasePath is where this service keeps its pid file. Predastore itself has
- // no base path — its directories come from the configuration file — but
- // Stop and Status find the running process through this one.
+ // no base path; its directories come from the configuration file.
BasePath string
// EncryptionKeyFile is the path to this node's 32-byte AES-256 master
@@ -48,12 +46,6 @@ type Config struct {
// Service runs one predastore host in this process.
type Service struct {
Config *Config
-
- // stop cancels the process context Start runs under, so Shutdown can
- // drain the gate and the nodes together. Guarded because Shutdown is
- // called from a different goroutine than Start.
- mu sync.Mutex
- stop context.CancelFunc
}
// New creates a new predastore service.
@@ -93,15 +85,11 @@ func (svc *Service) Start() (int, error) {
return 0, fmt.Errorf("load predastore master key: %w", err)
}
- // One context for the whole service: a signal, a fatal node error or
- // Shutdown stops the gate and every local node together.
+ // One context for the whole service: a signal or a fatal node error stops
+ // the gate and every local node together.
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
- svc.mu.Lock()
- svc.stop = stop
- svc.mu.Unlock()
-
if err := pds.Run(ctx, pds.Options{
Config: cfg,
HostID: hostID,
@@ -160,33 +148,3 @@ func (svc *Service) mergeHost(cfg *pds.Config) (pds.HostID, error) {
return hostID, cfg.Validate()
}
-
-// Stop stops the predastore service.
-func (svc *Service) Stop() error {
- return utils.StopProcessAt(svc.Config.BasePath, serviceName)
-}
-
-// Status returns the status of the predastore service.
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BasePath, serviceName)
-}
-
-// Shutdown gracefully shuts down the predastore service. When this process is
-// the one running the host it cancels the service context, which drains the
-// gate and every local node; otherwise it signals the running process.
-func (svc *Service) Shutdown() error {
- svc.mu.Lock()
- stop := svc.stop
- svc.mu.Unlock()
-
- if stop != nil {
- stop()
- return nil
- }
- return svc.Stop()
-}
-
-// Reload reloads the predastore service configuration.
-func (svc *Service) Reload() error {
- return nil
-}
diff --git a/spinifex/services/predastore/predastore_start_test.go b/spinifex/services/predastore/predastore_start_test.go
index 7e0e291f4..026b90d0c 100644
--- a/spinifex/services/predastore/predastore_start_test.go
+++ b/spinifex/services/predastore/predastore_start_test.go
@@ -12,6 +12,7 @@ import (
"os"
"path/filepath"
"strconv"
+ "syscall"
"testing"
"time"
@@ -166,10 +167,10 @@ func TestStartRejectsUnprotectedMasterKey(t *testing.T) {
assert.Contains(t, err.Error(), "load predastore master key")
}
-// TestStartServesUntilShutdown covers the service lifecycle end to end: Start
-// blocks for as long as the host serves, and Shutdown cancels the context the
+// TestStartServesUntilSIGTERM covers the service lifecycle end to end: Start
+// blocks for as long as the host serves, and SIGTERM cancels the context the
// gate and the local nodes share, which is what lets Start return at all.
-func TestStartServesUntilShutdown(t *testing.T) {
+func TestStartServesUntilSIGTERM(t *testing.T) {
dir := t.TempDir()
basePath := t.TempDir()
gatePort := freePort(t)
@@ -191,8 +192,8 @@ func TestStartServesUntilShutdown(t *testing.T) {
done <- result{pid: pid, err: err}
}()
- // Shutdown must not run before Start has published its cancel func, and an
- // accepted connection is the first observable point at which it has.
+ // SIGTERM must not arrive before Start has registered for it, or it kills
+ // the test binary; an accepted connection is the first point it has.
addr := net.JoinHostPort(startAddr, strconv.Itoa(gatePort))
require.Eventually(t, func() bool {
conn, err := net.DialTimeout("tcp", addr, time.Second)
@@ -204,13 +205,13 @@ func TestStartServesUntilShutdown(t *testing.T) {
}, 60*time.Second, 100*time.Millisecond, "gate never accepted a connection on %s", addr)
assert.FileExists(t, filepath.Join(basePath, serviceName+".pid"))
- require.NoError(t, svc.Shutdown())
+ require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGTERM))
select {
case res := <-done:
require.NoError(t, res.err)
assert.Equal(t, os.Getpid(), res.pid)
case <-time.After(60 * time.Second):
- t.Fatal("Start did not return after Shutdown")
+ t.Fatal("Start did not return after SIGTERM")
}
}
diff --git a/spinifex/services/predastore/predastore_test.go b/spinifex/services/predastore/predastore_test.go
index 95e12c25f..65acffd53 100644
--- a/spinifex/services/predastore/predastore_test.go
+++ b/spinifex/services/predastore/predastore_test.go
@@ -1,16 +1,13 @@
package predastore
import (
- "context"
"fmt"
"os"
- "os/exec"
"path/filepath"
"strings"
"testing"
pds "github.com/mulgadc/predastore"
- "github.com/mulgadc/spinifex/spinifex/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -148,15 +145,6 @@ func TestServiceNameConstant(t *testing.T) {
assert.Equal(t, "predastore", serviceName)
}
-// TestReloadIsANoOp pins the current contract: predastore has no reload, and
-// the service reports success rather than an error a caller would have to
-// special-case.
-func TestReloadIsANoOp(t *testing.T) {
- svc, err := New(&Config{})
- require.NoError(t, err)
- assert.NoError(t, svc.Reload())
-}
-
// TestMergeHostFlagsBeatFile covers the precedence spinifex.toml relies on:
// the service flags own the host-local fields, so a value from them replaces
// whatever the predastore file carried.
@@ -326,75 +314,3 @@ func TestStartRejectsMissingEncryptionKey(t *testing.T) {
assert.Zero(t, pid)
assert.Contains(t, err.Error(), "encryption key file is required")
}
-
-// TestStatusReportsStoppedWithoutPidFile covers the resting state: no pid file
-// under BasePath means nothing is running, not an error.
-func TestStatusReportsStoppedWithoutPidFile(t *testing.T) {
- svc, err := New(&Config{BasePath: t.TempDir()})
- require.NoError(t, err)
-
- status, err := svc.Status()
-
- require.NoError(t, err)
- assert.Equal(t, "stopped", status)
-}
-
-// TestStatusAndStopUseThePidFile covers the out-of-process path: Stop and
-// Status find a predastore started by an earlier invocation through the pid
-// file BasePath holds, and Stop clears it once the process is gone.
-func TestStatusAndStopUseThePidFile(t *testing.T) {
- basePath := t.TempDir()
- svc, err := New(&Config{BasePath: basePath})
- require.NoError(t, err)
-
- child := exec.CommandContext(t.Context(), "sleep", "60")
- require.NoError(t, child.Start())
- // Reaped as it dies, not at cleanup: an unwaited child of this process
- // lingers as a zombie, which still answers signal 0, so Stop would wait out
- // its whole grace period before escalating to SIGKILL.
- reaped := make(chan struct{})
- go func() {
- defer close(reaped)
- _ = child.Wait()
- }()
- t.Cleanup(func() { <-reaped })
- require.NoError(t, utils.WritePidFileTo(basePath, serviceName, child.Process.Pid))
-
- status, err := svc.Status()
- require.NoError(t, err)
- assert.Equal(t, fmt.Sprintf("running (pid: %d)", child.Process.Pid), status)
-
- require.NoError(t, svc.Stop())
- assert.NoFileExists(t, filepath.Join(basePath, serviceName+".pid"))
-
- status, err = svc.Status()
- require.NoError(t, err)
- assert.Equal(t, "stopped", status)
-}
-
-// TestShutdownWithoutRunningClusterFallsBackToPidFile covers the fallback: with
-// no cluster in this process there is no context to cancel, so Shutdown goes
-// through the pid file, which reports rather than signalling anything.
-func TestShutdownWithoutRunningClusterFallsBackToPidFile(t *testing.T) {
- svc, err := New(&Config{BasePath: t.TempDir()})
- require.NoError(t, err)
-
- assert.Error(t, svc.Shutdown())
-}
-
-// TestShutdownCancelsTheServiceContext covers the in-process path: once Start
-// has published its cancel func, Shutdown cancels the context the gate and the
-// local nodes share instead of signalling through the pid file.
-func TestShutdownCancelsTheServiceContext(t *testing.T) {
- svc, err := New(&Config{BasePath: t.TempDir()})
- require.NoError(t, err)
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
- svc.mu.Lock()
- svc.stop = cancel
- svc.mu.Unlock()
-
- require.NoError(t, svc.Shutdown())
- assert.Error(t, ctx.Err())
-}
diff --git a/spinifex/services/viperblockd/nbdkit_recovery.go b/spinifex/services/viperblockd/nbdkit_recovery.go
index 6291bc606..4fbda8fec 100644
--- a/spinifex/services/viperblockd/nbdkit_recovery.go
+++ b/spinifex/services/viperblockd/nbdkit_recovery.go
@@ -373,12 +373,9 @@ func cmdlineReferencesEndpoint(cmdline []byte, disc discoveredNbdkit) bool {
if disc.Socket != "" {
needle := []byte(disc.Socket)
- for _, arg := range args {
- if bytes.Contains(arg, needle) {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(args, func(arg []byte) bool {
+ return bytes.Contains(arg, needle)
+ })
}
if disc.Port <= 0 {
diff --git a/spinifex/services/viperblockd/provider_handlers.go b/spinifex/services/viperblockd/provider_handlers.go
index da4d3e8d9..a174c4766 100644
--- a/spinifex/services/viperblockd/provider_handlers.go
+++ b/spinifex/services/viperblockd/provider_handlers.go
@@ -48,7 +48,7 @@ var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore {
return objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(cfg.S3Host), cfg.Region, cfg.AccessKey, cfg.SecretKey)
}
-// registerProviderSubjects subscribes the ebs.provider.v1.* handlers that
+// RegisterProviderSubjects subscribes the ebs.provider.v1.* handlers that
// serve the ebsprovider.EBSProvider NATS contract from this daemon, moving
// viperblock engine construction out of the EC2 control-plane handlers and
// into the storage daemon that owns BaseDir and the mounted-volume registry.
@@ -57,19 +57,13 @@ var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore {
// / UnpublishSubject already route to one node), so they are only registered
// when cfg.NodeName is set; there is no queue-group fallback the way the
// legacy ebs.mount/ebs.unmount subjects have.
-// RegisterProviderSubjects serves the provider contract from cfg on nc without
-// launching the rest of the daemon. It exists for harnesses that need a real
-// provider behind the control plane, which otherwise has none to call.
+// Exported so harnesses can put a real provider behind the control plane.
func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
- return registerProviderSubjects(cfg, nc)
-}
-
-func registerProviderSubjects(cfg *Config, nc *nats.Conn) error {
// The lease store has to exist before any subject is served: a handler
// that reaches an engine open without one refuses, and refusing every
// publish is a worse failure than not starting.
if cfg.leases == nil {
- leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas)
+ leases, err := newVolumeLeasesWaiting(context.Background(), nc, cfg.leaseOwner())
if err != nil {
return fmt.Errorf("volume leases: %w", err)
}
@@ -80,7 +74,7 @@ func registerProviderSubjects(cfg *Config, nc *nats.Conn) error {
// Same reasoning as the lease store: a mount that cannot consult the dirty
// marker cannot tell a stale cross-node start from a routine one.
if cfg.dirty == nil {
- dirty, err := newVolumeDirty(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas)
+ dirty, err := newVolumeDirtyWaiting(context.Background(), nc, cfg.leaseOwner())
if err != nil {
return fmt.Errorf("volume dirty markers: %w", err)
}
@@ -1736,7 +1730,7 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (types.E
// handlePublishVolume serves ebs.provider.v1..mount, the provider-neutral
// front for the same nbdkit mount path ebs.mount uses. It is node-addressed
-// (registerProviderSubjects only subscribes it when cfg.NodeName is set), so
+// (RegisterProviderSubjects only subscribes it when cfg.NodeName is set), so
// unlike the legacy handler there is no queue-group fallback to reason about.
func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
var req ebsprovider.PublishVolumeRequest
diff --git a/spinifex/services/viperblockd/provider_handlers_test.go b/spinifex/services/viperblockd/provider_handlers_test.go
index c0724f568..0ba219501 100644
--- a/spinifex/services/viperblockd/provider_handlers_test.go
+++ b/spinifex/services/viperblockd/provider_handlers_test.go
@@ -1,6 +1,6 @@
package viperblockd
-// Tests for the ebs.provider.v1.* handlers registered by registerProviderSubjects.
+// Tests for the ebs.provider.v1.* handlers registered by RegisterProviderSubjects.
// Engine-backed cases (CreateVolume idempotency, ExpandVolume in-use, CreateSnapshot)
// use a live mounted VB on the file backend (createTestVBWithState from
// viperblockd_handlers_test.go), matching this package's existing convention for
@@ -81,7 +81,7 @@ type versionedErrorResponse struct {
Error *ebsprovider.ProviderError `json:"error,omitempty"`
}
-// startProviderSubjects wires registerProviderSubjects onto a fresh NATS
+// startProviderSubjects wires RegisterProviderSubjects onto a fresh NATS
// connection for cfg, without launching the rest of launchService (mount,
// unmount, sync, config, delete), which are unrelated to the ebs.provider.v1.*
// contract under test here.
@@ -90,7 +90,7 @@ func startProviderSubjects(t *testing.T, cfg *Config, natsURL string) *nats.Conn
nc, err := nats.Connect(natsURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
- require.NoError(t, registerProviderSubjects(cfg, nc))
+ require.NoError(t, RegisterProviderSubjects(cfg, nc))
return nc
}
@@ -943,7 +943,7 @@ func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
}
// TestProviderHandlers_PublishUnpublish_RequireNodeName covers
-// registerProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must
+// RegisterProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must
// not be registered when cfg.NodeName is empty (PublishSubject has no node to
// address), while every other ebs.provider.v1.* subject still is.
func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go
index 729f80f53..1dd7feaa4 100644
--- a/spinifex/services/viperblockd/viperblockd.go
+++ b/spinifex/services/viperblockd/viperblockd.go
@@ -143,11 +143,6 @@ type Config struct {
// If empty, falls back to generic ebs.mount / ebs.unmount with queue group (single-node compat).
NodeName string
- // KVReplicas is the replica count for the volume lease and dirty buckets,
- // which is the cluster's node count. 0 means one, for a single node and for
- // tests.
- KVReplicas int
-
// NBDTransport controls the transport type: "socket" (default) or "tcp"
// Socket is faster for local connections, TCP required for remote/DPU scenarios
NBDTransport NBDTransport
@@ -681,22 +676,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-func (svc *Service) Stop() (err error) {
- return utils.StopProcessAt(svc.Config.BaseDir, serviceName)
-}
-
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BaseDir, serviceName)
-}
-
-func (svc *Service) Shutdown() (err error) {
- return svc.Stop()
-}
-
-func (svc *Service) Reload() (err error) {
- return nil
-}
-
func launchService(cfg *Config) (err error) {
nc, err := utils.ConnectNATSWithRetry(netaddr.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert)
if err != nil {
@@ -721,7 +700,7 @@ func launchService(cfg *Config) (err error) {
// Bound before recovery, which opens engines: without the store every
// engine open refuses, and the daemon would come up unable to adopt the
// exports that outlived it.
- leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas)
+ leases, err := newVolumeLeasesWaiting(context.Background(), nc, cfg.leaseOwner())
if err != nil {
return fmt.Errorf("volume leases: %w", err)
}
@@ -1068,7 +1047,7 @@ func launchService(cfg *Config) (err error) {
return fmt.Errorf("failed to subscribe to %s: %w", mountTopic, err)
}
- if err := registerProviderSubjects(cfg, nc); err != nil {
+ if err := RegisterProviderSubjects(cfg, nc); err != nil {
return err
}
diff --git a/spinifex/services/viperblockd/viperblockd_integration_test.go b/spinifex/services/viperblockd/viperblockd_integration_test.go
index ccd0dc951..1761e3d62 100644
--- a/spinifex/services/viperblockd/viperblockd_integration_test.go
+++ b/spinifex/services/viperblockd/viperblockd_integration_test.go
@@ -17,6 +17,7 @@ import (
"testing"
"time"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
"github.com/mulgadc/spinifex/spinifex/types"
"github.com/nats-io/nats-server/v2/server"
natstest "github.com/nats-io/nats-server/v2/test"
@@ -42,6 +43,9 @@ func setupEmbeddedNATS(t *testing.T) (*server.Server, string) {
}
t.Cleanup(ns.Shutdown)
+ // One embedded server is a one-node cluster, and bucket creation refuses to
+ // guess a replica count it was never told.
+ clustersize.DeclareForTest(t, 1)
return ns, ns.ClientURL()
}
@@ -54,7 +58,7 @@ func installTestVolumeLeases(t *testing.T, cfg *Config, natsURL string) {
require.NoError(t, err)
t.Cleanup(nc.Close)
- leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner(), 1)
+ leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner())
require.NoError(t, err)
cfg.leases = leases
}
@@ -84,7 +88,7 @@ func setupTestConfig(t *testing.T, natsURL string) *Config {
cfg := &Config{
NatsHost: natsURL,
- S3Host: "http://127.0.0.1:1",
+ S3Host: fastFailingS3Host(t),
Bucket: "test-bucket",
Region: "us-east-1",
AccessKey: "test-access-key",
diff --git a/spinifex/services/viperblockd/viperblockd_test.go b/spinifex/services/viperblockd/viperblockd_test.go
index 554919914..cdc166969 100644
--- a/spinifex/services/viperblockd/viperblockd_test.go
+++ b/spinifex/services/viperblockd/viperblockd_test.go
@@ -53,60 +53,6 @@ func TestNewWithNilConfig(t *testing.T) {
assert.NotNil(t, svc.Config)
}
-// TestServiceMethods tests the service interface methods.
-func TestServiceMethods(t *testing.T) {
- cfg := &Config{
- NatsHost: "nats://localhost:4222",
- BaseDir: "/tmp/viperblock",
- }
-
- svc, err := New(cfg)
- assert.NoError(t, err)
-
- // Test Status - returns "stopped" when no PID file exists
- status, err := svc.Status()
- assert.NoError(t, err)
- assert.Equal(t, "stopped", status)
-
- // Test Reload - returns no error
- err = svc.Reload()
- assert.NoError(t, err)
-}
-
-// TestServiceShutdown tests that Shutdown calls Stop.
-func TestServiceShutdown(t *testing.T) {
- cfg := &Config{
- NatsHost: "nats://localhost:4222",
- BaseDir: "/tmp/viperblock",
- }
-
- svc, err := New(cfg)
- assert.NoError(t, err)
-
- // Shutdown should call Stop internally
- // Since we can't actually start the service (requires NATS),
- // we just verify it doesn't panic
- err = svc.Shutdown()
- // Will error because no PID file exists, which is expected
- assert.Error(t, err)
-}
-
-// TestServiceStop tests the Stop method.
-func TestServiceStop(t *testing.T) {
- cfg := &Config{
- NatsHost: "nats://localhost:4222",
- BaseDir: "/tmp/viperblock",
- }
-
- svc, err := New(cfg)
- assert.NoError(t, err)
-
- // Stop should try to read PID file and stop the process
- err = svc.Stop()
- // Will error because no PID file exists, which is expected in tests
- assert.Error(t, err)
-}
-
// TestServiceNameConstant tests the serviceName constant.
func TestServiceNameConstant(t *testing.T) {
assert.Equal(t, "viperblock", serviceName)
diff --git a/spinifex/services/viperblockd/volume_bucket_replicas_test.go b/spinifex/services/viperblockd/volume_bucket_replicas_test.go
index ec207fff7..5acbb5de7 100644
--- a/spinifex/services/viperblockd/volume_bucket_replicas_test.go
+++ b/spinifex/services/viperblockd/volume_bucket_replicas_test.go
@@ -1,11 +1,12 @@
package viperblockd
//test:in-package — both bucket constructors are unexported, and the replica
-//count they are given has no exported surface to read it back from.
+//count they are created with has no exported surface to read it back from.
import (
"testing"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
@@ -15,28 +16,28 @@ import (
// The lease bucket decides which node may write a volume, and the dirty bucket
// is read on every mount. Created on one replica, both live on one node, and a
// cluster that loses that node can neither fence nor mount anywhere — which is
-// the failure the whole recovery path is built to survive. So the replica count
-// has to reach NATS, and these assert that it does in both directions: a count
-// the server cannot satisfy fails loudly rather than quietly creating one
-// replica, and the count asked for is the count the bucket is created with.
-func TestVolumeBuckets_AreCreatedWithTheReplicaCountTheyAreGiven(t *testing.T) {
+// the failure the whole recovery path is built to survive. So the cluster's node
+// count has to reach NATS, and these assert that it does in both directions: a
+// count the server cannot satisfy fails loudly rather than quietly creating one
+// replica, and the cluster's count is the count the bucket is created with.
+func TestVolumeBuckets_AreCreatedAtTheClusterReplicaCount(t *testing.T) {
_, natsURL := setupEmbeddedNATS(t)
nc, err := nats.Connect(natsURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
- leases, err := newVolumeLeases(t.Context(), nc, "node-a", 1)
+ leases, err := newVolumeLeases(t.Context(), nc, "node-a")
require.NoError(t, err)
assert.Equal(t, 1, bucketReplicas(t, leases.kv))
- dirty, err := newVolumeDirty(t.Context(), nc, "node-a", 1)
+ dirty, err := newVolumeDirty(t.Context(), nc, "node-a")
require.NoError(t, err)
assert.Equal(t, 1, bucketReplicas(t, dirty.kv))
}
-// A single embedded server cannot hold three replicas, so asking for them is
-// how a test proves the number is not being dropped on the way.
+// A single embedded server cannot hold three replicas, so declaring a
+// three-node cluster is how a test proves the number is not dropped on the way.
func TestVolumeBuckets_RefuseAReplicaCountTheServerCannotHold(t *testing.T) {
_, natsURL := setupEmbeddedNATS(t)
@@ -44,13 +45,34 @@ func TestVolumeBuckets_RefuseAReplicaCountTheServerCannotHold(t *testing.T) {
require.NoError(t, err)
t.Cleanup(nc.Close)
- _, err = newVolumeLeases(t.Context(), nc, "node-a", 3)
+ clustersize.RedeclareForTest(t, 3)
+
+ _, err = newVolumeLeases(t.Context(), nc, "node-a")
require.Error(t, err, "a lease bucket that cannot be made quorate must fail the service, not serve unreplicated")
- _, err = newVolumeDirty(t.Context(), nc, "node-a", 3)
+ _, err = newVolumeDirty(t.Context(), nc, "node-a")
require.Error(t, err, "a dirty bucket that cannot be made quorate must fail the service, not serve unreplicated")
}
+// TestVolumeBuckets_RefuseAnUndeclaredClusterSize pins the fail-closed choice at
+// the call site that matters most: a process that never loaded a cluster config
+// must not create these buckets on one node by default.
+func TestVolumeBuckets_RefuseAnUndeclaredClusterSize(t *testing.T) {
+ _, natsURL := setupEmbeddedNATS(t)
+
+ nc, err := nats.Connect(natsURL)
+ require.NoError(t, err)
+ t.Cleanup(nc.Close)
+
+ clustersize.RedeclareForTest(t, 0)
+
+ _, err = newVolumeLeases(t.Context(), nc, "node-a")
+ require.Error(t, err)
+
+ _, err = newVolumeDirty(t.Context(), nc, "node-a")
+ require.Error(t, err)
+}
+
func bucketReplicas(t *testing.T, kv jetstream.KeyValue) int {
t.Helper()
diff --git a/spinifex/services/viperblockd/volume_bucket_startup_test.go b/spinifex/services/viperblockd/volume_bucket_startup_test.go
new file mode 100644
index 000000000..7f9bc4f52
--- /dev/null
+++ b/spinifex/services/viperblockd/volume_bucket_startup_test.go
@@ -0,0 +1,79 @@
+//test:in-package — the start-path bucket openers and the constructors beneath
+//them are unexported, and the point is which of the two waits.
+
+package viperblockd
+
+import (
+ "testing"
+ "time"
+
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
+ "github.com/nats-io/nats-server/v2/server"
+ natstest "github.com/nats-io/nats-server/v2/test"
+ "github.com/nats-io/nats.go"
+ "github.com/stretchr/testify/require"
+)
+
+// jetStreamArrivesAfter is how long the buckets are left failing before
+// JetStream appears. Only long enough to prove the first refusal did not end
+// the attempt; the retry interval decides how much longer the open then takes.
+const jetStreamArrivesAfter = 250 * time.Millisecond
+
+// TestVolumeBucketsWaitForJetStream is the cold multi-node start. Every service
+// comes up at once against a cluster with no stream leader yet, and both of
+// these opens are on the path to the daemon running at all — so a refusal that
+// only means "not yet" must not become "this node has no block storage".
+func TestVolumeBucketsWaitForJetStream(t *testing.T) {
+ // Deliberately no JetStream: the server answers, it just has nowhere to put
+ // a bucket, which is what a node whose cluster has not formed looks like.
+ ns := natstest.RunServer(&server.Options{Host: "127.0.0.1", Port: -1})
+ require.NotNil(t, ns)
+ t.Cleanup(ns.Shutdown)
+ clustersize.DeclareForTest(t, 1)
+
+ nc, err := nats.Connect(ns.ClientURL())
+ require.NoError(t, err)
+ t.Cleanup(nc.Close)
+
+ leaseErr := make(chan error, 1)
+ dirtyErr := make(chan error, 1)
+ go func() { _, err := newVolumeLeasesWaiting(t.Context(), nc, "node-a"); leaseErr <- err }()
+ go func() { _, err := newVolumeDirtyWaiting(t.Context(), nc, "node-a"); dirtyErr <- err }()
+
+ select {
+ case err := <-leaseErr:
+ t.Fatalf("lease bucket concluded before JetStream existed: %v", err)
+ case err := <-dirtyErr:
+ t.Fatalf("dirty bucket concluded before JetStream existed: %v", err)
+ case <-time.After(jetStreamArrivesAfter):
+ }
+
+ require.NoError(t, ns.EnableJetStream(&server.JetStreamConfig{StoreDir: t.TempDir()}))
+
+ require.NoError(t, <-leaseErr, "the lease bucket must open once JetStream is there")
+ require.NoError(t, <-dirtyErr, "the dirty bucket must open once JetStream is there")
+}
+
+// TestVolumeBucketsUnwaitingOpenStillRefusesAtOnce keeps the wait on the start
+// path alone. The constructors are what the replica-count and cluster-size
+// tests assert fail closed, and a refusal they have to wait out is one an
+// operator tool and those tests would both read as a hang.
+func TestVolumeBucketsUnwaitingOpenStillRefusesAtOnce(t *testing.T) {
+ ns := natstest.RunServer(&server.Options{Host: "127.0.0.1", Port: -1})
+ require.NotNil(t, ns)
+ t.Cleanup(ns.Shutdown)
+ clustersize.DeclareForTest(t, 1)
+
+ nc, err := nats.Connect(ns.ClientURL())
+ require.NoError(t, err)
+ t.Cleanup(nc.Close)
+
+ started := time.Now()
+ _, leaseOpenErr := newVolumeLeases(t.Context(), nc, "node-a")
+ _, dirtyOpenErr := newVolumeDirty(t.Context(), nc, "node-a")
+
+ require.Error(t, leaseOpenErr)
+ require.Error(t, dirtyOpenErr)
+ require.Less(t, time.Since(started), jetStreamArrivesAfter,
+ "the plain constructors must report a refusal rather than retry it")
+}
diff --git a/spinifex/services/viperblockd/volume_dirty.go b/spinifex/services/viperblockd/volume_dirty.go
index daae1e81a..308ccc7bc 100644
--- a/spinifex/services/viperblockd/volume_dirty.go
+++ b/spinifex/services/viperblockd/volume_dirty.go
@@ -8,7 +8,9 @@ import (
"log/slog"
"time"
- "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
"github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -26,22 +28,18 @@ type volumeDirty struct {
}
// newVolumeDirty binds the dirty bucket, creating it if this is the first node
-// up. owner identifies this node in the entries it writes, and replicas is the
-// cluster size: this is read on the mount path, so a single replica would make
-// every mount depend on one node being up.
-func newVolumeDirty(ctx context.Context, nc *nats.Conn, owner string, replicas int) (*volumeDirty, error) {
+// up. owner identifies this node in the entries it writes. This is read on the
+// mount path, so a single replica would make every mount in the cluster depend
+// on one node being up.
+func newVolumeDirty(ctx context.Context, nc *nats.Conn, owner string) (*volumeDirty, error) {
js, err := jetstream.New(nc)
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- // Create-or-update rather than create: a bucket left behind by a build that
- // made it single-replica is raised here, and a cluster cannot be asked to
- // lose its markers to be repaired.
- kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: vbwire.DirtyBucket,
+ kv, err := kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+ Name: vbwire.DirtyBucket,
Description: "volumes whose last seal failed, keyed by volume, naming the node holding the current copy",
History: 1,
- Replicas: max(replicas, 1),
})
if err != nil {
return nil, fmt.Errorf("volume dirty bucket: %w", err)
@@ -49,6 +47,14 @@ func newVolumeDirty(ctx context.Context, nc *nats.Conn, owner string, replicas i
return &volumeDirty{kv: kv, owner: owner}, nil
}
+// newVolumeDirtyWaiting is newVolumeDirty for a daemon that is starting, for the
+// same reason as newVolumeLeasesWaiting: a cold cluster has no stream leader
+// yet, and one refused open must not decide that this node has no storage.
+func newVolumeDirtyWaiting(ctx context.Context, nc *nats.Conn, owner string) (*volumeDirty, error) {
+ return kvstore.OpenWithRetry(ctx, vbwire.DirtyBucket, kvstore.DefaultOpenWindow,
+ func(ctx context.Context) (*volumeDirty, error) { return newVolumeDirty(ctx, nc, owner) })
+}
+
// mark records that this node holds writes for volumeName at generation.
// Conditional on the existing entry: a marker written by a later generation
// belongs to a node that has taken the volume over, and overwriting it would
@@ -234,7 +240,7 @@ func (cfg *Config) reportVolumeTakeover(ctx context.Context, volumeName string,
slog.WarnContext(ctx, "opening a volume whose writes were last held by another node, from the backend checkpoint instead",
"volume", volumeName, "previous_owner", record.Owner,
"unsealed_since", record.Since.Format(time.RFC3339), "previous_reason", record.Reason)
- otelsetup.RecordVolumeTakeover(ctx)
+ telemetry.RecordVolumeTakeover(ctx)
// Take the marker over. The previous owner's copy is now behind this one,
// so if that node returns it must not be treated as the better source.
diff --git a/spinifex/services/viperblockd/volume_dirty_test.go b/spinifex/services/viperblockd/volume_dirty_test.go
index 12fee61d7..546697e02 100644
--- a/spinifex/services/viperblockd/volume_dirty_test.go
+++ b/spinifex/services/viperblockd/volume_dirty_test.go
@@ -20,7 +20,7 @@ func newTestDirty(t *testing.T, natsURL, owner string) *volumeDirty {
require.NoError(t, err)
t.Cleanup(nc.Close)
- dirty, err := newVolumeDirty(t.Context(), nc, owner, 1)
+ dirty, err := newVolumeDirty(t.Context(), nc, owner)
require.NoError(t, err)
return dirty
}
diff --git a/spinifex/services/viperblockd/volume_lease.go b/spinifex/services/viperblockd/volume_lease.go
index bb9ade544..e712d317a 100644
--- a/spinifex/services/viperblockd/volume_lease.go
+++ b/spinifex/services/viperblockd/volume_lease.go
@@ -11,6 +11,8 @@ import (
"sync"
"time"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
@@ -127,24 +129,20 @@ type volumeLeases struct {
held map[string]*volumeLease
}
-// newVolumeLeases binds the lease bucket, creating it if this is the first
-// node up. owner identifies this node in the entries it writes, and replicas is
-// the cluster size: this bucket decides who may write a volume, so a single
-// replica would put every volume in the cluster behind one node staying up.
-func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string, replicas int) (*volumeLeases, error) {
+// newVolumeLeases binds the lease bucket, creating it if this is the first node
+// up. owner identifies this node in the entries it writes. This bucket decides
+// who may write a volume, so a single replica would put every volume in the
+// cluster behind one node staying up.
+func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string) (*volumeLeases, error) {
js, err := jetstream.New(nc)
if err != nil {
return nil, fmt.Errorf("jetstream: %w", err)
}
- // Create-or-update rather than create: a bucket left behind by a build that
- // made it single-replica is raised here, and a cluster cannot be asked to
- // lose its leases to be repaired.
- kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
- Bucket: volumeLeaseBucket,
+ kv, err := kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+ Name: volumeLeaseBucket,
Description: "one entry per volume with a viperblock engine open on it",
TTL: volumeLeaseTTL,
History: 1,
- Replicas: max(replicas, 1),
})
if err != nil {
return nil, fmt.Errorf("volume lease bucket: %w", err)
@@ -152,6 +150,14 @@ func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string, replicas
return &volumeLeases{kv: kv, owner: owner, held: make(map[string]*volumeLease)}, nil
}
+// newVolumeLeasesWaiting is newVolumeLeases for a daemon that is starting. On a
+// cold multi-node start every service races JetStream electing a leader, and a
+// refusal that only means "not yet" must not leave this node without storage.
+func newVolumeLeasesWaiting(ctx context.Context, nc *nats.Conn, owner string) (*volumeLeases, error) {
+ return kvstore.OpenWithRetry(ctx, volumeLeaseBucket, kvstore.DefaultOpenWindow,
+ func(ctx context.Context) (*volumeLeases, error) { return newVolumeLeases(ctx, nc, owner) })
+}
+
// volumeLease is one held lease and the goroutine keeping it alive.
type volumeLease struct {
leases *volumeLeases
diff --git a/spinifex/services/viperblockd/volume_lease_test.go b/spinifex/services/viperblockd/volume_lease_test.go
index e68f14f8a..823e6bb21 100644
--- a/spinifex/services/viperblockd/volume_lease_test.go
+++ b/spinifex/services/viperblockd/volume_lease_test.go
@@ -24,7 +24,7 @@ func newTestLeases(t *testing.T, natsURL, owner string) *volumeLeases {
require.NoError(t, err)
t.Cleanup(nc.Close)
- leases, err := newVolumeLeases(t.Context(), nc, owner, 1)
+ leases, err := newVolumeLeases(t.Context(), nc, owner)
require.NoError(t, err)
return leases
}
@@ -561,7 +561,7 @@ func TestVolumeLease_UnreachableStoreIsRetryable(t *testing.T) {
require.NoError(t, err)
t.Cleanup(nc.Close)
- leases, err := newVolumeLeases(t.Context(), nc, "node-a", 1)
+ leases, err := newVolumeLeases(t.Context(), nc, "node-a")
require.NoError(t, err)
// Close the connection under the bound store, which is the shape of a
diff --git a/spinifex/utils/images.go b/spinifex/utils/images.go
index 7d629b792..7d6b16868 100644
--- a/spinifex/utils/images.go
+++ b/spinifex/utils/images.go
@@ -24,6 +24,7 @@ import (
"os/signal"
"path/filepath"
"runtime"
+ "slices"
"strings"
"time"
@@ -349,12 +350,9 @@ func SelectNewestImage(images []*ec2.Image, excludeTagKey string) (imageID, crea
// imageHasTagKey reports whether img carries a tag with the given key.
func imageHasTagKey(img *ec2.Image, key string) bool {
- for _, t := range img.Tags {
- if t != nil && aws.StringValue(t.Key) == key {
- return true
- }
- }
- return false
+ return slices.ContainsFunc(img.Tags, func(t *ec2.Tag) bool {
+ return t != nil && aws.StringValue(t.Key) == key
+ })
}
type Images struct {
@@ -591,28 +589,6 @@ var AvailableImages = map[string]Images{
Tags: map[string]string{"gpu-vendor": "nvidia"},
},
- // Bedrock self-host serving AMI. Resolved by spinifex:managed-by=bedrock +
- // spinifex:bedrock-role=vllm-serving tags — LaunchServingVM's
- // resolveServingAMI filters on both, never by name.
- "ubuntu-26.04-vllm-serving-x86_64": {
- Name: "ubuntu-26.04-vllm-serving-x86_64",
- Description: "Ubuntu 26.04 Bedrock co-served bundle image — NVIDIA GPU base + vLLM and TEI baked side by side (uv-managed venv, TEI staged from HuggingFace's CUDA image), one bedrock-serve@ instance per bundle member against its own read-only weights mount",
- Distro: "ubuntu",
- Version: "26.04",
- Arch: "x86_64",
- Platform: "Linux/UNIX",
- CreatedAt: time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC),
- URL: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2",
- Checksum: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2.sha256",
- ChecksumType: "sha256",
- BootMode: "uefi",
- Tags: map[string]string{
- "spinifex:managed-by": "bedrock",
- "spinifex:bedrock-role": "vllm-serving",
- "gpu-vendor": "nvidia",
- },
- },
-
"ubuntu-26.04-amd-gpu-x86_64": {
Name: "ubuntu-26.04-amd-gpu-x86_64",
Description: "Ubuntu 26.04 AMD GPU base image — linux-firmware, ROCm CLI, Python toolchain, Docker",
@@ -742,6 +718,35 @@ var AvailableImages = map[string]Images{
},
}
+// WithheldImages are catalog entries whose artifact is not published, so an
+// import of one would fail for every operator. They are kept here rather than
+// deleted because the entry is the definition: moving one back into
+// AvailableImages is the whole of publishing it.
+var WithheldImages = map[string]Images{
+
+ // Bedrock self-host serving AMI. Resolved by spinifex:managed-by=bedrock +
+ // spinifex:bedrock-role=vllm-serving tags — LaunchServingVM's
+ // resolveServingAMI filters on both, never by name.
+ "ubuntu-26.04-vllm-serving-x86_64": {
+ Name: "ubuntu-26.04-vllm-serving-x86_64",
+ Description: "Ubuntu 26.04 Bedrock co-served bundle image — NVIDIA GPU base + vLLM and TEI baked side by side (uv-managed venv, TEI staged from HuggingFace's CUDA image), one bedrock-serve@ instance per bundle member against its own read-only weights mount",
+ Distro: "ubuntu",
+ Version: "26.04",
+ Arch: "x86_64",
+ Platform: "Linux/UNIX",
+ CreatedAt: time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC),
+ URL: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2",
+ Checksum: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2.sha256",
+ ChecksumType: "sha256",
+ BootMode: "uefi",
+ Tags: map[string]string{
+ "spinifex:managed-by": "bedrock",
+ "spinifex:bedrock-role": "vllm-serving",
+ "gpu-vendor": "nvidia",
+ },
+ },
+}
+
// ExtractDiskImageFromFile turns a downloaded artifact at imagepath into a bootable disk
// image and returns its path. tar-based archives are expanded into tmpdir and their listing
// is scanned for the member that carries the payload; single-file compressed artifacts
diff --git a/spinifex/utils/images_test.go b/spinifex/utils/images_test.go
index eb3807b01..027a29c34 100644
--- a/spinifex/utils/images_test.go
+++ b/spinifex/utils/images_test.go
@@ -21,12 +21,19 @@ import (
// resolveServingAMI filters purely on these two tags and never on the image
// name, so a typo in either hides the serving image from Ochre and every
// endpoint launch dies at "bedrock: no vllm-serving AMI found".
-func TestAvailableImages_VLLMServingCarriesBedrockTags(t *testing.T) {
+//
+// The entry is withheld while its artifact is unpublished, so the contract is
+// checked where the definition lives rather than dropped along with it.
+func TestWithheldImages_VLLMServingCarriesBedrockTags(t *testing.T) {
const key = "ubuntu-26.04-vllm-serving-x86_64"
- img, ok := AvailableImages[key]
+ if _, offered := AvailableImages[key]; offered {
+ t.Fatalf("%s is offered for import while its artifact is unpublished", key)
+ }
+
+ img, ok := WithheldImages[key]
if !ok {
- t.Fatalf("%s missing from AvailableImages", key)
+ t.Fatalf("%s missing from WithheldImages", key)
}
for tag, want := range map[string]string{
diff --git a/spinifex/utils/nats_trace.go b/spinifex/utils/nats_trace.go
index 1401ff137..174772da6 100644
--- a/spinifex/utils/nats_trace.go
+++ b/spinifex/utils/nats_trace.go
@@ -2,6 +2,8 @@ package utils
import (
"context"
+ "maps"
+ "slices"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
@@ -22,11 +24,7 @@ type natsHeaderCarrier nats.Header
func (c natsHeaderCarrier) Get(key string) string { return nats.Header(c).Get(key) }
func (c natsHeaderCarrier) Set(key, value string) { nats.Header(c).Set(key, value) }
func (c natsHeaderCarrier) Keys() []string {
- keys := make([]string, 0, len(c))
- for k := range c {
- keys = append(keys, k)
- }
- return keys
+ return slices.Collect(maps.Keys(c))
}
// InjectTraceContext writes ctx's span context into hdr (traceparent/tracestate).
diff --git a/spinifex/utils/nbd.go b/spinifex/utils/nbd.go
index c8aea0b59..0aa658d20 100644
--- a/spinifex/utils/nbd.go
+++ b/spinifex/utils/nbd.go
@@ -18,7 +18,7 @@ func GenerateSocketFile(name string) (string, error) {
return "", errors.New("name is required")
}
- pidPath := pidPath()
+ pidPath := RuntimeDir()
if pidPath == "" {
return "", errors.New("pid path is empty")
@@ -43,13 +43,13 @@ func GenerateUniqueSocketFile(volname string) (string, error) {
return filepath.Join(dir, filename), nil
}
-// NBDSocketDir returns the NBD socket directory (/run/spinifex/nbd under systemd; pidPath() as fallback).
+// NBDSocketDir returns the NBD socket directory (/run/spinifex/nbd under systemd; RuntimeDir() as fallback).
func NBDSocketDir() string {
const systemdNBDDir = "/run/spinifex/nbd"
if dirExists(systemdNBDDir) {
return systemdNBDDir
}
- return pidPath()
+ return RuntimeDir()
}
// IsSocketURI reports whether the NBD URI refers to a Unix socket (ends with ".sock" or contains "unix:").
diff --git a/spinifex/utils/pidfile.go b/spinifex/utils/pidfile.go
index 3b00ea1ad..4f52516a5 100644
--- a/spinifex/utils/pidfile.go
+++ b/spinifex/utils/pidfile.go
@@ -12,7 +12,7 @@ import (
)
func ReadPidFile(name string) (int, error) {
- pidPath := pidPath()
+ pidPath := RuntimeDir()
pidFile, err := os.ReadFile(filepath.Join(pidPath, fmt.Sprintf("%s.pid", name)))
@@ -30,7 +30,7 @@ func GeneratePidFile(name string) (string, error) {
return "", errors.New("name is required")
}
- pidPath := pidPath()
+ pidPath := RuntimeDir()
if pidPath == "" {
return "", errors.New("pid path is empty")
@@ -61,7 +61,7 @@ func WritePidFile(name string, pid int) error {
return nil
}
-// WritePidFileTo writes a PID file to dir (or pidPath() if empty).
+// WritePidFileTo writes a PID file to dir (or RuntimeDir() if empty).
// Per-service data directories prevent PID file collisions on multi-node hosts.
func WritePidFileTo(dir string, name string, pid int) error {
if dir == "" {
@@ -85,7 +85,7 @@ func WritePidFileTo(dir string, name string, pid int) error {
}
// ReadPidFileFrom reads a PID from a file in a specific directory. If dir is
-// empty, falls back to the default pidPath().
+// empty, falls back to the default RuntimeDir().
func ReadPidFileFrom(dir string, name string) (int, error) {
if dir == "" {
return ReadPidFile(name)
@@ -101,7 +101,7 @@ func ReadPidFileFrom(dir string, name string) (int, error) {
}
// RemovePidFileAt removes a PID file from a specific directory. If dir is
-// empty, falls back to the default pidPath().
+// empty, falls back to the default RuntimeDir().
func RemovePidFileAt(dir string, name string) error {
if dir == "" {
return RemovePidFile(name)
@@ -109,19 +109,6 @@ func RemovePidFileAt(dir string, name string) error {
return os.Remove(filepath.Join(dir, fmt.Sprintf("%s.pid", name)))
}
-// ServiceStatus returns a human-readable status string for a service by
-// checking its PID file. If dir is empty, the default pidPath() is used.
-func ServiceStatus(dir, name string) (string, error) {
- pid, err := ReadPidFileFrom(dir, name)
- if err != nil {
- if os.IsNotExist(err) {
- return "stopped", nil
- }
- return "", fmt.Errorf("read pid file: %w", err)
- }
- return fmt.Sprintf("running (pid: %d)", pid), nil
-}
-
// StopProcessAt stops a process using its PID file. Always removes the PID file, even if the process is already dead.
func StopProcessAt(dir string, name string) error {
pid, err := ReadPidFileFrom(dir, name)
@@ -139,7 +126,7 @@ func StopProcessAt(dir string, name string) error {
}
func RemovePidFile(serviceName string) error {
- pidPath := pidPath()
+ pidPath := RuntimeDir()
err := os.Remove(filepath.Join(pidPath, fmt.Sprintf("%s.pid", serviceName)))
if err != nil {
@@ -149,11 +136,6 @@ func RemovePidFile(serviceName string) error {
return nil
}
-// RuntimeDir returns the runtime directory used for PID files, sockets, and logs.
-func RuntimeDir() string {
- return pidPath()
-}
-
// QMPTelemetryPrefix names the per-VM telemetry QMP socket and its sidecar
// metadata file in the runtime dir: .sock / .json.
const QMPTelemetryPrefix = "qmp-telemetry-"
@@ -163,7 +145,8 @@ func TelemetryMetaPath(instanceID string) string {
return filepath.Join(RuntimeDir(), QMPTelemetryPrefix+instanceID+".json")
}
-func pidPath() string {
+// RuntimeDir returns the runtime directory used for PID files, sockets, and logs.
+func RuntimeDir() string {
if os.Getenv("XDG_RUNTIME_DIR") != "" {
return os.Getenv("XDG_RUNTIME_DIR")
}
diff --git a/spinifex/utils/platform.go b/spinifex/utils/platform.go
index d5bf4bef7..ace613a39 100644
--- a/spinifex/utils/platform.go
+++ b/spinifex/utils/platform.go
@@ -20,3 +20,23 @@ func PlatformFromDetails(platformDetails string) *string {
p := PlatformWindows
return &p
}
+
+// usageOperations maps PlatformDetails to the billing code AWS reports as an
+// image's UsageOperation, for the platforms checked against real AWS images.
+var usageOperations = map[string]string{
+ "Linux/UNIX": "RunInstances",
+ "Red Hat Enterprise Linux": "RunInstances:0010",
+ "SUSE Linux": "RunInstances:000g",
+ "Ubuntu Pro Linux": "RunInstances:0g00",
+ "Windows": "RunInstances:0002",
+}
+
+// UsageOperationFromDetails returns the UsageOperation for platformDetails, or
+// nil for a platform whose code has not been confirmed.
+func UsageOperationFromDetails(platformDetails string) *string {
+ op, ok := usageOperations[platformDetails]
+ if !ok {
+ return nil
+ }
+ return &op
+}
diff --git a/spinifex/utils/platform_test.go b/spinifex/utils/platform_test.go
index 9f2241b79..8706f5384 100644
--- a/spinifex/utils/platform_test.go
+++ b/spinifex/utils/platform_test.go
@@ -35,3 +35,28 @@ func TestPlatformFromDetails(t *testing.T) {
})
}
}
+
+func TestUsageOperationFromDetails(t *testing.T) {
+ tests := []struct {
+ platformDetails string
+ want string
+ }{
+ {platformDetails: "Linux/UNIX", want: "RunInstances"},
+ {platformDetails: "Windows", want: "RunInstances:0002"},
+ {platformDetails: "Ubuntu Pro Linux", want: "RunInstances:0g00"},
+ {platformDetails: "Windows BYOL", want: ""},
+ {platformDetails: "", want: ""},
+ }
+ for _, tt := range tests {
+ got := UsageOperationFromDetails(tt.platformDetails)
+ if tt.want == "" {
+ if got != nil {
+ t.Errorf("UsageOperationFromDetails(%q) = %q, want nil", tt.platformDetails, *got)
+ }
+ continue
+ }
+ if got == nil || *got != tt.want {
+ t.Errorf("UsageOperationFromDetails(%q) = %v, want %q", tt.platformDetails, got, tt.want)
+ }
+ }
+}
diff --git a/spinifex/utils/process.go b/spinifex/utils/process.go
index 192d1f2dc..ade890d6b 100644
--- a/spinifex/utils/process.go
+++ b/spinifex/utils/process.go
@@ -21,25 +21,6 @@ func SetOOMScore(pid int, score int) error {
return os.WriteFile(path, []byte(strconv.Itoa(score)), 0600)
}
-func StopProcess(serviceName string) error {
- pid, err := ReadPidFile(serviceName)
- if err != nil {
- return err
- }
-
- err = KillProcess(pid)
- if err != nil {
- return err
- }
-
- err = RemovePidFile(serviceName)
- if err != nil {
- return err
- }
-
- return nil
-}
-
// ProcessAlive reports whether the process is still running, via a
// signal-0 liveness probe. A PID file going missing does NOT imply the
// process exited, so callers that must reap a process check this directly.
diff --git a/spinifex/utils/utils.go b/spinifex/utils/utils.go
index 30b81c9f8..1d77b865b 100644
--- a/spinifex/utils/utils.go
+++ b/spinifex/utils/utils.go
@@ -18,6 +18,7 @@ import (
"os"
"os/signal"
"reflect"
+ "slices"
"strings"
"time"
@@ -91,8 +92,10 @@ func GenerateXMLPayload(locationName string, payload any) any {
// NormalizeXMLOutput returns a copy of output with every nil slice field
// (recursively) replaced by a non-nil empty slice, since aws-sdk-go's
// xmlutil.BuildXML omits a nil slice's container element entirely but
-// renders an empty one for a non-nil empty slice, unlike real AWS.
-func NormalizeXMLOutput(output any) any {
+// renders an empty one for a non-nil empty slice. AWS renders most empty lists
+// but omits some; asSet names those fields, per struct type, and they are
+// left as the handler set them so a nil one stays omitted.
+func NormalizeXMLOutput(output any, asSet map[reflect.Type][]string) any {
v := reflect.ValueOf(output)
if !v.IsValid() {
return output
@@ -101,33 +104,34 @@ func NormalizeXMLOutput(output any) any {
// can only Set fields through an addressable Value.
ptr := reflect.New(v.Type())
ptr.Elem().Set(v)
- normalizeNilSlices(ptr.Elem())
+ normalizeNilSlices(ptr.Elem(), asSet)
return ptr.Elem().Interface()
}
// normalizeNilSlices walks v in place, turning nil slice fields into empty
// ones and recursing into structs, pointers, and existing slice elements.
-func normalizeNilSlices(v reflect.Value) {
+func normalizeNilSlices(v reflect.Value, asSet map[reflect.Type][]string) {
switch v.Kind() {
case reflect.Pointer:
if !v.IsNil() {
- normalizeNilSlices(v.Elem())
+ normalizeNilSlices(v.Elem(), asSet)
}
case reflect.Struct:
- for _, field := range v.Fields() {
+ keep := asSet[v.Type()]
+ for sf, field := range v.Fields() {
switch field.Kind() {
case reflect.Slice:
if field.IsNil() {
- if field.CanSet() {
+ if field.CanSet() && !slices.Contains(keep, sf.Name) {
field.Set(reflect.MakeSlice(field.Type(), 0, 0))
}
} else {
for j := 0; j < field.Len(); j++ {
- normalizeNilSlices(field.Index(j))
+ normalizeNilSlices(field.Index(j), asSet)
}
}
case reflect.Pointer, reflect.Struct:
- normalizeNilSlices(field)
+ normalizeNilSlices(field, asSet)
}
}
}
diff --git a/spinifex/utils/utils_test.go b/spinifex/utils/utils_test.go
index cd23684ef..86a5aeb17 100644
--- a/spinifex/utils/utils_test.go
+++ b/spinifex/utils/utils_test.go
@@ -162,7 +162,7 @@ func TestExecProcessAndKill(t *testing.T) {
time.Sleep(20 * time.Millisecond)
// Kill the process
- err = StopProcess("utilsunittest")
+ err = StopProcessAt("", "utilsunittest")
assert.NoError(t, err)
// Test PID file removed
@@ -722,36 +722,6 @@ func TestForceKillProcess_AlreadyExitedIsSuccess(t *testing.T) {
"an already-exited process satisfies the request, it does not fail it")
}
-func TestStopProcess(t *testing.T) {
- // Create and start a test process
- cmd := exec.Command("sleep", "60")
- err := cmd.Start()
- require.NoError(t, err)
-
- // Write PID file
- testName := "stopprocess-test"
- err = WritePidFile(testName, cmd.Process.Pid)
- require.NoError(t, err)
-
- // Reap in background so StopProcess can detect termination
- var wg sync.WaitGroup
- wg.Go(func() {
- _ = cmd.Wait()
- })
-
- err = StopProcess(testName)
- assert.NoError(t, err)
- wg.Wait()
-
- // Verify PID file was removed
- _, err = ReadPidFile(testName)
- assert.Error(t, err, "PID file should be removed")
-
- // Test stopping non-existent process
- err = StopProcess("nonexistent-process")
- assert.Error(t, err, "Should error when stopping non-existent process")
-}
-
// Test file extraction process
func TestExtractDiskImageFromFile(t *testing.T) {
@@ -994,7 +964,7 @@ func TestWritePidFileTo(t *testing.T) {
}
func TestWritePidFileTo_EmptyDir(t *testing.T) {
- // With empty dir, should fall back to default pidPath()
+ // With empty dir, should fall back to default RuntimeDir()
cmd := exec.Command("cat")
require.NoError(t, cmd.Start())
defer cmd.Process.Kill()
@@ -1115,7 +1085,7 @@ func TestRuntimeDir(t *testing.T) {
func TestPidPath_XDG(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", "/tmp/test-xdg-runtime")
- assert.Equal(t, "/tmp/test-xdg-runtime", pidPath())
+ assert.Equal(t, "/tmp/test-xdg-runtime", RuntimeDir())
}
func TestPidPath_HomeSpinifexFallback(t *testing.T) {
@@ -1126,14 +1096,14 @@ func TestPidPath_HomeSpinifexFallback(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", "")
t.Setenv("HOME", tmpHome)
- assert.Equal(t, spinifexDir, pidPath())
+ assert.Equal(t, spinifexDir, RuntimeDir())
}
func TestPidPath_TempDirFallback(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", "")
t.Setenv("HOME", "/nonexistent-home-dir-utils-test")
- assert.Equal(t, os.TempDir(), pidPath())
+ assert.Equal(t, os.TempDir(), RuntimeDir())
}
func TestExtractDiskImagePath_NoMatch(t *testing.T) {
@@ -1173,39 +1143,6 @@ func TestReadPidFileFrom_EmptyDir(t *testing.T) {
assert.Error(t, err)
}
-func TestServiceStatus_Stopped(t *testing.T) {
- dir := t.TempDir()
- status, err := ServiceStatus(dir, fmt.Sprintf("no-such-svc-%d", time.Now().UnixNano()))
- require.NoError(t, err)
- assert.Equal(t, "stopped", status)
-}
-
-func TestServiceStatus_Running(t *testing.T) {
- dir := t.TempDir()
-
- cmd := exec.Command("sleep", "60")
- require.NoError(t, cmd.Start())
- defer func() {
- _ = cmd.Process.Kill()
- _ = cmd.Wait()
- }()
-
- require.NoError(t, WritePidFileTo(dir, "running-svc", cmd.Process.Pid))
-
- status, err := ServiceStatus(dir, "running-svc")
- require.NoError(t, err)
- assert.Equal(t, fmt.Sprintf("running (pid: %d)", cmd.Process.Pid), status)
-}
-
-func TestServiceStatus_CorruptPidFile(t *testing.T) {
- dir := t.TempDir()
- // Write a non-numeric pid file — ReadPidFileFrom should fail to parse it.
- require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-svc.pid"), []byte("not-a-pid"), 0o644))
-
- _, err := ServiceStatus(dir, "bad-svc")
- assert.Error(t, err)
-}
-
func TestWaitForProcessExit_ProcessAlreadyDead(t *testing.T) {
cmd := exec.Command("true")
require.NoError(t, cmd.Start())
@@ -1891,11 +1828,11 @@ type normalizeOuter struct {
func TestNormalizeXMLOutput_InvalidValue(t *testing.T) {
var output any
- assert.Nil(t, NormalizeXMLOutput(output))
+ assert.Nil(t, NormalizeXMLOutput(output, nil))
}
func TestNormalizeXMLOutput_TopLevelNilSlice(t *testing.T) {
- out := NormalizeXMLOutput(normalizeOuter{}).(normalizeOuter)
+ out := NormalizeXMLOutput(normalizeOuter{}, nil).(normalizeOuter)
require.NotNil(t, out.Names)
assert.Empty(t, out.Names)
}
@@ -1903,28 +1840,39 @@ func TestNormalizeXMLOutput_TopLevelNilSlice(t *testing.T) {
func TestNormalizeXMLOutput_NestedPointerStruct(t *testing.T) {
out := NormalizeXMLOutput(normalizeOuter{
Nested: &normalizeInner{},
- }).(normalizeOuter)
+ }, nil).(normalizeOuter)
require.NotNil(t, out.Nested)
require.NotNil(t, out.Nested.Tags, "nil slice inside a pointer field must be normalized")
assert.Empty(t, out.Nested.Tags)
}
func TestNormalizeXMLOutput_NestedValueStruct(t *testing.T) {
- out := NormalizeXMLOutput(normalizeOuter{}).(normalizeOuter)
+ out := NormalizeXMLOutput(normalizeOuter{}, nil).(normalizeOuter)
require.NotNil(t, out.Value.Tags, "nil slice inside a nested struct field must be normalized")
}
func TestNormalizeXMLOutput_RecursesIntoSliceElements(t *testing.T) {
out := NormalizeXMLOutput(normalizeOuter{
Children: []*normalizeInner{{}, {Tags: []string{"a"}}},
- }).(normalizeOuter)
+ }, nil).(normalizeOuter)
require.Len(t, out.Children, 2)
require.NotNil(t, out.Children[0].Tags, "nil slice inside a slice element must be normalized")
assert.Equal(t, []string{"a"}, out.Children[1].Tags, "an already-populated slice element must be left untouched")
}
+func TestNormalizeXMLOutput_AsSetFieldsKeepNil(t *testing.T) {
+ asSet := map[reflect.Type][]string{reflect.TypeFor[normalizeInner](): {"Tags"}}
+ out := NormalizeXMLOutput(normalizeOuter{
+ Nested: &normalizeInner{},
+ Children: []*normalizeInner{{Tags: []string{}}},
+ }, asSet).(normalizeOuter)
+ assert.Nil(t, out.Nested.Tags, "a nil as-set field must stay nil so BuildXML omits it")
+ require.NotNil(t, out.Children[0].Tags, "an empty as-set field must stay empty so BuildXML renders it")
+ require.NotNil(t, out.Names, "fields outside asSet are still normalized")
+}
+
func TestNormalizeXMLOutput_NilPointerFieldUntouched(t *testing.T) {
- out := NormalizeXMLOutput(normalizeOuter{}).(normalizeOuter)
+ out := NormalizeXMLOutput(normalizeOuter{}, nil).(normalizeOuter)
assert.Nil(t, out.Nested, "a nil pointer field must not be allocated")
}
diff --git a/spinifex/vpcd/imds_instance_state.go b/spinifex/vpcd/imds_instance_state.go
index d9a99a1ce..10b506098 100644
--- a/spinifex/vpcd/imds_instance_state.go
+++ b/spinifex/vpcd/imds_instance_state.go
@@ -86,7 +86,7 @@ type recordLoader interface {
// this node does not hold locally resolves to nothing, and IMDS serves that as
// "no such instance" — a guest that boots with no key and no error anywhere.
func newInstanceRecordLoader(ctx context.Context, nc *nats.Conn) (recordLoader, error) {
- jsm, err := daemon.NewJetStreamManager(nc, 1)
+ jsm, err := daemon.NewJetStreamManager(nc)
if err != nil {
return nil, fmt.Errorf("jetstream manager: %w", err)
}
diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go
index 7ab816e79..45c3cf916 100644
--- a/spinifex/vpcd/vpcd.go
+++ b/spinifex/vpcd/vpcd.go
@@ -13,6 +13,7 @@ import (
"os/exec"
"os/signal"
"path/filepath"
+ "slices"
"strings"
"syscall"
"time"
@@ -212,26 +213,6 @@ func (svc *Service) Start() (int, error) {
return os.Getpid(), nil
}
-// Stop stops the vpcd service.
-func (svc *Service) Stop() error {
- return utils.StopProcessAt(svc.Config.BaseDir, serviceName)
-}
-
-// Status returns the vpcd service status.
-func (svc *Service) Status() (string, error) {
- return utils.ServiceStatus(svc.Config.BaseDir, serviceName)
-}
-
-// Shutdown gracefully shuts down the vpcd service.
-func (svc *Service) Shutdown() error {
- return svc.Stop()
-}
-
-// Reload reloads the vpcd service configuration.
-func (svc *Service) Reload() error {
- return nil
-}
-
// checkBrInt verifies the OVS integration bridge (br-int) exists.
// This is the bridge that all VM TAP devices connect to.
var checkBrInt = func() error {
@@ -863,14 +844,9 @@ func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js jetstream.J
if cfg == nil || cfg.ExternalMode == "" {
return nil, nil, nil
}
- wantDHCP := false
- for _, p := range cfg.ExternalPools {
- if p.Source == external.SourceDHCP {
- wantDHCP = true
- break
- }
- }
- if !wantDHCP {
+ if !slices.ContainsFunc(cfg.ExternalPools, func(p external.ExternalPoolConfig) bool {
+ return p.Source == external.SourceDHCP
+ }) {
return nil, nil, nil
}
diff --git a/tests/bench/ebsctl/compare.go b/tests/bench/ebsctl/compare.go
index 0bbbc3841..59afc2e0b 100644
--- a/tests/bench/ebsctl/compare.go
+++ b/tests/bench/ebsctl/compare.go
@@ -3,8 +3,10 @@ package ebsctl
import (
"encoding/json"
"fmt"
+ "maps"
"math/rand/v2"
"os"
+ "slices"
"sort"
"strconv"
"strings"
@@ -74,12 +76,7 @@ func unionOps(a, b *RunResult) []string {
for name := range b.Operations {
seen[name] = struct{}{}
}
- out := make([]string, 0, len(seen))
- for name := range seen {
- out = append(out, name)
- }
- sort.Strings(out)
- return out
+ return slices.Sorted(maps.Keys(seen))
}
// seriesFor picks the api or settle series from op, or nil if op is nil or
diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile
index 413ba5e22..6ea508985 100644
--- a/tests/e2e/Makefile
+++ b/tests/e2e/Makefile
@@ -18,9 +18,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
# BIN_DIR/.test so CI can ship binaries (not source + go toolchain) to
# the remote VM.
BIN_DIR ?= $(CURDIR)/_bin
-SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery
+SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum
-.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-all clean e2e-build
+.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build
RUN_FLAG = $(if $(RUN),-run '$(RUN)',)
@@ -74,6 +74,9 @@ e2e-instancerecovery: ; $(MAKE) e2e PKG=./instancerecovery/... TIMEOUT=60m
# Stands a load balancer and its backends up before it breaks anything, then
# takes the load balancer's node away and brings it back.
e2e-lbrecovery: ; $(MAKE) e2e PKG=./lbrecovery/... TIMEOUT=75m
+# Audits every KV bucket, then takes a node away and brings it back. Most of its
+# budget is the node stopping and starting, not the assertions.
+e2e-kvquorum: ; $(MAKE) e2e PKG=./kvquorum/... MODE=multinode TIMEOUT=40m
e2e-all: ; $(MAKE) e2e
e2e-build:
diff --git a/tests/e2e/ddil/scenarios/coverage_drift_test.go b/tests/e2e/ddil/scenarios/coverage_drift_test.go
index 09eb243fe..b4c3f8f21 100644
--- a/tests/e2e/ddil/scenarios/coverage_drift_test.go
+++ b/tests/e2e/ddil/scenarios/coverage_drift_test.go
@@ -6,11 +6,12 @@ import (
"go/ast"
"go/parser"
"go/token"
+ "maps"
"os"
"path/filepath"
"regexp"
"runtime"
- "sort"
+ "slices"
"strings"
"testing"
)
@@ -117,12 +118,7 @@ func lettersFromDoc(path string) ([]string, error) {
}
func sortedKeys(m map[string]struct{}) []string {
- out := make([]string, 0, len(m))
- for k := range m {
- out = append(out, k)
- }
- sort.Strings(out)
- return out
+ return slices.Sorted(maps.Keys(m))
}
// setDiff returns elements in a that are not in b (both inputs must be sorted).
diff --git a/tests/e2e/ecr/crane_skopeo_test.go b/tests/e2e/ecr/crane_skopeo_test.go
index 6f224c7f2..2bf30e111 100644
--- a/tests/e2e/ecr/crane_skopeo_test.go
+++ b/tests/e2e/ecr/crane_skopeo_test.go
@@ -22,7 +22,7 @@ const ociCmdTimeout = 2 * time.Minute
func writeLayerTar(t *testing.T, dir string) string {
t.Helper()
path := filepath.Join(dir, "layer.tar")
- fh, err := os.Create(path) //nolint:gosec // test-controlled path under TmpDir
+ fh, err := os.Create(path)
require.NoError(t, err)
defer fh.Close()
diff --git a/tests/e2e/ecs/ecs_test.go b/tests/e2e/ecs/ecs_test.go
index 13629492b..bd9a80900 100644
--- a/tests/e2e/ecs/ecs_test.go
+++ b/tests/e2e/ecs/ecs_test.go
@@ -527,7 +527,7 @@ func guestUserData(t *testing.T, env *harness.Env, fx *ecsFixture) string {
t.Helper()
caPath, err := harness.ResolveCACert(env)
require.NoError(t, err, "resolve CA cert")
- caBytes, err := os.ReadFile(caPath) //nolint:gosec // CA path from harness env
+ caBytes, err := os.ReadFile(caPath)
require.NoError(t, err, "read CA cert")
var ca strings.Builder
diff --git a/tests/e2e/eks/eks_test.go b/tests/e2e/eks/eks_test.go
index 77cda0944..4c526d676 100644
--- a/tests/e2e/eks/eks_test.go
+++ b/tests/e2e/eks/eks_test.go
@@ -69,7 +69,7 @@ func TestEKS(t *testing.T) {
t.Run("DescribeKubeconfigArtifacts", func(t *testing.T) {
requireClusterReady(t, fx)
path := writeKubeconfig(t, artifacts, fx.Cluster)
- raw, err := os.ReadFile(path) //nolint:gosec // artifact path built by the test
+ raw, err := os.ReadFile(path)
require.NoError(t, err)
kc := string(raw)
assert.Contains(t, kc, "server: "+aws.StringValue(fx.Cluster.Endpoint), "kubeconfig server = cluster endpoint")
@@ -1018,7 +1018,7 @@ func setupClusterFixture(t *testing.T, c *harness.AWSClient, env *harness.Env, a
t.Cleanup(func() { harness.DeleteWorkerEgress(t, c, fx.Egress) })
harness.Phase(t, "Creating cluster %q", fx.ClusterName)
- roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s-role", fx.AccountID, fx.ClusterName)
+ roleArn := harness.CreateEKSClusterRole(t, c, fx.ClusterName+"-role")
_, err = c.EKS.CreateCluster(&eks.CreateClusterInput{
Name: aws.String(fx.ClusterName),
RoleArn: aws.String(roleArn),
diff --git a/tests/e2e/gpu/ecs_gpu_test.go b/tests/e2e/gpu/ecs_gpu_test.go
index 3fef7145f..1388b84a8 100644
--- a/tests/e2e/gpu/ecs_gpu_test.go
+++ b/tests/e2e/gpu/ecs_gpu_test.go
@@ -235,7 +235,7 @@ func ecsGPUUserData(t *testing.T, env *harness.Env, cluster string) string {
t.Helper()
caPath, err := harness.ResolveCACert(env)
require.NoError(t, err, "resolve CA cert")
- caBytes, err := os.ReadFile(caPath) //nolint:gosec // CA path from harness env
+ caBytes, err := os.ReadFile(caPath)
require.NoError(t, err, "read CA cert")
var ca strings.Builder
diff --git a/tests/e2e/gpu/eks_gpu_test.go b/tests/e2e/gpu/eks_gpu_test.go
index 0bb024e9a..438a58795 100644
--- a/tests/e2e/gpu/eks_gpu_test.go
+++ b/tests/e2e/gpu/eks_gpu_test.go
@@ -157,7 +157,7 @@ func TestEKSGPUPodExposure(t *testing.T) {
clusterName := fmt.Sprintf("%s-%d", eksGPUClusterPfx, time.Now().Unix())
harness.Phase(t, "Creating cluster %q", clusterName)
- roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s-role", accountID, clusterName)
+ roleArn := harness.CreateEKSClusterRole(t, c, clusterName+"-role")
// e2e:allow-create — the GPU cluster is the subject under test (GPU worker join + device-plugin exposure).
_, err = c.EKS.CreateCluster(&eks.CreateClusterInput{
Name: aws.String(clusterName),
diff --git a/tests/e2e/harness/account.go b/tests/e2e/harness/account.go
index 9ed44bba3..5be8cdf3f 100644
--- a/tests/e2e/harness/account.go
+++ b/tests/e2e/harness/account.go
@@ -3,7 +3,8 @@
package harness
import (
- "sort"
+ "maps"
+ "slices"
"sync"
"testing"
)
@@ -59,10 +60,5 @@ func (a *AccountCarousel) Get(name string) *Profile {
func (a *AccountCarousel) Names() []string {
a.mu.RLock()
defer a.mu.RUnlock()
- out := make([]string, 0, len(a.profiles))
- for k := range a.profiles {
- out = append(out, k)
- }
- sort.Strings(out)
- return out
+ return slices.Sorted(maps.Keys(a.profiles))
}
diff --git a/tests/e2e/harness/discover.go b/tests/e2e/harness/discover.go
index 02c7b5bde..f5d8f5250 100644
--- a/tests/e2e/harness/discover.go
+++ b/tests/e2e/harness/discover.go
@@ -65,11 +65,11 @@ func DiscoverNanoInstanceType(t *testing.T, fx *Fixture) (instanceType, arch str
if err != nil {
t.Fatalf("DiscoverNanoInstanceType: %v", err)
}
- parts := strings.SplitN(combined, "|", 2)
- if len(parts) != 2 {
+ instanceType, arch, ok := strings.Cut(combined, "|")
+ if !ok {
t.Fatalf("DiscoverNanoInstanceType: malformed memo value %q", combined)
}
- return parts[0], parts[1]
+ return instanceType, arch
}
// DiscoverUbuntuAMI returns the AMI ID for the architecture-appropriate Ubuntu
diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go
index 5a102ca2a..7009ecbd7 100644
--- a/tests/e2e/harness/ec2helpers.go
+++ b/tests/e2e/harness/ec2helpers.go
@@ -3,10 +3,8 @@
package harness
import (
- "bytes"
"context"
"fmt"
- "os/exec"
"strconv"
"strings"
"testing"
@@ -72,25 +70,7 @@ func InstancePrivateIP(t *testing.T, c *AWSClient, instanceID string) string {
func LsblkRootGiB(t *testing.T, tgt SSHTarget) int {
t.Helper()
cmd := `SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"`
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- cmd,
- }
- var stdout, stderr bytes.Buffer
- sshCmd := exec.Command("ssh", args...)
- sshCmd.Stdout = &stdout
- sshCmd.Stderr = &stderr
- if err := sshCmd.Run(); err != nil {
- t.Fatalf("ssh lsblk %s@%s:%d failed: %v\nstderr: %s", tgt.User, tgt.Host, tgt.Port, err, stderr.String())
- }
- raw := strings.TrimSpace(stdout.String())
+ raw := strings.TrimSpace(RunSSH(t, tgt, cmd))
bytesN, err := strconv.ParseInt(raw, 10, 64)
if err != nil {
t.Fatalf("ssh lsblk %s@%s:%d: parse %q: %v", tgt.User, tgt.Host, tgt.Port, raw, err)
diff --git a/tests/e2e/harness/fixtures.go b/tests/e2e/harness/fixtures.go
index 7704b9c0e..450ee1ab7 100644
--- a/tests/e2e/harness/fixtures.go
+++ b/tests/e2e/harness/fixtures.go
@@ -45,7 +45,7 @@ import (
type Fixture struct {
// parent is the testing.T that constructed the Fixture in test mode.
// nil in process mode (NewProcessFixture). Cleanup routing forks on
- // this — see registerCleanup.
+ // this — see RegisterCleanup.
parent *testing.T
EC2 ec2iface.EC2API
@@ -165,13 +165,16 @@ func (f *Fixture) Close() error {
return fmt.Errorf("%d resource(s) left behind:\n %s", len(leaks), strings.Join(leaks, "\n "))
}
-// registerCleanup routes a teardown callback to the appropriate sink based
+// RegisterCleanup routes a teardown callback to the appropriate sink based
// on construction mode. Test mode: t.Cleanup on the bound parent. Process
// mode: append to the LIFO drained by Close().
//
// Each callback is isolated from the rest: a panic in one teardown would
// otherwise abort the whole chain and strand every resource behind it.
-func (f *Fixture) registerCleanup(fn func()) {
+//
+// Use it for resources created mid-suite outside ensureOnce, which must outlive
+// the calling subtest rather than be torn down before a dependent one runs.
+func (f *Fixture) RegisterCleanup(fn func()) {
isolated := func() {
defer func() {
if r := recover(); r != nil {
@@ -219,19 +222,6 @@ func (f *Fixture) logf(format string, args ...any) {
// for tests that need to assert against names they didn't construct directly.
func (f *Fixture) Scratch() string { return f.scratch }
-// RegisterCleanup runs fn at fixture teardown, not at the end of the
-// calling subtest. Use for resources created mid-suite by code paths that
-// don't flow through ensureOnce (e.g. IAM users threaded across multiple
-// phase subtests) — registering on the calling subtest would tear them
-// down before the next dependent subtest runs.
-//
-// Test mode (NewFixture): fn runs at parent t.Cleanup.
-// Process mode (NewProcessFixture): fn runs at f.Close() (typically called
-// from TestMain after m.Run()).
-func (f *Fixture) RegisterCleanup(fn func()) {
- f.registerCleanup(fn)
-}
-
// ensureOnce returns the cached resource ID for key, or calls create exactly
// once (via singleflight) and caches the result. On failure no cleanup is
// registered and the cache is untouched, so a retry triggers a fresh create.
@@ -267,7 +257,7 @@ func (f *Fixture) ensureOnce(t *testing.T, key string, create func() (string, fu
f.mu.Unlock()
if !dup && cleanup != nil {
- f.registerCleanup(func() {
+ f.RegisterCleanup(func() {
if cerr := cleanup(); cerr != nil {
f.reportLeak("%s (%s): %v", key, id, cerr)
}
diff --git a/tests/e2e/harness/fixtures_test.go b/tests/e2e/harness/fixtures_test.go
index 472b5ddf2..17634c246 100644
--- a/tests/e2e/harness/fixtures_test.go
+++ b/tests/e2e/harness/fixtures_test.go
@@ -428,9 +428,7 @@ func TestPollUntil_TimeoutWrapsLastErr(t *testing.T) {
// Compile-time check: every exported Ensure* compiles with the Fixture.
// Never invoked at runtime; pure type assertion against the public API.
-//
-//nolint:unused,deadcode // exists only to fail compile when an Ensure* signature drifts
-func _ensureCompileCheck(t *testing.T, fx *Fixture) {
+var _ = func(t *testing.T, fx *Fixture) {
_, _ = EnsureKeyPair(t, fx)
_ = EnsureAMI(t, fx, AMISource{Existing: "ami-0"})
_ = EnsureDefaultVPC(t, fx)
diff --git a/tests/e2e/harness/guest_data.go b/tests/e2e/harness/guest_data.go
index 88dec1bc9..7a084539a 100644
--- a/tests/e2e/harness/guest_data.go
+++ b/tests/e2e/harness/guest_data.go
@@ -5,8 +5,10 @@ package harness
import (
"context"
"fmt"
+ "maps"
"os/exec"
"regexp"
+ "slices"
"strconv"
"strings"
"testing"
@@ -52,18 +54,7 @@ func GuestExec(tgt SSHTarget, cmd string) (string, error) {
func GuestExecTimeout(tgt SSHTarget, cmd string, timeout time.Duration) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- cmd,
- }
- out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
+ out, err := exec.CommandContext(ctx, "ssh", guestSSHArgs(tgt, 5, cmd)...).CombinedOutput()
if err != nil && ctx.Err() == context.DeadlineExceeded {
return string(out), fmt.Errorf("guest command exceeded its %s timeout (still running when the deadline hit): %w: %w", timeout, context.DeadlineExceeded, err)
}
@@ -115,10 +106,7 @@ func WaitForNewGuestDisk(t *testing.T, tgt SSHTarget, before map[string]struct{}
return nil
}
}
- names := make([]string, 0, len(now))
- for name := range now {
- names = append(names, name)
- }
+ names := slices.Collect(maps.Keys(now))
return fmt.Errorf("no new disk yet (visible: %s)", strings.Join(names, ","))
}, timeout, 2*time.Second)
return found
diff --git a/tests/e2e/harness/iam.go b/tests/e2e/harness/iam.go
index 0a9907df5..ebff0898a 100644
--- a/tests/e2e/harness/iam.go
+++ b/tests/e2e/harness/iam.go
@@ -16,6 +16,21 @@ func IAMRoleARN(account, name string) string {
return "arn:aws:iam::" + account + ":role/" + name
}
+// CreateEKSClusterRole creates a role EKS can assume as a cluster role and
+// deletes it at cleanup. CreateCluster refuses a role that does not exist.
+func CreateEKSClusterRole(t *testing.T, c *AWSClient, name string) string {
+ t.Helper()
+ const eksTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"eks.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
+ out, err := c.IAM.CreateRole(&iam.CreateRoleInput{
+ RoleName: aws.String(name),
+ AssumeRolePolicyDocument: aws.String(eksTrustPolicy),
+ Description: aws.String("E2E EKS cluster role"),
+ })
+ require.NoError(t, err, "create-role (cluster) %s", name)
+ t.Cleanup(func() { IAMDeleteRoleAndProfilesBestEffort(c, name, nil) })
+ return aws.StringValue(out.Role.Arn)
+}
+
// IAMPolicyARN builds the canonical policy ARN for a given account + policy
// "key" (a bare name, or "path/name" without the leading slash).
func IAMPolicyARN(account, key string) string {
diff --git a/tests/e2e/harness/jetstream.go b/tests/e2e/harness/jetstream.go
index f04f3587e..fe9b03172 100644
--- a/tests/e2e/harness/jetstream.go
+++ b/tests/e2e/harness/jetstream.go
@@ -3,9 +3,15 @@
package harness
import (
+ "fmt"
+ "sort"
+ "strings"
"testing"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/utils"
+ "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
@@ -42,3 +48,149 @@ func KVReplicaFactors(t *testing.T, env *Env, buckets ...string) map[string]int
}
return out
}
+
+// WantKVReplicas is how many nodes a KV bucket must be replicated across on a
+// cluster of this size.
+//
+// It defers to the production rule rather than restating it. A test that carried
+// its own copy would keep passing after the policy changed, which is the one
+// thing this assertion must not do.
+func WantKVReplicas(nodes int) int {
+ return clustersize.ReplicasFor(nodes)
+}
+
+// DialClusterNATS opens a NATS connection to the cluster, using the caller's
+// env for the address and credentials. The connection is closed on cleanup.
+func DialClusterNATS(t *testing.T, env *Env) *nats.Conn {
+ t.Helper()
+ host, token, ca := natsConn(t, env)
+ nc, err := utils.ConnectNATS(host, token, ca)
+ if err != nil {
+ t.Fatalf("connect NATS %s: %v", host, err)
+ }
+ t.Cleanup(nc.Close)
+ return nc
+}
+
+// KVReplication reports every KV bucket in the cluster: its replica count, what
+// the cluster's size entitles it to, and which nodes hold it.
+//
+// The node count is declared here because this process never loaded a cluster
+// config, and the audit refuses to guess one rather than assuming a single
+// replica. Declaring is write-once, so the reset makes a second call with the
+// same count safe and a third with a different one a visible failure.
+func KVReplication(t *testing.T, env *Env, nodes int) []kvutil.BucketReport {
+ t.Helper()
+ clustersize.ResetForTest()
+ if err := clustersize.Declare(nodes); err != nil {
+ t.Fatalf("declare cluster size %d: %v", nodes, err)
+ }
+ js, err := jetstream.New(DialClusterNATS(t, env))
+ if err != nil {
+ t.Fatalf("jetstream context: %v", err)
+ }
+ reports, err := kvutil.AuditBucketReplicas(t.Context(), js)
+ if err != nil {
+ t.Fatalf("audit KV bucket replicas: %v", err)
+ }
+ return reports
+}
+
+// RequireKVQuorum fails unless every KV bucket in the cluster is replicated
+// across every node, and unless each one is actually held by that many distinct
+// nodes rather than merely configured for it.
+//
+// A bucket below the node count lives on fewer nodes than depend on it, so
+// losing one of them makes it unreadable and unwritable from every node at
+// once — a cluster-wide outage caused by a single-node event. This runs in
+// every multi-node suite rather than one of them, because a bucket created by
+// any service on any cell is the one that can be wrong.
+func RequireKVQuorum(t *testing.T, env *Env, nodes int) {
+ t.Helper()
+ want := WantKVReplicas(nodes)
+
+ reports := KVReplication(t, env, nodes)
+ if len(reports) == 0 {
+ t.Fatal("the cluster reports no KV buckets at all, so nothing was checked")
+ }
+
+ var problems []string
+ for _, r := range reports {
+ if r.Replicas < want {
+ problems = append(problems, fmt.Sprintf(
+ "%s is on %d replicas, want %d: losing the node holding it takes it down cluster-wide",
+ r.Bucket, r.Replicas, want))
+ continue
+ }
+ // Configured, placed and serving are three facts. A stream can carry the
+ // right count while its group is short of peers or those peers are not
+ // caught up, and either reads as healthy right up until the node it is
+ // really on goes away.
+ if held := distinctPeers(r.Peers); held < want {
+ problems = append(problems, fmt.Sprintf(
+ "%s is configured for %d replicas but held by %d node(s) (%s)",
+ r.Bucket, r.Replicas, held, strings.Join(r.Peers, ",")))
+ continue
+ }
+ if r.Online < want {
+ problems = append(problems, fmt.Sprintf(
+ "%s is held by %d node(s) but only %d are caught up and reachable (leader %s)",
+ r.Bucket, len(r.Peers), r.Online, r.Leader))
+ }
+ }
+ if len(problems) > 0 {
+ sort.Strings(problems)
+ t.Fatalf("%d of %d KV buckets are not replicated across the cluster's %d nodes:\n %s",
+ len(problems), len(reports), nodes, strings.Join(problems, "\n "))
+ }
+ Detail(t, "kv_buckets", len(reports), "replicas", want, "nodes", nodes)
+}
+
+// RequireKVSingleNode fails unless a one-node cluster's buckets are exactly what
+// one node can give them: one replica each, all of them serving.
+//
+// It is the counterpart assertion to RequireKVQuorum, and it is not the same
+// check with a smaller number. On one node R1 is correct rather than a defect, so
+// what has to be proved is the opposite: that nothing declared a count this
+// deployment cannot place, and that a bucket which cannot be raised has not been
+// left unreadable by the attempt.
+func RequireKVSingleNode(t *testing.T, env *Env) {
+ t.Helper()
+
+ reports := KVReplication(t, env, 1)
+ if len(reports) == 0 {
+ t.Fatal("the cluster reports no KV buckets at all, so nothing was checked")
+ }
+
+ var problems []string
+ for _, r := range reports {
+ if r.Replicas != 1 {
+ problems = append(problems, fmt.Sprintf(
+ "%s is configured for %d replicas on a single node, which JetStream cannot place",
+ r.Bucket, r.Replicas))
+ continue
+ }
+ if !r.HasQuorum() {
+ problems = append(problems, fmt.Sprintf("%s is not serving: %d of %d replicas online",
+ r.Bucket, r.Online, r.Replicas))
+ }
+ }
+ if len(problems) > 0 {
+ sort.Strings(problems)
+ t.Fatalf("%d of %d KV buckets are wrong for a single-node cluster:\n %s",
+ len(problems), len(reports), strings.Join(problems, "\n "))
+ }
+ Detail(t, "kv_buckets", len(reports), "replicas", 1, "nodes", 1)
+}
+
+// distinctPeers counts the unique node names holding a stream, since the leader
+// is reported separately from the followers and could repeat.
+func distinctPeers(peers []string) int {
+ seen := make(map[string]struct{}, len(peers))
+ for _, p := range peers {
+ if p != "" {
+ seen[p] = struct{}{}
+ }
+ }
+ return len(seen)
+}
diff --git a/tests/e2e/harness/kubectl.go b/tests/e2e/harness/kubectl.go
index d5b8cc0bb..b3fc403c8 100644
--- a/tests/e2e/harness/kubectl.go
+++ b/tests/e2e/harness/kubectl.go
@@ -43,7 +43,7 @@ func NewKubectl(t *testing.T, kubeconfig string, awsEnv []string) *Kubectl {
// output. Errors are returned (not fatal) so callers can poll readiness.
func (k *Kubectl) Run(timeout time.Duration, args ...string) (string, error) {
full := append([]string{"--kubeconfig", k.Kubeconfig}, args...)
- cmd := exec.Command(k.Bin, full...) //nolint:gosec // bin is LookPath-resolved, args test-controlled
+ cmd := exec.Command(k.Bin, full...)
env := os.Environ()
env = append(env, k.AWSEnv...)
cmd.Env = env
@@ -116,7 +116,7 @@ func (k *Kubectl) execPluginProbe(timeout time.Duration) string {
if err != nil {
return "exec plugin " + argv[0] + " not on PATH: " + err.Error() + "\n"
}
- cmd := exec.Command(bin, argv[1:]...) //nolint:gosec // argv comes from the kubeconfig this test wrote
+ cmd := exec.Command(bin, argv[1:]...)
cmd.Env = append(os.Environ(), k.AWSEnv...)
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
diff --git a/tests/e2e/harness/multinode_cluster.go b/tests/e2e/harness/multinode_cluster.go
index 4870b6f67..3cd2e93e8 100644
--- a/tests/e2e/harness/multinode_cluster.go
+++ b/tests/e2e/harness/multinode_cluster.go
@@ -30,6 +30,9 @@ type natsRoutezResponse struct {
// WaitNATSPeers polls every node's NATS /routez until each reports at least want
// distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222
// only, so queries run via PeerSSH + curl rather than dialling node.Addr directly.
+//
+// want is a count of *other* nodes, because /routez reports routes to peers and
+// not a node's own membership. A fully formed cluster is len(c.Nodes)-1.
// Nodes passed to WithSkipNodes are excluded — use it when a node has been
// stopped so its dead monitor port doesn't fail the poll.
func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) {
diff --git a/tests/e2e/harness/registry.go b/tests/e2e/harness/registry.go
index 0d1acfd1c..68cc24921 100644
--- a/tests/e2e/harness/registry.go
+++ b/tests/e2e/harness/registry.go
@@ -55,7 +55,7 @@ func NewSkopeo(t *testing.T, dockerConfig string) *ExternalCLI {
// Run executes the client with args and returns combined output. Errors are
// returned (not fatal) so callers can assert on output or poll.
func (c *ExternalCLI) Run(timeout time.Duration, args ...string) (string, error) {
- cmd := exec.Command(c.Bin, args...) //nolint:gosec // bin is LookPath-resolved, args test-controlled
+ cmd := exec.Command(c.Bin, args...)
cmd.Env = append(os.Environ(), c.Env...)
var buf bytes.Buffer
cmd.Stdout = &buf
@@ -78,7 +78,7 @@ func (c *ExternalCLI) Run(timeout time.Duration, args ...string) (string, error)
// RunStdin is Run with data piped to the process stdin (docker login
// --password-stdin).
func (c *ExternalCLI) RunStdin(timeout time.Duration, stdin string, args ...string) (string, error) {
- cmd := exec.Command(c.Bin, args...) //nolint:gosec // bin is LookPath-resolved, args test-controlled
+ cmd := exec.Command(c.Bin, args...)
cmd.Env = append(os.Environ(), c.Env...)
cmd.Stdin = strings.NewReader(stdin)
var buf bytes.Buffer
diff --git a/tests/e2e/harness/ssh.go b/tests/e2e/harness/ssh.go
index 9c1dae0f6..9efdb085f 100644
--- a/tests/e2e/harness/ssh.go
+++ b/tests/e2e/harness/ssh.go
@@ -3,14 +3,21 @@
package harness
import (
+ "bytes"
"context"
"fmt"
"net"
"os"
"os/exec"
"strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
"time"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/ec2"
+
"golang.org/x/crypto/ssh"
)
@@ -28,24 +35,181 @@ type SSH interface {
// RunGuestSSH executes cmd in a guest addressed by an SSHTarget. It returns
// combined output so callers can retry assertions without terminating a test.
func RunGuestSSH(ctx context.Context, target SSHTarget, cmd string) ([]byte, error) {
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(target.Port),
- "-i", target.KeyPath,
- target.User + "@" + target.Host,
- cmd,
- }
- out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
+ out, err := exec.CommandContext(ctx, "ssh", guestSSHArgs(target, 5, cmd)...).CombinedOutput()
if err != nil {
return out, fmt.Errorf("guest ssh %s@%s:%d: %w: %s", target.User, target.Host, target.Port, err, out)
}
return out, nil
}
+// guestSSHArgs is the non-interactive, no-host-key-check ssh(1) argument list
+// every guest probe uses; connectTimeout is in seconds.
+func guestSSHArgs(tgt SSHTarget, connectTimeout int, cmd string) []string {
+ return []string{
+ "-o", "StrictHostKeyChecking=no",
+ "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "LogLevel=ERROR",
+ "-o", "ConnectTimeout=" + strconv.Itoa(connectTimeout),
+ "-o", "BatchMode=yes",
+ "-p", strconv.Itoa(tgt.Port),
+ "-i", tgt.KeyPath,
+ tgt.User + "@" + tgt.Host,
+ cmd,
+ }
+}
+
+// RunSSH runs command in the guest and returns stdout. It t.Fatals on a
+// non-zero exit so callers can chain assertions on the output.
+func RunSSH(t *testing.T, tgt SSHTarget, command string) string {
+ t.Helper()
+ var stdout, stderr bytes.Buffer
+ cmd := exec.Command("ssh", guestSSHArgs(tgt, 5, command)...)
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ t.Fatalf("ssh %s@%s:%d %q failed: %v\nstderr: %s",
+ tgt.User, tgt.Host, tgt.Port, command, err, stderr.String())
+ }
+ return stdout.String()
+}
+
+// RunSSHCombined runs command in the guest and returns combined stdout+stderr
+// regardless of exit status, for probes (ping, curl) where a non-zero exit is
+// an expected outcome.
+func RunSSHCombined(tgt SSHTarget, command string) (string, error) {
+ out, err := exec.Command("ssh", guestSSHArgs(tgt, 5, command)...).CombinedOutput()
+ return string(out), err
+}
+
+// SSHReadyBudget bounds the first SSH-handshake pass. Baremetal boots q35
+// guests slower (OVMF + NBD disks), so the harness widens it via
+// SPINIFEX_SSH_READY_TIMEOUT (Go duration); default 3m bounds shared runners.
+var SSHReadyBudget = durationEnv("SPINIFEX_SSH_READY_TIMEOUT", 3*time.Minute)
+
+// sshReprimeBudget bounds the second SSH pass after an ARP re-prime; short
+// because a helpful re-prime lands quickly. SPINIFEX_SSH_REPRIME_TIMEOUT.
+var sshReprimeBudget = durationEnv("SPINIFEX_SSH_REPRIME_TIMEOUT", 60*time.Second)
+
+// wanBridge is the host bridge an EIP is presented on, i.e. the L2 segment the
+// runner ARPs to reach a guest's public IP. Override via SPINIFEX_WAN_BRIDGE.
+var wanBridge = stringEnv("SPINIFEX_WAN_BRIDGE", "br-wan")
+
+func durationEnv(key string, def time.Duration) time.Duration {
+ if v := os.Getenv(key); v != "" {
+ if d, err := time.ParseDuration(v); err == nil && d > 0 {
+ return d
+ }
+ }
+ return def
+}
+
+func stringEnv(key, def string) string {
+ if v := strings.TrimSpace(os.Getenv(key)); v != "" {
+ return v
+ }
+ return def
+}
+
+// SSHHealth is a suite's sticky SSH-datapath verdict: the first handshake
+// timeout marks it broken and later SSH-dependent tests skip rather than
+// re-run the same multi-minute wait. Each suite keeps its own value.
+type SSHHealth struct {
+ broken atomic.Bool
+}
+
+// Require skips t if an earlier SSH probe in this suite has failed.
+func (h *SSHHealth) Require(t *testing.T) {
+ t.Helper()
+ if h.broken.Load() {
+ t.Skipf("skipping: earlier SSH probe failed; " +
+ "not retrying to keep suite time bounded")
+ }
+}
+
+// WaitReady waits for a full SSH handshake to host:port. TCP reachability is
+// not enough: sshd accepts while pam/cloud-init finish, and the first real
+// command then times out during banner exchange.
+func (h *SSHHealth) WaitReady(t *testing.T, host string, port int, keyPath string) {
+ t.Helper()
+ h.Require(t)
+ addr := net.JoinHostPort(host, strconv.Itoa(port))
+ Step(t, "waiting for SSH handshake %s", addr)
+ if h.TryReady(host, port, keyPath, SSHReadyBudget) {
+ return
+ }
+ // OVN emits no GARP on a same-chassis EIP rebind, so a decayed host neigh
+ // entry can blackhole host->guest SSH past the budget. Re-prime and retry
+ // once before declaring the datapath broken.
+ Step(t, "SSH handshake %s missed %s budget; re-priming ARP + retrying", addr, SSHReadyBudget)
+ reprimeSSHReachability(t, host)
+ if h.TryReady(host, port, keyPath, sshReprimeBudget) {
+ return
+ }
+ h.broken.Store(true)
+ t.Fatalf("Eventually: condition not met within %s (+%s after ARP re-prime): "+
+ "[SSH handshake %s never completed] "+
+ "(sticky-skip enabled for downstream)", SSHReadyBudget, sshReprimeBudget, addr)
+}
+
+// TryReady is WaitReady without the re-prime or t.Fatal: it reports whether
+// the handshake completed within budget and never marks the suite broken.
+func (h *SSHHealth) TryReady(host string, port int, keyPath string, budget time.Duration) bool {
+ if h.broken.Load() {
+ return false
+ }
+ tgt := SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
+ deadline := time.Now().Add(budget)
+ for {
+ if exec.Command("ssh", guestSSHArgs(tgt, 3, "true")...).Run() == nil {
+ return true
+ }
+ if time.Now().After(deadline) {
+ return false
+ }
+ time.Sleep(2 * time.Second)
+ }
+}
+
+// reprimeSSHReachability best-effort flushes the host ARP entry for host on
+// the WAN bridge so the next pass re-resolves it. Needs ip(8) and passwordless
+// sudo; non-fatal so a missing tool never masks the real result.
+func reprimeSSHReachability(t *testing.T, host string) {
+ t.Helper()
+ if _, err := exec.LookPath("ip"); err != nil {
+ Step(t, "skip ARP re-prime: ip(8) unavailable (%v)", err)
+ return
+ }
+ if err := exec.Command("sudo", "-n", "true").Run(); err != nil {
+ Step(t, "skip ARP re-prime: passwordless sudo unavailable (%v)", err)
+ return
+ }
+ Step(t, "flushing host neigh for %s dev %s", host, wanBridge)
+ out, err := exec.Command("sudo", "-n", "ip", "neigh", "flush", "to", host, "dev", wanBridge).CombinedOutput()
+ if err != nil {
+ Step(t, "ARP re-prime flush failed (best-effort): %v: %s", err, strings.TrimSpace(string(out)))
+ }
+}
+
+// WaitForInstanceStateSoft is the cleanup-time analogue of
+// WaitForInstanceState: it polls and returns an error instead of t.Fatal.
+func WaitForInstanceStateSoft(c *AWSClient, id, target string, wait time.Duration) error {
+ deadline := time.Now().Add(wait)
+ for {
+ out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
+ InstanceIds: []*string{aws.String(id)},
+ })
+ if err == nil && len(out.Reservations) > 0 && len(out.Reservations[0].Instances) > 0 {
+ if aws.StringValue(out.Reservations[0].Instances[0].State.Name) == target {
+ return nil
+ }
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf("instance %s did not reach %s within %s", id, target, wait)
+ }
+ time.Sleep(2 * time.Second)
+ }
+}
+
// sshClient is the production SSH transport, backed by
// golang.org/x/crypto/ssh. One client is opened per (user, key, node) tuple
// on first use and cached for the harness lifetime.
diff --git a/tests/e2e/harness/system_account.go b/tests/e2e/harness/system_account.go
index 9173de656..45b0f4ff6 100644
--- a/tests/e2e/harness/system_account.go
+++ b/tests/e2e/harness/system_account.go
@@ -35,7 +35,7 @@ func SystemAWSClient(t *testing.T, env *Env) *AWSClient {
t.Fatalf("SystemAWSClient: no config dir located; set SPINIFEX_CONFIG_DIR")
}
path := filepath.Join(env.ConfigDir, systemCredentialsFile)
- out, err := exec.Command("sudo", "-n", "cat", path).Output() //nolint:gosec // path is env-derived, not caller input
+ out, err := exec.Command("sudo", "-n", "cat", path).Output()
if err != nil {
t.Fatalf("SystemAWSClient: sudo -n cat %s: %v", path, err)
}
diff --git a/tests/e2e/iam/authorization_test.go b/tests/e2e/iam/authorization_test.go
index 871ff8bd7..fc9a8edff 100644
--- a/tests/e2e/iam/authorization_test.go
+++ b/tests/e2e/iam/authorization_test.go
@@ -46,7 +46,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
createKeyPair(t, fix, principal, granted)
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String(sibling)}) // e2e:allow-create — the create is the authorization probe
return err
})
@@ -80,7 +80,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
// Same client, same key: the credential is a pointer into live IAM state,
// not a capability minted when the key was issued.
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-deny-2"),
})
@@ -123,7 +123,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
_, err = fix.AWS.IAM.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(grantARN)})
require.NoError(t, err, "delete-policy %s", grantARN)
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-attached-2"),
})
@@ -163,7 +163,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
require.NoError(t, err, "create-policy-version %s", policyARN)
t.Cleanup(func() { deleteSecondPolicyVersionBestEffort(t, fix, policyARN) })
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-version-2"),
})
@@ -190,7 +190,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
})
require.NoError(t, err, "delete-access-key %s", principal.KeyID)
- // Not AccessDenied: a deleted key no longer authenticates, so the request
+ // Not UnauthorizedOperation: a deleted key no longer authenticates, so the request
// never reaches policy evaluation.
harness.ExpectError(t, "InvalidClientTokenId", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
@@ -242,7 +242,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
})
createKeyPair(t, fix, principal, run+"-user-1")
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := principal.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-role-1"),
})
@@ -263,7 +263,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
createKeyPair(t, fix, &authzPrincipal{Client: session}, run+"-role-2")
// The grant that carried the assuming user does not follow the session:
// the role's policies are the whole of what it holds.
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := session.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-user-2"),
})
@@ -288,7 +288,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
Resource: []string{keyPairARN(account, run+"-ip-*")},
Condition: sourceIPCondition(authzForeignCIDR),
})
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := elsewhere.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-ip-2"),
})
@@ -309,7 +309,7 @@ func runIAMAuthorization(t *testing.T, fix *Fixture) {
Resource: []string{"*"},
Condition: sourceIPCondition(here),
})
- harness.ExpectError(t, "AccessDenied", func() error {
+ harness.ExpectError(t, "UnauthorizedOperation", func() error {
_, err := fenced.Client.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ // e2e:allow-create — the create is the authorization probe
KeyName: aws.String(run + "-ip-3"),
})
@@ -382,7 +382,7 @@ func addAccessKey(t *testing.T, fix *Fixture, principal *authzPrincipal) *authzP
client := harness.NewAWSClientWithCreds(t, fix.Env, keyID, aws.StringValue(out.AccessKey.SecretAccessKey))
harness.EventuallyErr(t, func() error {
_, err := client.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
- if err == nil || harness.ErrorCodeIs(err, "AccessDenied") {
+ if err == nil || harness.ErrorCodeIs(err, "UnauthorizedOperation") {
return nil
}
return fmt.Errorf("credentials for %s are not live yet: %w", principal.UserName, err)
diff --git a/tests/e2e/iam/datapath_helpers_test.go b/tests/e2e/iam/datapath_helpers_test.go
index 1ba317a08..14afec0ea 100644
--- a/tests/e2e/iam/datapath_helpers_test.go
+++ b/tests/e2e/iam/datapath_helpers_test.go
@@ -5,205 +5,18 @@ package iam
import (
"bufio"
"bytes"
- "fmt"
- "net"
"os"
- "os/exec"
"path/filepath"
- "strconv"
"strings"
- "sync/atomic"
"testing"
- "time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
-// SSH / datapath helpers ported from the single suite so the IMDS and
-// instance-profile tests run self-contained inside iam. The single suite keeps
-// its own copy — other single tests still use these. They are separate test
-// binaries (separate processes), so the sticky-skip state below is per-suite.
-
-// sshDatapathBroken is set the first time an SSH probe in this suite times out.
-// Downstream SSH-dependent assertions call requireSSHHealthy(t) and skip rather
-// than re-running the same multi-minute timeout against the same broken datapath.
-var sshDatapathBroken atomic.Bool
-
-// sshReadyBudget bounds the first SSH-handshake pass. Widen via
-// SPINIFEX_SSH_READY_TIMEOUT (Go duration); default 3m bounds shared runners.
-var sshReadyBudget = resolveDurationEnv("SPINIFEX_SSH_READY_TIMEOUT", 3*time.Minute)
-
-// sshReprimeBudget bounds the second SSH pass after an ARP re-prime. Short by
-// design — if the re-prime helps, the handshake lands quickly. Override via
-// SPINIFEX_SSH_REPRIME_TIMEOUT.
-var sshReprimeBudget = resolveDurationEnv("SPINIFEX_SSH_REPRIME_TIMEOUT", 60*time.Second)
-
-// wanBridge is the host bridge an EIP is presented on — the L2 segment the
-// runner ARPs to reach a guest's public IP. Default br-wan; override for
-// non-standard pool fixtures via SPINIFEX_WAN_BRIDGE.
-var wanBridge = resolveStringEnv("SPINIFEX_WAN_BRIDGE", "br-wan")
-
-func resolveDurationEnv(key string, def time.Duration) time.Duration {
- if v := os.Getenv(key); v != "" {
- if d, err := time.ParseDuration(v); err == nil && d > 0 {
- return d
- }
- }
- return def
-}
-
-func resolveStringEnv(key, def string) string {
- if v := strings.TrimSpace(os.Getenv(key)); v != "" {
- return v
- }
- return def
-}
-
-// requireSSHHealthy skips the calling test if a prior SSH probe in this suite
-// has already failed, cutting the cascade cost of a broken datapath.
-func requireSSHHealthy(t *testing.T) {
- t.Helper()
- if sshDatapathBroken.Load() {
- t.Skipf("skipping: earlier SSH probe failed; " +
- "not retrying to keep suite time bounded")
- }
-}
-
-// waitForSSHReady probes a full SSH handshake (BatchMode + ConnectTimeout)
-// against host:port, retrying until the daemon completes banner exchange.
-// TCP-port reachability alone is insufficient — sshd accepts the connect
-// while pam/cloud-init are still finishing, and the first real runSSH then
-// hits "Connection timed out during banner exchange".
-func waitForSSHReady(t *testing.T, host string, port int, keyPath string) {
- t.Helper()
- requireSSHHealthy(t)
- addr := net.JoinHostPort(host, strconv.Itoa(port))
- harness.Step(t, "waiting for SSH handshake %s", addr)
- if trySSHReady(host, port, keyPath, sshReadyBudget) {
- return
- }
- // First budget elapsed. OVN emits no GARP on a same-chassis EIP rebind, so a
- // decayed host neigh entry for the EIP can blackhole host->guest SSH until it
- // ages out — past our budget. Re-prime the ARP entry and retry once before
- // declaring the datapath broken, riding out the transient learning gap.
- harness.Step(t, "SSH handshake %s missed %s budget; re-priming ARP + retrying", addr, sshReadyBudget)
- reprimeSSHReachability(t, host)
- if trySSHReady(host, port, keyPath, sshReprimeBudget) {
- return
- }
- sshDatapathBroken.Store(true)
- t.Fatalf("Eventually: condition not met within %s (+%s after ARP re-prime): "+
- "[SSH handshake %s never completed] "+
- "(sticky-skip enabled for downstream)", sshReadyBudget, sshReprimeBudget, addr)
-}
-
-// reprimeSSHReachability best-effort flushes the host ARP entry for the EIP on
-// the WAN bridge, forcing a fresh broadcast re-resolution before the next SSH
-// pass. Flush (not replace) is mode-agnostic: it needs no MAC and can't install
-// a wrong centralised-mode entry. Gated on ip + passwordless sudo like the IMDS
-// datapath dump; non-fatal so a missing tool never masks the real result.
-func reprimeSSHReachability(t *testing.T, host string) {
- t.Helper()
- if _, err := exec.LookPath("ip"); err != nil {
- harness.Step(t, "skip ARP re-prime: ip(8) unavailable (%v)", err)
- return
- }
- if err := exec.Command("sudo", "-n", "true").Run(); err != nil {
- harness.Step(t, "skip ARP re-prime: passwordless sudo unavailable (%v)", err)
- return
- }
- harness.Step(t, "flushing host neigh for %s dev %s", host, wanBridge)
- out, err := exec.Command("sudo", "-n", "ip", "neigh", "flush", "to", host, "dev", wanBridge).CombinedOutput()
- if err != nil {
- harness.Step(t, "ARP re-prime flush failed (best-effort): %v: %s", err, strings.TrimSpace(string(out)))
- }
-}
-
-// waitForSSHHandshake is an alias kept for the imds_test.go call site.
-func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) {
- waitForSSHReady(t, host, port, keyPath)
-}
-
-// trySSHReady mirrors waitForSSHReady but returns reachability as a bool
-// instead of t.Fatal-ing on timeout.
-func trySSHReady(host string, port int, keyPath string, timeout time.Duration) bool {
- if sshDatapathBroken.Load() {
- return false
- }
- deadline := time.Now().Add(timeout)
- for {
- cmd := exec.Command("ssh",
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=3",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(port),
- "-i", keyPath,
- "ubuntu@"+host,
- "true")
- if cmd.Run() == nil {
- return true
- }
- if time.Now().After(deadline) {
- return false
- }
- time.Sleep(2 * time.Second)
- }
-}
-
-// runSSH is a thin wrapper around `ssh` matching the option set used by
-// harness.LsblkRootGiB. Returns stdout. t.Fatal on non-zero exit so callers
-// can chain assertions on the output without nil-checking err.
-func runSSH(t *testing.T, tgt harness.SSHTarget, command string) string {
- t.Helper()
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- command,
- }
- var stdout, stderr bytes.Buffer
- cmd := exec.Command("ssh", args...)
- cmd.Stdout = &stdout
- cmd.Stderr = &stderr
- if err := cmd.Run(); err != nil {
- t.Fatalf("ssh %s@%s:%d %q failed: %v\nstderr: %s",
- tgt.User, tgt.Host, tgt.Port, command, err, stderr.String())
- }
- return stdout.String()
-}
-
-// runSSHCombined runs `command` over SSH against tgt and returns combined
-// stdout+stderr regardless of exit status. Unlike runSSH it does not t.Fatal
-// on non-zero exit — needed for probes (e.g. curl status codes) where a
-// non-zero exit is an expected outcome.
-func runSSHCombined(tgt harness.SSHTarget, command string) (string, error) {
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- command,
- }
- var combined bytes.Buffer
- cmd := exec.Command("ssh", args...)
- cmd.Stdout = &combined
- cmd.Stderr = &combined
- err := cmd.Run()
- return combined.String(), err
-}
+// sshHealth is this suite's sticky SSH-datapath verdict; see harness.SSHHealth.
+var sshHealth harness.SSHHealth
// primaryENI returns the NetworkInterfaceId of an instance's first ENI.
// t.Fatal if the instance has no ENI — every running EC2 instance must.
@@ -219,26 +32,6 @@ func primaryENI(t *testing.T, inst *ec2.Instance) string {
return eni
}
-// waitForInstanceStateSoft is the cleanup-time analogue of
-// harness.WaitForInstanceState — no t.Fatal, just polls and returns.
-func waitForInstanceStateSoft(c *harness.AWSClient, id, target string, timeout time.Duration) error {
- deadline := time.Now().Add(timeout)
- for {
- out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
- InstanceIds: []*string{aws.String(id)},
- })
- if err == nil && len(out.Reservations) > 0 && len(out.Reservations[0].Instances) > 0 {
- if aws.StringValue(out.Reservations[0].Instances[0].State.Name) == target {
- return nil
- }
- }
- if time.Now().After(deadline) {
- return fmt.Errorf("instance %s did not reach %s within %s", id, target, timeout)
- }
- time.Sleep(2 * time.Second)
- }
-}
-
// detectPoolMode reads external_mode from spinifex.toml. Defaults to false
// (dev_networking) which is the single-node CI fixture. Any non-empty
// external_mode value ("pool" / "nat") means external IPAM is in play, which
diff --git a/tests/e2e/iam/imds_test.go b/tests/e2e/iam/imds_test.go
index 9d35b5f84..dbbaa545b 100644
--- a/tests/e2e/iam/imds_test.go
+++ b/tests/e2e/iam/imds_test.go
@@ -104,7 +104,7 @@ type imdsCredDoc struct {
func runIMDS(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — IMDSv2 Host-Served Instance Metadata")
harness.SkipIfNoOVN(t)
- requireSSHHealthy(t)
+ sshHealth.Require(t)
adminAccount := harness.IAMAccountID(t, fix.AWS)
keyName, keyPath := needKeyPair(t, fix)
@@ -228,7 +228,7 @@ func runIMDS(t *testing.T, fix *Fixture) {
"public-hostname must be the AWS-shaped EC2 name for public-ipv4")
harness.Step(t, "resolve IMDS public-hostname %s from the guest", gotHost)
- resolved := runSSH(t, tgtX, "getent ahostsv4 "+gotHost)
+ resolved := harness.RunSSH(t, tgtX, "getent ahostsv4 "+gotHost)
require.Containsf(t, strings.Fields(resolved), pubIP,
"public-hostname %s did not resolve to public-ipv4 %s: %s", gotHost, pubIP, resolved)
} else {
@@ -420,7 +420,7 @@ func imdsProbe(t *testing.T, fix *Fixture, keyPath string, spec imdsVMSpec) (str
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(id)},
})
- _ = waitForInstanceStateSoft(fix.AWS, id, "terminated", 5*time.Minute)
+ _ = harness.WaitForInstanceStateSoft(fix.AWS, id, "terminated", 5*time.Minute)
})
inst := harness.WaitForInstanceState(t, fix.AWS, id, "running")
@@ -430,7 +430,7 @@ func imdsProbe(t *testing.T, fix *Fixture, keyPath string, spec imdsVMSpec) (str
host, port := harness.InstancePublicSSHHost(t, inst)
harness.Step(t, "wait for %s SSH at %s:%d", id, host, port)
- waitForSSHHandshake(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
return id, priv, eni, harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
}
@@ -540,34 +540,34 @@ func imdsAssertBootFromIMDS(t *testing.T, tgt harness.SSHTarget, privIP string)
t.Helper()
harness.Step(t, "boot-from-IMDS: cloud-init selected the aws (Ec2) datasource")
ds := strings.ToLower(strings.TrimSpace(
- runSSH(t, tgt, "cloud-id 2>/dev/null || cloud-init query --format '{{datasource}}'")))
+ harness.RunSSH(t, tgt, "cloud-id 2>/dev/null || cloud-init query --format '{{datasource}}'")))
require.Truef(t, strings.Contains(ds, "aws") || strings.Contains(ds, "ec2"),
"cloud-init must report the aws/Ec2 datasource (got %q) — not the retired NoCloud seed", ds)
harness.Step(t, "boot-from-IMDS: no NoCloud cidata seed device attached")
- labels := strings.ToLower(runSSH(t, tgt, "lsblk -no LABEL"))
+ labels := strings.ToLower(harness.RunSSH(t, tgt, "lsblk -no LABEL"))
require.NotContainsf(t, labels, "cidata",
"no cidata-labelled device may be attached — the seed ISO is retired (lsblk LABELs: %q)", labels)
harness.Step(t, "boot-from-IMDS: login is the AMI stock default user %q", tgt.User)
- require.Equal(t, tgt.User, strings.TrimSpace(runSSH(t, tgt, "id -un")),
+ require.Equal(t, tgt.User, strings.TrimSpace(harness.RunSSH(t, tgt, "id -un")),
"in-guest login must be the AMI stock default user, not a Spinifex-forced account")
harness.Step(t, "boot-from-IMDS: AWS-form hostname ip-")
wantHost := "ip-" + strings.ReplaceAll(privIP, ".", "-")
- gotHost := strings.TrimSpace(runSSH(t, tgt, "hostname"))
+ gotHost := strings.TrimSpace(harness.RunSSH(t, tgt, "hostname"))
require.Truef(t, strings.HasPrefix(gotHost, wantHost),
"hostname must be the AWS form %q rendered from IMDS local-hostname (got %q)", wantHost, gotHost)
harness.Step(t, "boot-from-IMDS: primary NIC up with the VPC IP %s", privIP)
- addrs := runSSH(t, tgt, "ip -4 -o addr show scope global")
+ addrs := harness.RunSSH(t, tgt, "ip -4 -o addr show scope global")
require.Containsf(t, addrs, privIP,
"the Ec2 datasource must render the primary NIC with the VPC IP %s from IMDS (ip addr: %q)", privIP, addrs)
harness.Step(t, "boot-from-IMDS: user-data runcmd executed")
deadline := time.Now().Add(90 * time.Second)
for {
- out, _ := runSSHCombined(tgt, "cat "+imdsUDDoneFile+" 2>/dev/null")
+ out, _ := harness.RunSSHCombined(tgt, "cat "+imdsUDDoneFile+" 2>/dev/null")
if strings.Contains(out, imdsUDMarker) {
break
}
@@ -777,7 +777,7 @@ func imdsAwaitToken(t *testing.T, fix *Fixture, tgt harness.SSHTarget, subnetID,
for {
cmd := fmt.Sprintf(
`curl -sf -X PUT "%s/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 120"`, metaURL)
- out, err := runSSHCombined(tgt, cmd)
+ out, err := harness.RunSSHCombined(tgt, cmd)
switch {
case err != nil:
lastErr = fmt.Errorf("token PUT failed: %w (out=%q)", err, out)
@@ -797,11 +797,11 @@ func imdsAwaitToken(t *testing.T, fix *Fixture, tgt harness.SSHTarget, subnetID,
// imdsGet fetches a token-gated path from inside the guest and returns the
// trimmed body. -sf turns any HTTP >= 400 into a non-zero exit, so a 4xx/5xx
-// fails the test loudly with the server status visible via runSSH's stderr.
+// fails the test loudly with the server status visible via harness.RunSSH's stderr.
func imdsGet(t *testing.T, tgt harness.SSHTarget, token, path string) string {
t.Helper()
cmd := fmt.Sprintf(`curl -sf -H "X-aws-ec2-metadata-token: %s" %s%s`, token, metaURL, path)
- return strings.TrimSpace(runSSH(t, tgt, cmd))
+ return strings.TrimSpace(harness.RunSSH(t, tgt, cmd))
}
// imdsExpectedPubKey derives the OpenSSH public key (type + base64) the IMDS
@@ -826,7 +826,7 @@ func imdsExpectedPubKey(t *testing.T, pemPath string) (keyType, base64Key string
// argument (empty for the tokenless probe).
func imdsCode(tgt harness.SSHTarget, headerArg, path string) string {
cmd := fmt.Sprintf(`curl -s -o /dev/null -w '%%{http_code}' %s %s%s`, headerArg, metaURL, path)
- out, _ := runSSHCombined(tgt, cmd)
+ out, _ := harness.RunSSHCombined(tgt, cmd)
return strings.TrimSpace(out)
}
diff --git a/tests/e2e/kvquorum/main_test.go b/tests/e2e/kvquorum/main_test.go
new file mode 100644
index 000000000..5a534864d
--- /dev/null
+++ b/tests/e2e/kvquorum/main_test.go
@@ -0,0 +1,123 @@
+//go:build e2e
+
+// Package kvquorum proves that the cluster's own state survives losing a node.
+//
+// Every control-plane service keeps its state in a JetStream KV bucket, and a
+// bucket is replicated across however many nodes it was created with. At one
+// replica it lives on one JetStream-chosen server recorded in no config, so
+// losing that node makes the bucket unreadable and unwritable from every node
+// at once. Leader leases are the worst case: nobody can acquire one whose
+// bucket has no quorum, so every reconciler sharing it stops cluster-wide at
+// exactly the moment a node has failed.
+//
+// It is its own suite because it takes a node away, so anything sharing the
+// environment would report that outage as its own failure, and because the
+// question it answers — "is our internal state actually replicated" — has to
+// read as itself rather than as one more red subtest somewhere else.
+package kvquorum
+
+import (
+ "fmt"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/mulgadc/spinifex/tests/e2e/harness"
+)
+
+const (
+ // How long a survivor is given to answer for every bucket after a node
+ // goes away. A raft group only has to elect a new leader for the streams
+ // the departed node led, which is seconds on a healthy cluster; the budget
+ // is generous because a failure here is the whole point of the suite and a
+ // flake would be read as one.
+ quorumRecoveryBudget = 2 * time.Minute
+
+ // How long the returning node is given to rejoin and carry its share again.
+ rejoinBudget = 5 * time.Minute
+
+ // How often a survivor is polled while quorum is expected to return.
+ pollInterval = 5 * time.Second
+)
+
+var (
+ pkgFixOnce sync.Once
+ pkgFix *Fixture
+ pkgFixErr error
+)
+
+func TestMain(m *testing.M) {
+ code := m.Run()
+ if pkgFix != nil && pkgFix.Harness != nil {
+ if err := pkgFix.Harness.Close(); err != nil {
+ fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err)
+ code = 1
+ }
+ }
+ os.Exit(code)
+}
+
+// Fixture carries per-process state shared across this package's tests.
+type Fixture struct {
+ Env *harness.Env
+ AWS *harness.AWSClient
+ Harness *harness.Fixture
+ Cluster *harness.Cluster
+}
+
+// requireFixture returns the package-scoped Fixture, building it on first call.
+//
+// Three nodes is the floor. On two, a majority is two, so losing either one
+// leaves no quorum for any stream and there is nothing to assert beyond "raft
+// needs a majority" — which is true of NATS and not a property of ours.
+func requireFixture(t *testing.T) *Fixture {
+ t.Helper()
+ pkgFixOnce.Do(func() {
+ if os.Getenv("SPINIFEX_E2E") == "" {
+ return
+ }
+ env := harness.LoadEnv(t)
+ cluster, err := harness.ClusterFromEnv()
+ if err != nil {
+ pkgFixErr = fmt.Errorf("this suite needs node SSH to take a node away: %w", err)
+ return
+ }
+ awsCli := harness.NewAWSClient(t, env)
+ h, err := harness.NewProcessFixture(awsCli)
+ if err != nil {
+ pkgFixErr = err
+ return
+ }
+ pkgFix = &Fixture{Env: env, AWS: awsCli, Harness: h, Cluster: cluster}
+ })
+ if pkgFixErr != nil {
+ t.Fatalf("kvquorum fixture init: %v", pkgFixErr)
+ }
+ if pkgFix == nil {
+ t.Skip("SPINIFEX_E2E unset")
+ }
+ if n := len(pkgFix.Cluster.Nodes); n < 3 {
+ t.Skipf("quorum under node loss needs a surviving majority, have %d node(s)", n)
+ }
+ return pkgFix
+}
+
+// TestKVBucketsAreReplicatedAcrossTheCluster is read-only and runs first, so a
+// cluster that is already wrong is reported as wrong rather than as a failure
+// to survive a node loss.
+func TestKVBucketsAreReplicatedAcrossTheCluster(t *testing.T) {
+ runBucketsAreReplicated(t, requireFixture(t))
+}
+
+// TestKVReplicasCommandReportsHealthy proves the operator surface agrees with
+// the cluster, on every node, and that its exit status can gate a deployment.
+func TestKVReplicasCommandReportsHealthy(t *testing.T) {
+ runReplicasCommand(t, requireFixture(t))
+}
+
+// TestKVBucketsSurviveANodeLoss is the assertion the suite exists for, and the
+// only one that produces the fault rather than inspecting for it.
+func TestKVBucketsSurviveANodeLoss(t *testing.T) {
+ runBucketsSurviveANodeLoss(t, requireFixture(t))
+}
diff --git a/tests/e2e/kvquorum/node_loss_test.go b/tests/e2e/kvquorum/node_loss_test.go
new file mode 100644
index 000000000..0dcb42008
--- /dev/null
+++ b/tests/e2e/kvquorum/node_loss_test.go
@@ -0,0 +1,187 @@
+//go:build e2e
+
+package kvquorum
+
+import (
+ "fmt"
+ "maps"
+ "slices"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ natssvc "github.com/mulgadc/spinifex/spinifex/services/nats"
+ "github.com/mulgadc/spinifex/tests/e2e/harness"
+ "github.com/stretchr/testify/require"
+)
+
+// runBucketsSurviveANodeLoss produces the fault rather than inspecting for it.
+//
+// The node taken away is the one leading the most buckets, which is the worst
+// one to lose and the one a test that picked arbitrarily would usually miss.
+// On a correctly replicated cluster the survivors elect new leaders and keep
+// answering; on a cluster of single-replica buckets, every bucket that lived
+// on that node answers "no responders" from every node at once, which is the
+// outage this whole change exists to remove.
+func runBucketsSurviveANodeLoss(t *testing.T, fix *Fixture) {
+ harness.Phase(t, "KV quorum — the cluster's state survives losing a node")
+
+ nodes := len(fix.Cluster.Nodes)
+ want := harness.WantKVReplicas(nodes)
+
+ harness.Step(t, "audit the cluster before anything is broken")
+ before := harness.KVReplication(t, fix.Env, nodes)
+ require.NotEmpty(t, before, "no KV buckets to lose")
+ harness.RequireKVQuorum(t, fix.Env, nodes)
+
+ victim, led := busiestLeader(fix.Cluster.Nodes, before)
+ require.NotZerof(t, victim.Index,
+ "no bucket named a leader this test can map to a node; peers were %s", peerNames(before))
+ harness.Detail(t, "victim", victim.Name, "buckets_led", led, "buckets_total", len(before))
+
+ survivors := fix.Cluster.Peers(victim)
+
+ harness.Step(t, "take %s away", victim.Name)
+ // Registered before the stop, so a panic or a cancelled run still brings
+ // the node back rather than leaving the cluster a node short.
+ t.Cleanup(func() {
+ harness.StartNode(t, victim)
+ harness.WaitNodeServiceReady(t, victim, harness.WithTimeout(rejoinBudget))
+ })
+ harness.StopNode(t, victim)
+
+ harness.Step(t, "every survivor must still answer for every bucket")
+ for _, node := range survivors {
+ requireNodeSeesEveryBucket(t, node, before, want)
+ }
+
+ harness.Step(t, "bring %s back", victim.Name)
+ harness.StartNode(t, victim)
+ harness.WaitNodeServiceReady(t, victim, harness.WithTimeout(rejoinBudget))
+ fix.Cluster.WaitNATSPeers(t, nodes-1, harness.WithTimeout(rejoinBudget))
+
+ harness.Step(t, "the returning node must carry its share again")
+ for _, node := range fix.Cluster.Nodes {
+ requireNodeSeesEveryBucket(t, node, before, want)
+ }
+
+ // Configured replicas must be unchanged. A cluster that repaired itself by
+ // lowering a count would pass every assertion above and have quietly traded
+ // the durability this suite is about for a green run.
+ harness.Step(t, "no bucket lost replicas on the way through")
+ after := harness.KVReplication(t, fix.Env, nodes)
+ requireNoBucketLostReplicas(t, before, after)
+ harness.RequireKVQuorum(t, fix.Env, nodes)
+}
+
+// requireNodeSeesEveryBucket polls a node until its own audit names every
+// bucket the cluster had before the outage, with none under-replicated.
+//
+// Polling rather than asserting once: a raft group whose leader has gone has to
+// elect a new one, and that is the interval this test is required to tolerate.
+// What it is not required to tolerate is a bucket that never comes back.
+func requireNodeSeesEveryBucket(t *testing.T, node harness.Node, before []kvutil.BucketReport, want int) {
+ t.Helper()
+ wantBuckets := bucketNames(before)
+
+ harness.EventuallyErr(t, func() error {
+ reports, err := nodeReplicaReport(t, node)
+ if err != nil {
+ return fmt.Errorf("%s: %w", node.Name, err)
+ }
+ if missing := missingFrom(wantBuckets, bucketNames(reports)); len(missing) > 0 {
+ return fmt.Errorf("%s cannot read %d bucket(s): %s",
+ node.Name, len(missing), strings.Join(missing, ", "))
+ }
+ if under := underReplicated(reports, want); len(under) > 0 {
+ return fmt.Errorf("%s reports %d under-replicated bucket(s): %s",
+ node.Name, len(under), strings.Join(under, ", "))
+ }
+ return nil
+ }, quorumRecoveryBudget, pollInterval)
+
+ harness.Detail(t, "node", node.Name, "buckets_readable", len(wantBuckets))
+}
+
+// requireNoBucketLostReplicas fails if any bucket came back with fewer replicas
+// than it had, which is a silent downgrade rather than a repair.
+func requireNoBucketLostReplicas(t *testing.T, before, after []kvutil.BucketReport) {
+ t.Helper()
+ was := make(map[string]int, len(before))
+ for _, r := range before {
+ was[r.Bucket] = r.Replicas
+ }
+ for _, r := range after {
+ if prev, ok := was[r.Bucket]; ok && r.Replicas < prev {
+ t.Fatalf("KV bucket %s went from %d replicas to %d across the outage", r.Bucket, prev, r.Replicas)
+ }
+ }
+}
+
+// busiestLeader returns the node leading the most buckets, and how many.
+// Leadership is where the loss hurts most: a follower going away costs a raft
+// group a peer, a leader going away costs it an election.
+//
+// The leader is read from the field that holds it rather than guessed at the head
+// of the peer list. It is a NATS server name, which is the node name behind a
+// prefix rather than the node name itself, so it goes through the helper that
+// owns that prefix.
+func busiestLeader(nodes []harness.Node, reports []kvutil.BucketReport) (harness.Node, int) {
+ led := make(map[string]int, len(nodes))
+ for _, r := range reports {
+ if node, ok := natssvc.NodeFromServerName(r.Leader); ok {
+ led[node]++
+ }
+ }
+
+ var best harness.Node
+ bestN := -1
+ for _, n := range nodes {
+ if c := led[n.Name]; c > bestN {
+ best, bestN = n, c
+ }
+ }
+ if bestN <= 0 {
+ return harness.Node{}, 0
+ }
+ return best, bestN
+}
+
+// bucketNames returns the sorted bucket names in a report set.
+func bucketNames(reports []kvutil.BucketReport) []string {
+ out := make([]string, 0, len(reports))
+ for _, r := range reports {
+ out = append(out, r.Bucket)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// missingFrom returns every name in want that is absent from got.
+func missingFrom(want, got []string) []string {
+ have := make(map[string]struct{}, len(got))
+ for _, g := range got {
+ have[g] = struct{}{}
+ }
+ var out []string
+ for _, w := range want {
+ if _, ok := have[w]; !ok {
+ out = append(out, w)
+ }
+ }
+ return out
+}
+
+// peerNames is diagnostic only: it says what the audit did report when no
+// leader could be mapped to a node, which is otherwise an opaque failure.
+func peerNames(reports []kvutil.BucketReport) string {
+ seen := map[string]struct{}{}
+ for _, r := range reports {
+ for _, p := range r.Peers {
+ seen[p] = struct{}{}
+ }
+ }
+ out := slices.Sorted(maps.Keys(seen))
+ return strings.Join(out, ",")
+}
diff --git a/tests/e2e/kvquorum/replication_test.go b/tests/e2e/kvquorum/replication_test.go
new file mode 100644
index 000000000..b0a82ce2a
--- /dev/null
+++ b/tests/e2e/kvquorum/replication_test.go
@@ -0,0 +1,148 @@
+//go:build e2e
+
+package kvquorum
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ "github.com/mulgadc/spinifex/tests/e2e/harness"
+ "github.com/stretchr/testify/require"
+)
+
+// replicasCmd is the operator surface under test. No --config: on a node the
+// command finds /etc/spinifex/spinifex.toml itself, which is how anyone
+// gating a deployment on it would actually run it.
+const replicasCmd = "sudo spx admin kv replicas --json"
+
+// runBucketsAreReplicated sweeps every KV bucket in the cluster rather than a
+// named list. A replica count is set where a bucket is created, and there is a
+// bucket-creating call in most services, so a named list only ever covers the
+// ones somebody remembered.
+func runBucketsAreReplicated(t *testing.T, fix *Fixture) {
+ harness.Phase(t, "KV quorum — every bucket is replicated across the cluster")
+
+ nodes := len(fix.Cluster.Nodes)
+ want := harness.WantKVReplicas(nodes)
+ harness.Detail(t, "cluster_nodes", nodes, "want_replicas", want)
+
+ harness.Step(t, "audit every KV bucket")
+ reports := harness.KVReplication(t, fix.Env, nodes)
+ require.NotEmpty(t, reports, "the cluster reports no KV buckets at all, so nothing was checked")
+
+ for _, r := range reports {
+ harness.Detail(t, "bucket", r.Bucket, "replicas", r.Replicas, "held_by", strings.Join(r.Peers, ","))
+ }
+ harness.RequireKVQuorum(t, fix.Env, nodes)
+
+ // The leader-lease buckets are named as well as swept. They were the worst
+ // case of the defect this suite guards — created with no replica count at
+ // all — and a regression that reached only them would otherwise be one more
+ // line in a long list rather than the headline.
+ harness.Step(t, "confirm the leader-lease buckets are among those checked")
+ leases := leaseBuckets(reports)
+ require.NotEmpty(t, leases,
+ "no leader-lease bucket was found; either the cluster has never run a reconciler or the audit missed them")
+ harness.Detail(t, "lease_buckets", strings.Join(leases, ","))
+}
+
+// leaseBuckets returns the names of the reconciler leader-lease buckets, which
+// are the ones a reconciler must acquire before it can do anything at all.
+func leaseBuckets(reports []kvutil.BucketReport) []string {
+ var out []string
+ for _, r := range reports {
+ if strings.HasSuffix(r.Bucket, "-leader") || strings.HasSuffix(r.Bucket, "-reconcile") {
+ out = append(out, r.Bucket)
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+// runReplicasCommand proves the operator surface on every node, not one. The
+// command reads cluster-global raft metadata, so every node must give the same
+// answer — a node that disagrees is reporting on a cluster it is not fully part
+// of, which is worth knowing before anything is gated on its exit status.
+func runReplicasCommand(t *testing.T, fix *Fixture) {
+ harness.Phase(t, "KV quorum — spx admin kv replicas agrees on every node")
+
+ want := harness.WantKVReplicas(len(fix.Cluster.Nodes))
+
+ counts := make(map[string]int, len(fix.Cluster.Nodes))
+ for _, node := range fix.Cluster.Nodes {
+ harness.Step(t, "run the replica audit on %s", node.Name)
+ reports, err := nodeReplicaReport(t, node)
+ require.NoErrorf(t, err,
+ "%s could not audit its own KV buckets, so nothing can be gated on its exit status", node.Name)
+
+ under := underReplicated(reports, want)
+ harness.Detail(t, "node", node.Name, "buckets", len(reports), "under_replicated", len(under))
+ require.Emptyf(t, under,
+ "%s reports %d under-replicated bucket(s): %s", node.Name, len(under), strings.Join(under, ", "))
+ counts[node.Name] = len(reports)
+ }
+
+ harness.Step(t, "confirm every node sees the same set of buckets")
+ var first string
+ for _, node := range fix.Cluster.Nodes {
+ if first == "" {
+ first = node.Name
+ continue
+ }
+ require.Equalf(t, counts[first], counts[node.Name],
+ "%s sees %d KV buckets and %s sees %d; the two are not looking at the same cluster",
+ first, counts[first], node.Name, counts[node.Name])
+ }
+}
+
+// nodeReplicaReport runs the replica audit on one node and decodes it. A
+// non-zero exit is returned as an error rather than fatalled, because callers
+// polling a recovering cluster need to retry it.
+func nodeReplicaReport(t *testing.T, node harness.Node) ([]kvutil.BucketReport, error) {
+ t.Helper()
+ ssh := harness.NewPeerSSH()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ out, err := ssh.Run(ctx, node.Addr, replicasCmd)
+ if err != nil {
+ // The command exits 1 when a bucket is under-replicated, and it still
+ // prints its report, so the output is decoded either way — a report
+ // naming the bucket is a better failure than "exit status 1".
+ if len(out) == 0 {
+ return nil, fmt.Errorf("%s: %w", node.Name, err)
+ }
+ }
+ var reports []kvutil.BucketReport
+ if jsonErr := json.Unmarshal(trimToJSON(out), &reports); jsonErr != nil {
+ return nil, fmt.Errorf("%s: decode replica report (%q): %w", node.Name, string(out), jsonErr)
+ }
+ return reports, nil
+}
+
+// trimToJSON drops anything before the JSON document, since sudo and the login
+// shell can both write to the same stream.
+func trimToJSON(out []byte) []byte {
+ if i := strings.IndexAny(string(out), "[{"); i > 0 {
+ return out[i:]
+ }
+ return out
+}
+
+// underReplicated names every bucket below want replicas.
+func underReplicated(reports []kvutil.BucketReport, want int) []string {
+ var out []string
+ for _, r := range reports {
+ if r.Replicas < want {
+ out = append(out, fmt.Sprintf("%s(%d/%d)", r.Bucket, r.Replicas, want))
+ }
+ }
+ sort.Strings(out)
+ return out
+}
diff --git a/tests/e2e/manifest-lint/lint.go b/tests/e2e/manifest-lint/lint.go
index 1946947e9..ed2c1218a 100644
--- a/tests/e2e/manifest-lint/lint.go
+++ b/tests/e2e/manifest-lint/lint.go
@@ -21,7 +21,9 @@ package manifestlint
import (
"bufio"
+ "maps"
"os"
+ "slices"
"sort"
"strings"
)
@@ -75,11 +77,7 @@ func WriteBaseline(path string, keys []string) error {
for _, k := range keys {
uniq[k] = true
}
- out := make([]string, 0, len(uniq))
- for k := range uniq {
- out = append(out, k)
- }
- sort.Strings(out)
+ out := slices.Sorted(maps.Keys(uniq))
var b strings.Builder
b.WriteString("# manifest-lint baseline — pre-existing drift accepted by the ratchet.\n")
diff --git a/tests/e2e/manifest-lint/subjects.go b/tests/e2e/manifest-lint/subjects.go
index d6479e950..4e445a08d 100644
--- a/tests/e2e/manifest-lint/subjects.go
+++ b/tests/e2e/manifest-lint/subjects.go
@@ -5,9 +5,11 @@ import (
"go/ast"
"go/parser"
"go/token"
+ "maps"
"os"
"path/filepath"
"regexp"
+ "slices"
"sort"
"strconv"
"strings"
@@ -137,12 +139,7 @@ func serviceDirs(m *manifestcheck.Manifest) []string {
set[p] = true
}
}
- out := make([]string, 0, len(set))
- for d := range set {
- out = append(out, d)
- }
- sort.Strings(out)
- return out
+ return slices.Sorted(maps.Keys(set))
}
// scanSubjectsInDir walks dir recursively, parsing every non-test .go file and
diff --git a/tests/e2e/multinode/jetstream_replicas_test.go b/tests/e2e/multinode/jetstream_replicas_test.go
index 48573dece..e91977fe5 100644
--- a/tests/e2e/multinode/jetstream_replicas_test.go
+++ b/tests/e2e/multinode/jetstream_replicas_test.go
@@ -11,34 +11,49 @@ import (
"github.com/stretchr/testify/require"
)
-// jetStreamReplicaBuckets are the daemon-owned KV buckets created at replicas=1
-// during initJetStream and bumped to the cluster size by upgradeJetStreamReplicas.
-// A regression there leaves them stuck at 1, silently losing EC2/cluster state
-// on a single node loss — exactly what this guards (the upgrade path is 0% unit).
+// jetStreamReplicaBuckets are the daemon-owned KV buckets. They are named
+// explicitly as well as swept, so a run that somehow enumerates nothing still
+// fails rather than passing on an empty set.
var jetStreamReplicaBuckets = []string{
daemon.InstanceStateBucket,
daemon.ClusterStateBucket,
daemon.TerminatedInstanceBucket,
}
-// runJetStreamReplicas asserts every daemon KV bucket's configured replica factor
-// matches the cluster node count, proving upgradeJetStreamReplicas ran end-to-end
-// (the assembled NATS path no unit test reaches). Read-only over NATS.
+// runJetStreamReplicas asserts that every KV bucket in the cluster is
+// replicated across every node, not just the three the daemon owns.
+//
+// The sweep is the assertion that matters. A replica count is set where a
+// bucket is created, and there is a bucket-creating call in most services, so
+// naming buckets here would only ever cover the ones somebody remembered. A
+// bucket below the node count lives on fewer nodes than depend on it: losing
+// one of them makes it unreadable from every node at once, which is how a
+// single-node event becomes a cluster-wide outage.
func runJetStreamReplicas(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — JetStream KV Replica Factor")
- want := len(fix.Cluster.Nodes)
- if want < 2 {
- t.Skipf("cluster has %d node(s); replica upgrade is a no-op below 2", want)
+ nodes := len(fix.Cluster.Nodes)
+ if nodes < 2 {
+ t.Skipf("cluster has %d node(s); replication across nodes is a no-op below 2", nodes)
}
- harness.Detail(t, "cluster_nodes", want, "buckets", strings.Join(jetStreamReplicaBuckets, ","))
+ want := harness.WantKVReplicas(nodes)
+ harness.Detail(t, "cluster_nodes", nodes, "want_replicas", want)
- harness.Step(t, "introspect KV stream replica factors")
+ harness.Step(t, "introspect the daemon's own KV stream replica factors")
got := harness.KVReplicaFactors(t, fix.Env, jetStreamReplicaBuckets...)
for _, b := range jetStreamReplicaBuckets {
harness.Detail(t, "bucket", b, "replicas", got[b])
require.Equalf(t, want, got[b],
- "KV bucket %s replicas=%d want=%d — upgradeJetStreamReplicas did not match cluster size; a single node loss would lose this bucket",
+ "KV bucket %s replicas=%d want=%d — a single node loss would take this bucket down cluster-wide",
b, got[b], want)
}
+
+ harness.Step(t, "sweep every KV bucket in the cluster")
+ reports := harness.KVReplication(t, fix.Env, nodes)
+ names := make([]string, 0, len(reports))
+ for _, r := range reports {
+ names = append(names, r.Bucket)
+ }
+ harness.Detail(t, "buckets", len(reports), "names", strings.Join(names, ","))
+ harness.RequireKVQuorum(t, fix.Env, nodes)
}
diff --git a/tests/e2e/single/createimage_test.go b/tests/e2e/single/createimage_test.go
index 29de465ab..69c0a2332 100644
--- a/tests/e2e/single/createimage_test.go
+++ b/tests/e2e/single/createimage_test.go
@@ -145,10 +145,10 @@ func runBootVerifyAMI(t *testing.T, fix *Fixture, amiID, label string) {
probeInst := harness.WaitForInstanceState(t, fix.AWS, probeInstanceID, "running")
host, port := harness.InstancePublicSSHHost(t, probeInst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
assert.Containsf(t, idOut, "ubuntu", "boot-verify %s: ssh id should report ubuntu\n%s", label, idOut)
harness.Detail(t, "boot_verify_"+label, "ok")
diff --git a/tests/e2e/single/datapath_test.go b/tests/e2e/single/datapath_test.go
index 63d049c92..b8dab3f72 100644
--- a/tests/e2e/single/datapath_test.go
+++ b/tests/e2e/single/datapath_test.go
@@ -105,7 +105,7 @@ func sgDatapathRevokeRounds() int {
func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — Security Group Policy on a Real Datapath (OVN)")
harness.SkipIfNoOVN(t)
- requireSSHHealthy(t)
+ sshHealth.Require(t)
// Bootstrap every prereq up front. runSGEInstance / primaryENI use
// `fix.AMIID / InstanceType / KeyName / KeyPath` indirectly; resolve
@@ -257,7 +257,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
// Non-fatal probe so a timeout dumps the guest console + OVN/datapath
// state before Fatal. A full 2min unreachable window on a fresh public-IP
// VM is the flake signature; capture it from CI artifacts alone.
- if !trySSHReady(clientHost, clientPort, keyPath, 2*time.Minute) {
+ if !sshHealth.TryReady(clientHost, clientPort, keyPath, 2*time.Minute) {
harness.DumpVPCFlowDiagnostics(t, fix.AWS, clientID,
fmt.Sprintf("8e-3 client-vm SSH timeout — vpc=%s sg=%s pub=%s", def.VPCID, clientSG, clientHost),
harness.VPCDiagnosticsOpts{
@@ -326,7 +326,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
// so the only variable left is ovn-controller installing the
// chassis flows. Bash used 30 attempts at 2s; keep that budget.
harness.EventuallyErr(t, func() error {
- out, err := runSSHCombined(clientTgt, curlCmd)
+ out, err := harness.RunSSHCombined(clientTgt, curlCmd)
if err != nil {
return fmt.Errorf("client -> target:8080 failed: %w (out=%q)", err, out)
}
@@ -343,7 +343,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
// merits even if the 8080 path above failed.
t.Run("DeniedTraffic", func(t *testing.T) {
harness.Step(t, "8e-5 denied traffic client -> target:22 (no SSH ingress)")
- if _, err := runSSHCombined(clientTgt, fmt.Sprintf("nc -z -w 5 %s 22", targetPriv)); err == nil {
+ if _, err := harness.RunSSHCombined(clientTgt, fmt.Sprintf("nc -z -w 5 %s 22", targetPriv)); err == nil {
t.Fatalf("FAIL: client reached target:22 — default-deny ACL not enforced")
}
harness.Detail(t, "step5", "denied_traffic_ok")
@@ -449,7 +449,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
derived, clientLL, iface)
targetLL := linkLocalFromMAC(eniMAC(t, fix, targetENI)).String()
- out, err := runSSHCombined(clientTgt, fmt.Sprintf("ping -6 -c 3 -W 3 %s%%%s", targetLL, iface))
+ out, err := harness.RunSSHCombined(clientTgt, fmt.Sprintf("ping -6 -c 3 -W 3 %s%%%s", targetLL, iface))
require.Errorf(t, err, "FAIL: client reached target over IPv6 at %s\n%s", targetLL, out)
// A non-zero exit alone also covers a broken ssh session, a missing
@@ -498,7 +498,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
// Fresh TCP connection — conntrack does not affect new
// connections. Bash treats a single curl success as failure.
harness.Step(t, "8e-6 verify client -> target:8080 now blocked")
- if _, err := runSSHCombined(clientTgt, curlCmd); err == nil {
+ if _, err := harness.RunSSHCombined(clientTgt, curlCmd); err == nil {
t.Fatalf("FAIL: client still reached target:8080 after revoke — propagation not immediate")
}
harness.Detail(t, "step6", "revoke_blocked_ok")
@@ -529,7 +529,7 @@ func runSGPolicyDatapath(t *testing.T, fix *Fixture) {
require.NoError(t, err, "re-authorize tcp/8080 from %s", clientSG)
harness.EventuallyErr(t, func() error {
- out, err := runSSHCombined(clientTgt, curlCmd)
+ out, err := harness.RunSSHCombined(clientTgt, curlCmd)
if err != nil {
return fmt.Errorf("client -> target:8080 still blocked after re-add: %w (out=%q)", err, out)
}
@@ -706,12 +706,12 @@ func linkLocalFromMAC(mac net.HardwareAddr) net.IP {
// link-local address configured on it.
func guestLinkLocal(t *testing.T, tgt harness.SSHTarget) (string, string) {
t.Helper()
- iface, err := runSSHCombined(tgt, "ip -o -4 route show default | awk '{print $5; exit}'")
+ iface, err := harness.RunSSHCombined(tgt, "ip -o -4 route show default | awk '{print $5; exit}'")
require.NoError(t, err, "guest could not report its default-route interface")
iface = strings.TrimSpace(iface)
require.NotEmpty(t, iface, "guest has no default-route interface")
- ll, err := runSSHCombined(tgt,
+ ll, err := harness.RunSSHCombined(tgt,
fmt.Sprintf("ip -6 -o addr show dev %s scope link | awk '{print $4; exit}' | cut -d/ -f1", iface))
require.NoError(t, err, "guest could not report its link-local address")
ll = strings.TrimSpace(ll)
diff --git a/tests/e2e/single/guestchurn_test.go b/tests/e2e/single/guestchurn_test.go
index ae2a62647..ee3da6d92 100644
--- a/tests/e2e/single/guestchurn_test.go
+++ b/tests/e2e/single/guestchurn_test.go
@@ -4,8 +4,10 @@ package single
import (
"fmt"
+ "maps"
"net"
"os/exec"
+ "slices"
"strconv"
"strings"
"testing"
@@ -118,13 +120,13 @@ func runGuestChurnDurability(t *testing.T, fix *Fixture) {
addr := net.JoinHostPort(host, strconv.Itoa(port))
harness.Step(t, "waiting for SSH handshake %s", addr)
- waitForSSHHandshake(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
_ = addr
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
harness.Step(t, "ssh id")
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
assert.Containsf(t, idOut, "ubuntu", "ssh id should report ubuntu\n%s", idOut)
harness.Step(t, "lsblk root-volume cross-check vs API")
@@ -149,7 +151,7 @@ func runGuestChurnDurability(t *testing.T, fix *Fixture) {
harness.Detail(t, "guest_gib", guestGiB, "api_gib", apiGiB)
harness.Step(t, "ssh hostname")
- hn := strings.TrimSpace(runSSH(t, tgt, "hostname"))
+ hn := strings.TrimSpace(harness.RunSSH(t, tgt, "hostname"))
// Bash uses `spinifex-vm-` and treats
// a missing prefix as a non-fatal warning. Replicate the soft check via
// t.Logf rather than asserting — the spx hostname format is not part of
@@ -249,7 +251,7 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
t.Run("HotplugENI", func(t *testing.T) {
harness.SkipIfNoOVN(t)
- requireSSHHealthy(t)
+ sshHealth.Require(t)
tgt := resolveGuestSSHTarget(t, fix, instanceID, keyPath)
def := harness.EnsureDefaultVPC(t, fix.Harness)
@@ -473,11 +475,11 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
host, port := harness.InstancePublicSSHHost(t, runInst)
harness.Detail(t, "ssh_host", host, "ssh_port", port)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
harness.Step(t, "ssh nproc")
- nprocOut := strings.TrimSpace(runSSH(t, tgt, "nproc"))
+ nprocOut := strings.TrimSpace(harness.RunSSH(t, tgt, "nproc"))
vmVCPUs, err := strconv.ParseInt(nprocOut, 10, 64)
require.NoErrorf(t, err, "parse nproc output %q", nprocOut)
require.Equalf(t, expectedVCPUs, vmVCPUs,
@@ -485,7 +487,7 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
harness.Detail(t, "vm_vcpus", vmVCPUs)
harness.Step(t, "ssh MemTotal")
- memOut := strings.TrimSpace(runSSH(t, tgt, "awk '/MemTotal/ {print $2}' /proc/meminfo"))
+ memOut := strings.TrimSpace(harness.RunSSH(t, tgt, "awk '/MemTotal/ {print $2}' /proc/meminfo"))
vmMemKB, err := strconv.ParseInt(memOut, 10, 64)
require.NoErrorf(t, err, "parse MemTotal output %q", memOut)
vmMemMiB := vmMemKB / 1024
@@ -520,7 +522,7 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
runInst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
host, port := harness.InstancePublicSSHHost(t, runInst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
gotSha := harness.GuestReadSentinelSha(t, tgt, "/dev/disk/by-label/"+guestChurnSentinelLabel, guestChurnSentinelLabel)
require.Equalf(t, wantSha, gotSha, "sha256 mismatch after stop/start")
@@ -530,12 +532,12 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
t.Run("Reboot", func(t *testing.T) {
inst := describeSingletonInstance(t, fix, instanceID)
host, port := harness.InstancePublicSSHHost(t, inst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
// The boot ID is what proves a reboot happened. Uptime cannot on its
// own: a reboot is asynchronous, so the guest answering shortly after
// the call may still be the one that has not shut down yet.
- preBootID := strings.TrimSpace(runSSH(t,
+ preBootID := strings.TrimSpace(harness.RunSSH(t,
harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath},
"cat /proc/sys/kernel/random/boot_id"))
require.NotEmpty(t, preBootID, "guest returned an empty boot ID before the reboot")
@@ -579,7 +581,7 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
require.NotEmpty(t, descPost.Reservations[0].Instances, "no instances post-reboot")
postInst := descPost.Reservations[0].Instances[0]
host, port = harness.InstancePublicSSHHost(t, postInst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
waitForNewBootID(t, tgt, preBootID)
@@ -587,7 +589,7 @@ func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPat
// Now that the guest is known to be on a new boot, uptime is a second
// reading of the same fact rather than a guess at it.
harness.Step(t, "ssh uptime")
- uptimeOut := strings.TrimSpace(runSSH(t, tgt, "cat /proc/uptime | cut -d. -f1"))
+ uptimeOut := strings.TrimSpace(harness.RunSSH(t, tgt, "cat /proc/uptime | cut -d. -f1"))
uptimeSecs, err := strconv.ParseInt(uptimeOut, 10, 64)
require.NoErrorf(t, err, "parse uptime output %q", uptimeOut)
require.LessOrEqualf(t, uptimeSecs, int64(120),
@@ -665,7 +667,7 @@ func resolveGuestSSHTarget(t *testing.T, fix *Fixture, instanceID, keyPath strin
t.Helper()
inst := describeSingletonInstance(t, fix, instanceID)
host, port := harness.InstancePublicSSHHost(t, inst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
return harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
}
@@ -692,11 +694,7 @@ func guestMACSet(t *testing.T, tgt harness.SSHTarget) map[string]struct{} {
// setKeys returns the keys of a string set as a slice for log/Detail output.
func setKeys(m map[string]struct{}) []string {
- keys := make([]string, 0, len(m))
- for k := range m {
- keys = append(keys, k)
- }
- return keys
+ return slices.Collect(maps.Keys(m))
}
// describeENI returns the single ENI record for eniID, erroring if absent.
diff --git a/tests/e2e/single/imagecatalog_test.go b/tests/e2e/single/imagecatalog_test.go
index 2bb2f36c3..2ee348a16 100644
--- a/tests/e2e/single/imagecatalog_test.go
+++ b/tests/e2e/single/imagecatalog_test.go
@@ -34,6 +34,10 @@ const catalogHeadTimeout = 30 * time.Second
// purpose. It is the check most likely to catch a problem that is not ours,
// and the sooner it is seen the smaller the window in which a released binary
// points at a dead URL.
+//
+// utils.WithheldImages is deliberately not walked: those are the entries whose
+// artifact is known to be unpublished, so HEADing them would assert a failure
+// we already know about and hide the ones we do not.
func runImageCatalogReachable(t *testing.T, _ *Fixture) {
harness.Phase(t, "Single — Image catalog URLs still answer")
diff --git a/tests/e2e/single/instance_test.go b/tests/e2e/single/instance_test.go
index 1859fc8ae..053a9e7ae 100644
--- a/tests/e2e/single/instance_test.go
+++ b/tests/e2e/single/instance_test.go
@@ -3,9 +3,6 @@
package single
import (
- "bytes"
- "os/exec"
- "strconv"
"testing"
"github.com/aws/aws-sdk-go/aws"
@@ -85,30 +82,3 @@ func runConsoleOutput(t *testing.T, fix *Fixture) {
"has_output", out.Output != nil && aws.StringValue(out.Output) != "",
)
}
-
-// runSSH is a thin wrapper around `ssh` matching the option set used by
-// harness.LsblkRootGiB. Returns stdout. t.Fatal on non-zero exit so callers
-// can chain assertions on the output without nil-checking err.
-func runSSH(t *testing.T, tgt harness.SSHTarget, command string) string {
- t.Helper()
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- command,
- }
- var stdout, stderr bytes.Buffer
- cmd := exec.Command("ssh", args...)
- cmd.Stdout = &stdout
- cmd.Stderr = &stderr
- if err := cmd.Run(); err != nil {
- t.Fatalf("ssh %s@%s:%d %q failed: %v\nstderr: %s",
- tgt.User, tgt.Host, tgt.Port, command, err, stderr.String())
- }
- return stdout.String()
-}
diff --git a/tests/e2e/single/kvreplication_test.go b/tests/e2e/single/kvreplication_test.go
new file mode 100644
index 000000000..0335b74b1
--- /dev/null
+++ b/tests/e2e/single/kvreplication_test.go
@@ -0,0 +1,118 @@
+//go:build e2e
+
+package single
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+ "github.com/mulgadc/spinifex/tests/e2e/harness"
+ "github.com/stretchr/testify/require"
+)
+
+// runKVReplication proves what a single-node cluster's own state should look
+// like, which is not the multi-node assertion with a smaller number in it.
+//
+// On one node R1 is the correct answer rather than the defect, so the thing worth
+// proving is the opposite: that nothing asked for a replica count this deployment
+// cannot place, that the operator surface says so plainly, and that repairing a
+// cluster with nothing to repair changes nothing. The multi-node suites cannot
+// cover any of that, because on three nodes the unplaceable count is the right
+// one.
+func runKVReplication(t *testing.T, fix *Fixture) {
+ harness.Phase(t, "Single — the cluster's own state on one node")
+
+ harness.Step(t, "every bucket is on the one replica a single node can hold")
+ harness.RequireKVSingleNode(t, fix.Env)
+
+ harness.Step(t, "the operator surface reports a healthy single node")
+ before, err := localReplicaReport(t)
+ require.NoError(t, err,
+ "spx admin kv replicas failed on a healthy single node, so nothing can be gated on it")
+ require.NotEmpty(t, before, "the command reported no buckets at all, so nothing was checked")
+ harness.Detail(t, "buckets", len(before), "want_replicas", before[0].Want)
+
+ // Exit 0 is the contract a deployment gate reads. On one node every bucket is
+ // at the count it should be and serving, so anything else is a bug in the
+ // audit rather than in the cluster.
+ require.Equal(t, 1, before[0].Want,
+ "a single-node cluster wants one replica per bucket, not %d", before[0].Want)
+ for _, r := range before {
+ require.Truef(t, r.HasQuorum(),
+ "%s reports %d of %d replicas online on a single node", r.Bucket, r.Online, r.Replicas)
+ }
+
+ // Twice, because --repair is documented as safe to run on a healthy cluster
+ // and safe to run twice. A repair that rewrote a stream would show up as a
+ // changed replica count or a bucket that stopped serving.
+ harness.Step(t, "--repair is idempotent and changes nothing it should not")
+ for i := 1; i <= 2; i++ {
+ out, repairErr := runLocal(t, "sudo -n spx admin kv replicas --repair --json")
+ require.NoErrorf(t, repairErr, "repair pass %d failed: %s", i, string(out))
+ }
+
+ after, err := localReplicaReport(t)
+ require.NoError(t, err, "the audit stopped working after two repair passes")
+ requireSameReplication(t, before, after)
+ harness.Detail(t, "repair_passes", 2, "buckets", len(after))
+}
+
+// requireSameReplication fails if any bucket's replication changed across the
+// repair passes, naming the bucket rather than reporting that two lists differ.
+func requireSameReplication(t *testing.T, before, after []kvutil.BucketReport) {
+ t.Helper()
+ was := make(map[string]kvutil.BucketReport, len(before))
+ for _, r := range before {
+ was[r.Bucket] = r
+ }
+ require.Len(t, after, len(before), "the number of KV buckets changed across the repair passes")
+ for _, now := range after {
+ prev, ok := was[now.Bucket]
+ require.Truef(t, ok, "%s appeared during a repair that should create nothing", now.Bucket)
+ require.Equalf(t, prev.Replicas, now.Replicas,
+ "%s went from %d replicas to %d across a repair that had nothing to repair",
+ now.Bucket, prev.Replicas, now.Replicas)
+ require.Truef(t, now.HasQuorum(),
+ "%s stopped serving after a repair that had nothing to repair", now.Bucket)
+ }
+}
+
+// localReplicaReport runs the replica audit on this node and decodes it. The
+// suite runs on the node it is testing, so there is no ssh hop.
+func localReplicaReport(t *testing.T) ([]kvutil.BucketReport, error) {
+ t.Helper()
+ out, err := runLocal(t, "sudo -n spx admin kv replicas --json")
+ if err != nil && len(out) == 0 {
+ return nil, err
+ }
+ var reports []kvutil.BucketReport
+ if jsonErr := json.Unmarshal(trimToJSON(out), &reports); jsonErr != nil {
+ return nil, fmt.Errorf("decode replica report (%q): %w", string(out), jsonErr)
+ }
+ return reports, err
+}
+
+// runLocal runs a shell command on this node, returning its combined output
+// whether or not it succeeded — the audit prints its report even when it exits
+// non-zero, and a report naming a bucket is a better failure than an exit status.
+func runLocal(t *testing.T, cmd string) ([]byte, error) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+ return exec.CommandContext(ctx, "sh", "-c", cmd).CombinedOutput()
+}
+
+// trimToJSON drops anything before the JSON document, since sudo and the login
+// shell can both write to the same stream.
+func trimToJSON(out []byte) []byte {
+ if i := strings.IndexAny(string(out), "[{"); i > 0 {
+ return out[i:]
+ }
+ return out
+}
diff --git a/tests/e2e/single/launchtemplate_test.go b/tests/e2e/single/launchtemplate_test.go
index 60269043c..1c3402ac2 100644
--- a/tests/e2e/single/launchtemplate_test.go
+++ b/tests/e2e/single/launchtemplate_test.go
@@ -134,7 +134,7 @@ func runLaunchTemplateBoot(t *testing.T, fix *Fixture) {
host, port := harness.InstancePublicSSHHost(t, inst)
harness.Detail(t, "instance_id", instanceID, "ssh_host", host, "ssh_port", port)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
harness.Step(t, "guest: template user-data ran")
diff --git a/tests/e2e/single/lifecycle_test.go b/tests/e2e/single/lifecycle_test.go
index 8292ba4d8..859f6b138 100644
--- a/tests/e2e/single/lifecycle_test.go
+++ b/tests/e2e/single/lifecycle_test.go
@@ -3,13 +3,7 @@
package single
import (
- "net"
- "os"
- "os/exec"
- "strconv"
- "sync/atomic"
"testing"
- "time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -17,36 +11,8 @@ import (
"github.com/stretchr/testify/require"
)
-// sshDatapathBroken is set the first time a single-node SSH probe times
-// out. Downstream SSH-dependent tests call requireSSHHealthy(t) and skip
-// rather than re-running the same 3-minute timeout against the same
-// broken datapath.
-var sshDatapathBroken atomic.Bool
-
-// sshReadyBudget bounds the single-node SSH-handshake probes. Baremetal boots
-// q35 guests slower (OVMF + NBD disks), so the harness widens it via
-// SPINIFEX_SSH_READY_TIMEOUT (Go duration); default 3m bounds shared runners.
-var sshReadyBudget = resolveSSHReadyBudget(3 * time.Minute)
-
-func resolveSSHReadyBudget(def time.Duration) time.Duration {
- if v := os.Getenv("SPINIFEX_SSH_READY_TIMEOUT"); v != "" {
- if d, err := time.ParseDuration(v); err == nil && d > 0 {
- return d
- }
- }
- return def
-}
-
-// requireSSHHealthy skips the calling test if a prior SSH probe on this
-// single-node VM has already failed. Cuts the cascade cost of a broken
-// datapath from 4×3min down to one fail + four skips.
-func requireSSHHealthy(t *testing.T) {
- t.Helper()
- if sshDatapathBroken.Load() {
- t.Skipf("skipping: earlier single-node SSH probe failed; " +
- "not retrying to keep suite time bounded")
- }
-}
+// sshHealth is this suite's sticky SSH-datapath verdict; see harness.SSHHealth.
+var sshHealth harness.SSHHealth
// runStopStart stops fix.Instance and waits for it to reach the
// "stopped" state, then asserts that rebooting a stopped instance is
@@ -157,58 +123,3 @@ func runAttachToStoppedError(t *testing.T, fix *Fixture) {
require.NoError(t, err, "delete-volume %s", stoppedVolID)
cleaned = true
}
-
-// waitForSSHReady probes a full SSH handshake (BatchMode + ConnectTimeout)
-// against host:port, retrying until the daemon completes banner exchange.
-// TCP-port reachability alone is insufficient — sshd accepts the connect
-// while pam/cloud-init are still finishing, and the first real runSSH then
-// hits "Connection timed out during banner exchange" (CI run 26034322018).
-func waitForSSHReady(t *testing.T, host string, port int, keyPath string) {
- t.Helper()
- requireSSHHealthy(t)
- addr := net.JoinHostPort(host, strconv.Itoa(port))
- harness.Step(t, "waiting for SSH handshake %s", addr)
- if !trySSHReady(host, port, keyPath, sshReadyBudget) {
- sshDatapathBroken.Store(true)
- t.Fatalf("Eventually: condition not met within %s: "+
- "[SSH handshake %s never completed] "+
- "(sticky-skip enabled for downstream)", sshReadyBudget, addr)
- }
-}
-
-// waitForSSHHandshake is an alias kept for guestchurn_test.go's call site.
-func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) {
- waitForSSHReady(t, host, port, keyPath)
-}
-
-// trySSHReady mirrors waitForSSHReady but returns reachability as a bool
-// instead of t.Fatal-ing on timeout. Matches run-e2e.sh ~2850–2860, which
-// treats SSH-via-public-IP timeout as a WARN ("bridge not ready") rather
-// than a hard test failure — new-subnet IGW SNAT/DNAT
-// flow installation is timing-sensitive on shared CI runners and not part
-// of the EC2 surface contract this test exists to validate.
-func trySSHReady(host string, port int, keyPath string, timeout time.Duration) bool {
- if sshDatapathBroken.Load() {
- return false
- }
- deadline := time.Now().Add(timeout)
- for {
- cmd := exec.Command("ssh",
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=3",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(port),
- "-i", keyPath,
- "ubuntu@"+host,
- "true")
- if cmd.Run() == nil {
- return true
- }
- if time.Now().After(deadline) {
- return false
- }
- time.Sleep(2 * time.Second)
- }
-}
diff --git a/tests/e2e/single/rebootliveness_test.go b/tests/e2e/single/rebootliveness_test.go
index 13797b277..3b5e27bd6 100644
--- a/tests/e2e/single/rebootliveness_test.go
+++ b/tests/e2e/single/rebootliveness_test.go
@@ -34,10 +34,10 @@ func runEarlyRebootLiveness(t *testing.T, fix *Fixture) {
instanceID := launchBaselineInstance(t, fix, amiID, instType, keyName, vpc.SubnetID, []string{vpc.SGID})
inst := describeSingletonInstance(t, fix, instanceID)
host, port := harness.InstancePublicSSHHost(t, inst)
- waitForSSHReady(t, host, port, keyPath)
+ sshHealth.WaitReady(t, host, port, keyPath)
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
- bootID := strings.TrimSpace(runSSH(t, tgt, "cat /proc/sys/kernel/random/boot_id"))
+ bootID := strings.TrimSpace(harness.RunSSH(t, tgt, "cat /proc/sys/kernel/random/boot_id"))
require.NotEmpty(t, bootID, "fresh guest returned an empty boot ID")
waitForCloudInitTarget(t, tgt)
diff --git a/tests/e2e/single/securitygroup_test.go b/tests/e2e/single/securitygroup_test.go
index f589dc120..90aaef2d9 100644
--- a/tests/e2e/single/securitygroup_test.go
+++ b/tests/e2e/single/securitygroup_test.go
@@ -3,10 +3,7 @@
package single
import (
- "bytes"
- "os/exec"
"regexp"
- "strconv"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -40,28 +37,3 @@ func authorizeAllowAllEgress(c *harness.AWSClient, sgID string) error {
})
return err
}
-
-// runSSHCombined runs `command` over SSH against tgt and returns combined
-// stdout+stderr regardless of exit status. Unlike runSSH (instance_test.go)
-// it does not t.Fatal on non-zero exit — needed because `ping` legitimately
-// exits non-zero when packets are dropped, which is the expected outcome of
-// the revoke test.
-func runSSHCombined(tgt harness.SSHTarget, command string) (string, error) {
- args := []string{
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "ConnectTimeout=5",
- "-o", "BatchMode=yes",
- "-p", strconv.Itoa(tgt.Port),
- "-i", tgt.KeyPath,
- tgt.User + "@" + tgt.Host,
- command,
- }
- var combined bytes.Buffer
- cmd := exec.Command("ssh", args...)
- cmd.Stdout = &combined
- cmd.Stderr = &combined
- err := cmd.Run()
- return combined.String(), err
-}
diff --git a/tests/e2e/single/sgreachability_test.go b/tests/e2e/single/sgreachability_test.go
index b87e38e69..7cfa0bdf0 100644
--- a/tests/e2e/single/sgreachability_test.go
+++ b/tests/e2e/single/sgreachability_test.go
@@ -85,7 +85,7 @@ func runSGReachabilityPolicy(t *testing.T, fix *Fixture) {
// No ingress rules yet; egress is allow-all by default so only inbound
// is gated. Probe a short window to confirm the default-deny ACL is
// applied and stable. This overlaps guest boot, and Authorize below
- // still pays the full boot wait via trySSHReady, so a longer window
+ // still pays the full boot wait via sshHealth.TryReady, so a longer window
// buys little extra coverage.
harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
deadline := time.Now().Add(10 * time.Second)
@@ -99,12 +99,12 @@ func runSGReachabilityPolicy(t *testing.T, fix *Fixture) {
authorizeOK := t.Run("Authorize", func(t *testing.T) {
harness.Step(t, "authorizing tcp/22 ingress, expecting reachability")
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
- require.Truef(t, trySSHReady(pubIP, 22, keyPath, sshReadyBudget),
+ require.Truef(t, sshHealth.TryReady(pubIP, 22, keyPath, harness.SSHReadyBudget),
"tcp/22 to %s never became reachable after authorizing ingress — "+
"default subnet egress/IGW datapath is broken", pubIP)
tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath}
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
assert.Containsf(t, idOut, "ubuntu", "ssh id after authorize\n%s", idOut)
})
if !authorizeOK {
@@ -193,7 +193,7 @@ func runSGReachabilityPolicy(t *testing.T, fix *Fixture) {
harness.Detail(t, "probe_wan", wanIP)
probeWAN := func() string {
- out, _ := runSSHCombined(tgt, fmt.Sprintf("ping -c 3 -W 2 %s", wanIP))
+ out, _ := harness.RunSSHCombined(tgt, fmt.Sprintf("ping -c 3 -W 2 %s", wanIP))
return out
}
diff --git a/tests/e2e/single/smtpblock_test.go b/tests/e2e/single/smtpblock_test.go
index 47b800ce6..0a92d588f 100644
--- a/tests/e2e/single/smtpblock_test.go
+++ b/tests/e2e/single/smtpblock_test.go
@@ -66,7 +66,7 @@ func runWANEgressSMTPBlock(t *testing.T, fix *Fixture) {
harness.Step(t, "authorizing tcp/22 ingress, expecting reachability")
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
- require.Truef(t, trySSHReady(pubIP, 22, keyPath, sshReadyBudget),
+ require.Truef(t, sshHealth.TryReady(pubIP, 22, keyPath, harness.SSHReadyBudget),
"tcp/22 to %s never became reachable after authorizing ingress", pubIP)
tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath}
diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go
index a1d4b5b8f..8d11a67c3 100644
--- a/tests/e2e/single/tests_test.go
+++ b/tests/e2e/single/tests_test.go
@@ -30,6 +30,13 @@ func TestAccountScoping(t *testing.T) {
runAccountScoping(t, requireSingleNodeFixture(t))
}
+// TestKVReplication reads the cluster's own state and runs a repair that has
+// nothing to repair, so it mutates nothing another test can see.
+func TestKVReplication(t *testing.T) {
+ t.Parallel()
+ runKVReplication(t, requireSingleNodeFixture(t))
+}
+
// Reaches the public internet rather than the cluster: it asks whether the
// upstreams our image catalog names are still publishing what it claims.
func TestImageCatalogReachable(t *testing.T) {
diff --git a/tests/e2e/single/vpcegress_test.go b/tests/e2e/single/vpcegress_test.go
index 9e0f4f524..55d6e95a2 100644
--- a/tests/e2e/single/vpcegress_test.go
+++ b/tests/e2e/single/vpcegress_test.go
@@ -268,7 +268,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
_, _ = c.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(pubInstanceID)},
})
- _ = waitForInstanceStateSoft(c, pubInstanceID, "terminated", 5*time.Minute)
+ _ = harness.WaitForInstanceStateSoft(c, pubInstanceID, "terminated", 5*time.Minute)
})
harness.Detail(t, "instance", pubInstanceID)
@@ -281,7 +281,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
harness.Detail(t, "public_ip", pubIP, "private_ip", pubPrivIP)
pubOK := t.Run("PublicSubnetEgress", func(t *testing.T) {
- if !trySSHReady(pubIP, 22, keyPath, sshReadyBudget) {
+ if !sshHealth.TryReady(pubIP, 22, keyPath, harness.SSHReadyBudget) {
harness.DumpVPCFlowDiagnostics(t, c, pubInstanceID,
fmt.Sprintf("PublicSubnetEgress SSH timeout — vpc=%s igw=%s pub=%s", vpcID, igwID, pubIP),
harness.VPCDiagnosticsOpts{
@@ -289,13 +289,13 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
LogicalIP: pubPrivIP,
ArtifactDir: fix.ArtifactDir(t),
})
- t.Fatalf("SSH handshake %s:22 never completed within %s (see diagnostics above)", pubIP, sshReadyBudget)
+ t.Fatalf("SSH handshake %s:22 never completed within %s (see diagnostics above)", pubIP, harness.SSHReadyBudget)
}
tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath}
harness.Step(t, "ssh id (smoke)")
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
require.Containsf(t, idOut, "ubuntu", "ssh id did not report ubuntu\n%s", idOut)
// Verify external connectivity through the IGW + 0.0.0.0/0 route. ping
@@ -405,7 +405,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
harness.Detail(t, "instance", instanceID, "public_ip", rbPubIP)
harness.Step(t, "expecting external SSH to instance in fresh-VPC public subnet")
- if !trySSHReady(rbPubIP, 22, keyPath, sshReadyBudget) {
+ if !sshHealth.TryReady(rbPubIP, 22, keyPath, harness.SSHReadyBudget) {
harness.DumpVPCFlowDiagnostics(t, c, instanceID,
fmt.Sprintf("RouteBeforeSubnet SSH timeout — vpc=%s igw=%s pub=%s", vID, iID, rbPubIP),
harness.VPCDiagnosticsOpts{
@@ -419,7 +419,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
}
tgt := harness.SSHTarget{User: "ubuntu", Host: rbPubIP, Port: 22, KeyPath: keyPath}
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
assert.Containsf(t, idOut, "ubuntu", "ssh id in fresh-VPC subnet\n%s", idOut)
})
@@ -442,7 +442,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
_, _ = c.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(privInstanceID)},
})
- _ = waitForInstanceStateSoft(c, privInstanceID, "terminated", 5*time.Minute)
+ _ = harness.WaitForInstanceStateSoft(c, privInstanceID, "terminated", 5*time.Minute)
})
privInst := harness.WaitForInstanceState(t, c, privInstanceID, "running")
@@ -456,7 +456,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
// Copy the keypair to the bastion so it can hop into the private guest.
harness.Step(t, "scp keypair -> bastion:/tmp/key.pem")
scpKey(t, keyPath, pubIP)
- _ = runSSH(t, bastionTgt, "chmod 600 /tmp/key.pem")
+ _ = harness.RunSSH(t, bastionTgt, "chmod 600 /tmp/key.pem")
privProbe := fmt.Sprintf(
"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "+
@@ -727,7 +727,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
}
harness.Step(t, "ssh to guest via %s after restart", newIP)
- if !trySSHReady(newIP, 22, keyPath, sshReadyBudget) {
+ if !sshHealth.TryReady(newIP, 22, keyPath, harness.SSHReadyBudget) {
harness.DumpVPCFlowDiagnostics(t, c, pubInstanceID,
fmt.Sprintf("StopStartAddress SSH timeout — was=%s now=%s instance=%s", pubIP, newIP, pubInstanceID),
harness.VPCDiagnosticsOpts{
@@ -735,9 +735,9 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
LogicalIP: pubPrivIP,
ArtifactDir: fix.ArtifactDir(t),
})
- t.Fatalf("guest unreachable via %s within %s after restart (see diagnostics above)", newIP, sshReadyBudget)
+ t.Fatalf("guest unreachable via %s within %s after restart (see diagnostics above)", newIP, harness.SSHReadyBudget)
}
- out := runSSH(t, harness.SSHTarget{User: "ubuntu", Host: newIP, Port: 22, KeyPath: keyPath}, "id")
+ out := harness.RunSSH(t, harness.SSHTarget{User: "ubuntu", Host: newIP, Port: 22, KeyPath: keyPath}, "id")
require.Containsf(t, out, "ubuntu", "ssh after restart did not report ubuntu\n%s", out)
harness.Detail(t, "datapath", "reachable_on_new_address_ok")
@@ -789,7 +789,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
// association publishes; poll the SSH handshake rather than
// asserting immediately.
harness.Step(t, "ssh to guest via EIP %s", eipIP)
- if !trySSHReady(eipIP, 22, keyPath, sshReadyBudget) {
+ if !sshHealth.TryReady(eipIP, 22, keyPath, harness.SSHReadyBudget) {
harness.DumpVPCFlowDiagnostics(t, c, pubInstanceID,
fmt.Sprintf("EIP SSH timeout — eip=%s instance=%s", eipIP, pubInstanceID),
harness.VPCDiagnosticsOpts{
@@ -797,10 +797,10 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
LogicalIP: pubPrivIP,
ArtifactDir: fix.ArtifactDir(t),
})
- t.Fatalf("guest unreachable via EIP %s within %s (see diagnostics above)", eipIP, sshReadyBudget)
+ t.Fatalf("guest unreachable via EIP %s within %s (see diagnostics above)", eipIP, harness.SSHReadyBudget)
}
tgt := harness.SSHTarget{User: "ubuntu", Host: eipIP, Port: 22, KeyPath: keyPath}
- idOut := runSSH(t, tgt, "id")
+ idOut := harness.RunSSH(t, tgt, "id")
require.Containsf(t, idOut, "ubuntu", "ssh via EIP id did not report ubuntu\n%s", idOut)
harness.Detail(t, "datapath", "eip_reachable_ok")
@@ -851,7 +851,7 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
"instance came back on a different address than the Elastic IP associated to it")
harness.Step(t, "ssh to guest via EIP %s after restart", eipIP)
- if !trySSHReady(eipIP, 22, keyPath, sshReadyBudget) {
+ if !sshHealth.TryReady(eipIP, 22, keyPath, harness.SSHReadyBudget) {
harness.DumpVPCFlowDiagnostics(t, c, pubInstanceID,
fmt.Sprintf("EIP SSH timeout after restart — eip=%s instance=%s", eipIP, pubInstanceID),
harness.VPCDiagnosticsOpts{
@@ -859,9 +859,9 @@ func runVPCEgressPaths(t *testing.T, fix *Fixture) {
LogicalIP: aws.StringValue(started.PrivateIpAddress),
ArtifactDir: fix.ArtifactDir(t),
})
- t.Fatalf("guest unreachable via EIP %s within %s after restart (see diagnostics above)", eipIP, sshReadyBudget)
+ t.Fatalf("guest unreachable via EIP %s within %s after restart (see diagnostics above)", eipIP, harness.SSHReadyBudget)
}
- out := runSSH(t, harness.SSHTarget{User: "ubuntu", Host: eipIP, Port: 22, KeyPath: keyPath}, "id")
+ out := harness.RunSSH(t, harness.SSHTarget{User: "ubuntu", Host: eipIP, Port: 22, KeyPath: keyPath}, "id")
require.Containsf(t, out, "ubuntu", "ssh via EIP after restart did not report ubuntu\n%s", out)
harness.Detail(t, "datapath", "eip_reachable_after_restart_ok")
})
@@ -958,26 +958,6 @@ func waitForNATGatewayStateSoft(c *harness.AWSClient, id, target string, timeout
}
}
-// waitForInstanceStateSoft is the cleanup-time analogue of
-// harness.WaitForInstanceState — no t.Fatal, just polls and returns.
-func waitForInstanceStateSoft(c *harness.AWSClient, id, target string, timeout time.Duration) error {
- deadline := time.Now().Add(timeout)
- for {
- out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
- InstanceIds: []*string{aws.String(id)},
- })
- if err == nil && len(out.Reservations) > 0 && len(out.Reservations[0].Instances) > 0 {
- if aws.StringValue(out.Reservations[0].Instances[0].State.Name) == target {
- return nil
- }
- }
- if time.Now().After(deadline) {
- return fmt.Errorf("instance %s did not reach %s within %s", id, target, timeout)
- }
- time.Sleep(2 * time.Second)
- }
-}
-
// scpKey copies the harness PEM to /tmp/key.pem on the bastion so it can
// hop into a guest that has no direct SSH ingress of its own.
func scpKey(t *testing.T, keyPath, host string) {
@@ -1004,7 +984,7 @@ func scpKey(t *testing.T, keyPath, host string) {
// EventuallyErr budget. The ctx deadline kills ssh regardless of which phase hangs.
const sshQuietHardTimeout = 30 * time.Second
-// runSSHQuiet is the EventuallyErr-friendly variant of runSSH: it returns
+// runSSHQuiet is the EventuallyErr-friendly variant of harness.RunSSH: it returns
// stdout+stderr and the error rather than calling t.Fatal, so polling
// loops can iterate on transient failures (cloud-init not done, OVN
// flow not installed yet, etc).
diff --git a/tests/e2e/storagefault/stop_start_durability_test.go b/tests/e2e/storagefault/stop_start_durability_test.go
index 3b6c6b949..e32e7aeb4 100644
--- a/tests/e2e/storagefault/stop_start_durability_test.go
+++ b/tests/e2e/storagefault/stop_start_durability_test.go
@@ -31,6 +31,10 @@ const (
// Generous because the stop is deliberately made to run against a backend
// that cannot answer, which is the slowest this path ever gets.
stopStartTimeout = 10 * time.Minute
+
+ // nodeConfigPath is where the installed layout keeps the cluster config an
+ // spx subcommand run over SSH has to be pointed at.
+ nodeConfigPath = "/etc/spinifex/spinifex.toml"
)
// viperblockBaseDir is where a node keeps per-volume local state. Overridable
@@ -147,6 +151,12 @@ func TestCrossNodeStartAfterFailedSealTakesOverLoudly(t *testing.T) {
thawPredastore(t, fix, freezeSet)
time.Sleep(recoverySettle)
+ // Read after the thaw has settled, not before it. A marker is written when
+ // a volume opens and cleared by the seal, and the seal the stop gave up
+ // waiting for can still finish once the backend answers again. Only what
+ // survives that is genuinely left behind for the takeover to be loud about.
+ unsealed := requireUnsealedVolumes(t, fix, hostNode)
+
// The node has to be gone, not merely quiet. A live viperblockd goes on
// renewing the lease on a volume whose seal failed, and that refusal is
// correct: the node could still be writing.
@@ -167,7 +177,7 @@ func TestCrossNodeStartAfterFailedSealTakesOverLoudly(t *testing.T) {
// The loss is accepted; being unable to see it is not. This is the only
// record that the volume opened from an older checkpoint than existed.
- assertTakeoverLogged(t, fix, landed, hostNode, volID)
+ assertTakeoverLogged(t, fix, landed, hostNode, unsealed)
host, port := harness.InstancePublicSSHHost(t, inst)
if !harness.TryGuestSSHReady(host, port, "ubuntu", tgt.KeyPath, 5*time.Minute) {
@@ -229,7 +239,7 @@ func guestDatasource(tgt harness.SSHTarget) string {
// This is the whole guarantee. Losing the previous holder's unsealed writes is
// a deliberate trade against the instance not running at all, and it is only
// defensible while an operator can find out it happened.
-func assertTakeoverLogged(t *testing.T, fix *Fixture, landed, previous *harness.Node, volID string) {
+func assertTakeoverLogged(t *testing.T, fix *Fixture, landed, previous *harness.Node, unsealed []string) {
t.Helper()
if landed == nil {
return
@@ -239,27 +249,98 @@ func assertTakeoverLogged(t *testing.T, fix *Fixture, landed, previous *harness.
defer cancel()
cmd := "sudo journalctl -u spinifex-viperblock -u spinifex-daemon --since '-10 min' --no-pager 2>/dev/null | " +
- "grep -F " + volID + " | grep -i 'last held by another node' | tail -5"
+ "grep -i 'last held by another node' | tail -20"
out, err := fix.SSH.Run(ctx, *landed, cmd)
if err != nil {
t.Errorf("could not read %s's journal to confirm the takeover was logged: %v", landed.Name, err)
return
}
+
+ // Any one of them: the instance's volumes are sealed together and the
+ // guarantee is that the operator is told, not that every volume is named.
journal := string(out)
- if strings.TrimSpace(journal) == "" {
+ var named string
+ for _, volID := range unsealed {
+ if strings.Contains(journal, volID) {
+ named = volID
+ break
+ }
+ }
+ if named == "" {
t.Errorf("%s opened %s from the backend checkpoint while %s held unsealed writes, and logged "+
"nothing about it. The loss is acceptable; a silent loss is not.",
- landed.Name, volID, previous.Name)
+ landed.Name, strings.Join(unsealed, ", "), previous.Name)
return
}
// The marker records the spinifex node name, which is the host's own name
// and not the harness's nodeN label, so accept either identity.
if !strings.Contains(journal, previous.Name) && !strings.Contains(journal, previous.Addr) {
t.Errorf("%s logged a takeover for %s but did not name %s (%s), so an operator cannot tell "+
- "whose writes were left behind:\n%s", landed.Name, volID, previous.Name, previous.Addr, journal)
+ "whose writes were left behind:\n%s", landed.Name, named, previous.Name, previous.Addr, journal)
return
}
- harness.Step(t, "%s logged the takeover and named %s", landed.Name, previous.Name)
+ harness.Step(t, "%s logged the takeover of %s and named %s", landed.Name, named, previous.Name)
+}
+
+// requireUnsealedVolumes returns the volumes the cluster records node as
+// holding writes the backend may not have, and skips the test if there are none.
+//
+// The stop returning an error does not establish this. The unmount is a NATS
+// request, and its caller giving up says nothing about the seal behind it, so
+// local state kept with no seal receipt is equally what an in-flight seal looks
+// like. A marker that is still there once the backend is answering again is not.
+//
+// Which volume is left unsealed is not the test's to choose. Freezing
+// predastore fails a seal that has to read from it — a root volume loading its
+// AMI snapshot — while a blank attached volume can seal straight through the
+// outage. That is correct behaviour, so the takeover is asserted against
+// whatever was actually left behind rather than a volume picked in advance.
+func requireUnsealedVolumes(t *testing.T, fix *Fixture, node *harness.Node) []string {
+ t.Helper()
+
+ held := unsealedVolumesHeldBy(t, fix, node)
+ if len(held) == 0 {
+ t.Skipf("no seal failed on %s while the backend was frozen, so there are no unsealed writes "+
+ "to take over and this test would assert a warning that would be wrong", node.Name)
+ }
+ harness.Step(t, "%s still holds unsealed writes for %s after the backend came back",
+ node.Name, strings.Join(held, ", "))
+ return held
+}
+
+// unsealedVolumesHeldBy reads the cluster's unsealed-volume report from node and
+// returns the volumes it names node as holding. The report is a block per
+// volume: the id on its own line, then indented fields.
+func unsealedVolumesHeldBy(t *testing.T, fix *Fixture, node *harness.Node) []string {
+ t.Helper()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ out, err := fix.SSH.Run(ctx, *node, "sudo spx admin volumes unsealed --config "+nodeConfigPath)
+ if err != nil {
+ // The node is up and the report reads NATS, not the frozen backend, so
+ // this is worth seeing rather than retrying silently.
+ t.Logf("could not read the unsealed volumes from %s: %v", node.Name, err)
+ return nil
+ }
+
+ var held []string
+ current := ""
+ for line := range strings.SplitSeq(string(out), "\n") {
+ trimmed := strings.TrimSpace(line)
+ switch {
+ case strings.HasPrefix(trimmed, "vol-") && !strings.Contains(trimmed, " "):
+ current = trimmed
+ case current != "" && strings.HasPrefix(trimmed, "held by:"):
+ owner := strings.TrimSpace(strings.TrimPrefix(trimmed, "held by:"))
+ if owner == node.Name || owner == node.Addr {
+ held = append(held, current)
+ }
+ current = ""
+ }
+ }
+ return held
}
// assertLocalStateKept checks the node kept the volume's local files and wrote
diff --git a/tests/integration/assumed_role_test.go b/tests/integration/assumed_role_test.go
index 6b46a4960..f17993ab9 100644
--- a/tests/integration/assumed_role_test.go
+++ b/tests/integration/assumed_role_test.go
@@ -75,7 +75,7 @@ func TestAssumedRoleControlPlaneEnforcement(t *testing.T) {
// Zero-policy role: the session authenticates, then default-denies.
_, err = sessionCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
policyOut, err := gw.IAMClient(t).CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String("are-ec2-describe-regions"),
diff --git a/tests/integration/conformance-known-findings.md b/tests/integration/conformance-known-findings.md
index 40ddf14af..062470f88 100644
--- a/tests/integration/conformance-known-findings.md
+++ b/tests/integration/conformance-known-findings.md
@@ -17,13 +17,7 @@ Twelve responses are reported separately as `errors_unmodelled`: four IAM and ei
EC2's model declares no operation errors, so EC2 uses a separate checked-in catalog curated from AWS's official EC2 error-code reference. It contains every documented common and server error plus the documented action-specific codes Spinifex currently emits. The validator checks the EC2 XML envelope, catalog membership, and AWS's documented 4xx client / 5xx server classification.
-The same baseline examined 42 EC2 error responses and exposed 16 violations. The
-`InsufficientInstanceCapacity` status-class defect has since been fixed — that
-code now returns 503, matching the other capacity errors — leaving 15 open:
-
-| Finding | Count | Classification | Implementation cause |
-|---|---:|---|---|
-| EC2 authorization failures return `AccessDenied`, which is absent from the EC2 catalog; EC2 documents `UnauthorizedOperation` | 15 | Defect | The shared policy evaluator returns the Query/IAM-oriented `AccessDenied` code for EC2 requests. |
+The same baseline examined 42 EC2 error responses and exposed 16 violations. The `InsufficientInstanceCapacity` status-class defect has since been fixed — that code now returns 503, matching the other capacity errors. The 2026-09-29 run examined 50 EC2 error responses and found 23 violations, all EC2 policy denials answered with the IAM-oriented `AccessDenied`. EC2 now answers a policy denial, including an `iam:PassRole` denial, with `UnauthorizedOperation`, and a rerun of the same 50 responses reports no violations.
Six additional production-referenced values are deliberately not accepted by the catalog because the chosen EC2 reference does not list them: `ExpiredToken`, `IamInstanceProfileAlreadyAssociated`, `InvalidIamInstanceProfile.NotFound`, `NoSuchAssociation`, `NoSuchEntity`, and `RequestEntityTooLarge`. A runtime occurrence remains a visible violation until it is either backed by an authoritative EC2 source or replaced with a documented EC2 code.
@@ -41,19 +35,24 @@ A rejection that succeeds is always a finding. A refusal counts as a pass only w
Request findings and undeclared errors block only for services listed under `promotedRequestServices` in `conformance-promoted-services.json`, which starts empty; response promotion does not carry over.
-The 2026-09-28 baseline sent 3986 requests across 368 operations: 794 pass, 396 findings, 101 undeclared errors, 2695 inconclusive. None of it has been checked against AWS, so each finding below is a lead to verify before any fix.
+The 2026-09-29 baseline sent 4047 requests across 371 operations: 980 pass, 506 findings, 96 undeclared errors, 2465 inconclusive. None of it had been checked against AWS, so each finding below is a lead to verify before any fix. Since then, EC2 range-checks `MaxResults` on six `Describe*` calls and refuses a tag resource type the operation cannot tag, as AWS answers both, which clears 12 EC2 findings. IAM tag and untag calls now refuse a tag with no value or outside the model's key and value character sets, count tag lengths in characters rather than bytes, treat keys that differ only in case as one key on users and roles but as two elsewhere, and refuse the reserved `aws:` prefix, each as AWS answered it. IAM also checks tags on every create, `PathPrefix` on every list call (and `ListAttached*Policies` now filters by it), the `Scope` and `PolicyUsageFilter` enums, role and policy `Description` length, role `MaxSessionDuration` against 3600–43200, OIDC client ID and thumbprint lengths, `ListEntitiesForPolicy` `PolicyArn` length and an empty `CreatePolicy` `Path`, and answers name and path length breaks with `ValidationError`, each verified against AWS. STS `GetSessionToken` refuses an out-of-range `DurationSeconds` rather than clamping it, as AWS answers `AssumeRole`'s, and both calls check `SerialNumber` and `TokenCode` against the model's length and pattern before refusing MFA. Every finding is traced to its cause, and grouped into fix items, in the mulga plan `docs/development/archive/bugs/aws-integration-conformance-findings.md`.
| Finding | Count | Notes |
|---|---:|---|
-| A request breaking a model constraint is accepted | 336 | IAM 116, ECS 63, ELBv2 48, ECR 41, EC2 34, ACM 15, RDS 7, STS 6, EKS 6. Examples: IAM `List*` accept out-of-range `MaxItems` and `Marker`; IAM, ELBv2 and ECR accept a tag without `Key`; EC2 `Describe*` accept out-of-range `MaxResults`; ECS `CreateCapacityProvider` accepts every enum and range break inside `autoScalingGroupProvider`; STS `GetSessionToken` accepts a `SerialNumber` and `TokenCode` below their minimum length. |
-| A model-valid request is refused as invalid | 60 | Mostly deliberate limitations that say so: RDS `ModifyDBInstance` members, STS session policies, tags and MFA, IAM `PermissionsBoundary`, ECS `availabilityZoneRebalancing`. Others return a generic message whose cause is unresolved (ECS, EC2, EKS), or the model leaves out a value AWS itself requires, such as an RDS `AllocatedStorage` below the engine minimum. |
-| A refusal uses an undeclared error code | 101 | ECR (89) and EKS (1) answer with `InvalidParameterValueException`; both models declare `InvalidParameterException`. IAM (11) answers `CreateGroup` and `Put*Policy` constraint breaks with `InvalidInput`, which those operations do not declare. |
+| A request breaking a model constraint is accepted | 420 | ECS 116, IAM 112, ACM 56, ELBv2 48, ECR 47, EC2 22, RDS 7, STS 6, EKS 6. Examples: ACM and ELBv2 accept a tag without `Key`, as IAM did; EC2 network `Describe*` accepted out-of-range `MaxResults`; ECS `CreateCapacityProvider` accepts every enum and range break inside `autoScalingGroupProvider` and `managedInstancesProvider`; ACM `RequestCertificate` accepts every break inside `DomainValidationOptions`; STS `GetSessionToken` accepted out-of-range `DurationSeconds` and a `SerialNumber` and `TokenCode` below their minimum length, and still accepts an out-of-range `MinimumSessionTokenSize`. |
+| A model-valid request is refused as invalid | 74 | RDS 42, ECS 10, STS 10, EC2 8, EKS 2, IAM 2. Mostly deliberate limitations that say so: RDS `CreateDBInstance`, `ModifyDBInstance` and `RestoreDBInstanceFromDBSnapshot` members, STS session policies, tags and MFA, ECS `availabilityZoneRebalancing`. Others return a generic message whose cause is unresolved (ECS task and task-definition operations, EC2 `Modify*Attribute`), or add a member next to a seeded one that AWS may refuse in combination, such as EC2 `CreateCapacityReservation` `AvailabilityZoneId` beside `AvailabilityZone`. |
+| A refusal uses an undeclared error code | 96 | ECR (90) and EKS (2) answer with `InvalidParameterValueException`; both models declare `InvalidParameterException`. IAM (4) answered `CreateGroup` and `Put*Policy` name-length breaks with `InvalidInput`, which those operations do not declare; they now answer `ValidationError`, as AWS does. |
+
+How the sweep reaches its targets:
+
+- Models mark conditionally required members optional. The members AWS requires anyway are seeded into every request for the operations listed in `conditionallyRequired` (`internal/awsmodel/request_hints.go`). For any other operation, an acceptance request refused with `MissingParameter` or `InvalidParameterCombination` is inconclusive, and so is every later refusal for that operation once its required-only request is refused.
+- Members whose model shape accepts values AWS refuses, such as ARNs, IDs, zones, policy documents, certificates and ECR manifests, take hinted values of the right form (`request_hints.go`). An EC2 create's `TagSpecifications` names the resource type that operation creates, since AWS refuses any other.
+- A pattern Go's regexp cannot compile, because it uses lookahead, is sampled through a hand-written RE2 equivalent (`re2Equivalents` in `internal/awsmodel/pattern.go`). An operation whose required member has a pattern with no equivalent is listed as `UNTESTED`; none is today.
+- The runner creates an IAM role, managed policy, OIDC provider, instance profile, group and user, an ECS cluster and an ECR repository per run, and every operation except a create or delete names them. Other services have no fixture, so their tag, update and describe cases still end in not-found.
Known limits of the sweep:
-- Models mark conditionally required members optional, so an acceptance request refused with `MissingParameter` or `InvalidParameterCombination` is inconclusive, and so is every later refusal for that operation once its required-only request is refused.
- Filter names and pagination tokens cannot be generated validly, so `Filters`, `Marker` and `NextToken` get rejection requests only, and a refusal of one is inconclusive.
-- ACM `RequestCertificate` sends no requests: its `DomainName` pattern uses lookahead, which Go's regexp cannot sample. The report lists such operations as `UNTESTED`.
- Constraints no request breaks are counted as `unbroken_constraints`: required URI labels, maximum lengths above the generated limit (such as IAM policy documents), map entry counts, and patterns no value of a valid length breaks.
-- Most inconclusives are resource lookups that run before validation (`NoSuchEntity`, `*NotFound`), EC2 requests missing a conditionally required member (`MissingParameter`), EC2 subjects no daemon-lite answers (`InternalError`), and EKS operations whose backends are absent (`ServiceUnavailableException`, `NotImplementedException`).
+- Most inconclusives are resource lookups that run before validation (`NoSuchEntity`, `*NotFound`), ECS's bare `InvalidParameterException` for a missing resource, EC2 requests missing a conditionally required member (`MissingParameter`), EC2 subjects no daemon-lite answers (`InternalError`), RDS `ServerInternal`, and EKS operations whose backends are absent (`ServiceUnavailableException`, `NotImplementedException`).
- The EC2 query parser ignores parameters it does not know, which no generated request can reveal because every member it sends is modelled.
diff --git a/tests/integration/conformance.go b/tests/integration/conformance.go
index 374cfc725..cee3835e6 100644
--- a/tests/integration/conformance.go
+++ b/tests/integration/conformance.go
@@ -6,8 +6,10 @@ import (
"bytes"
"fmt"
"io"
+ "maps"
"net/http"
"net/url"
+ "slices"
"sort"
"strings"
"sync"
@@ -210,11 +212,7 @@ func (c *conformanceCollector) report(policy conformancePolicy, mode conformance
for service := range policy.promoted {
checkedServices[service] = true
}
- services := make([]awsmodel.Service, 0, len(checkedServices))
- for service := range checkedServices {
- services = append(services, service)
- }
- sort.Slice(services, func(i, j int) bool { return services[i] < services[j] })
+ services := slices.Sorted(maps.Keys(checkedServices))
for _, service := range services {
fmt.Fprintf(&report, "CHECKED %s success=%d errors=%d unmodelled_errors=%d promoted=%t\n",
service, c.checkedByService[service], c.errorCheckedByService[service], c.errorUnmodelledByService[service], policy.isPromoted(service))
diff --git a/tests/integration/conformance_policy.go b/tests/integration/conformance_policy.go
index 486016a6b..8b9a85bd2 100644
--- a/tests/integration/conformance_policy.go
+++ b/tests/integration/conformance_policy.go
@@ -6,8 +6,9 @@ import (
_ "embed"
"encoding/json"
"fmt"
+ "maps"
"os"
- "sort"
+ "slices"
"strings"
"github.com/mulgadc/spinifex/internal/awsmodel"
@@ -89,12 +90,7 @@ func (p conformancePolicy) isRequestPromoted(service awsmodel.Service) bool {
}
func (p conformancePolicy) services() []awsmodel.Service {
- services := make([]awsmodel.Service, 0, len(p.promoted))
- for service := range p.promoted {
- services = append(services, service)
- }
- sort.Slice(services, func(i, j int) bool { return services[i] < services[j] })
- return services
+ return slices.Sorted(maps.Keys(p.promoted))
}
func conformanceModeFromEnvironment() (conformanceMode, error) {
diff --git a/tests/integration/cross_tenant_test.go b/tests/integration/cross_tenant_test.go
index e1ca20bcd..6b7269a8e 100644
--- a/tests/integration/cross_tenant_test.go
+++ b/tests/integration/cross_tenant_test.go
@@ -227,6 +227,9 @@ func crossTenantCases() []crossTenantCase {
// a grant on another tenant's resource taking effect.
func TestCrossTenant_ForeignResourceARNNeverAuthorizes(t *testing.T) {
gw := StartGateway(t)
+ // Without the services behind NATS, the own/allowed probes end in
+ // no-responders, which the gateway renders as InternalError.
+ StartServiceDaemonLite(t, gw)
home, creds := createProbePrincipal(t, gw, "cross-tenant-home")
foreign, err := gw.Config.IAMService.CreateAccount("cross-tenant-foreign")
diff --git a/tests/integration/harness.go b/tests/integration/harness.go
index 8abdc2241..a3efd4b18 100644
--- a/tests/integration/harness.go
+++ b/tests/integration/harness.go
@@ -137,10 +137,10 @@ func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option)
masterKey, err := handlers_iam.GenerateMasterKey()
require.NoError(t, err)
- iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey, 1)
+ iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey)
require.NoError(t, err)
- stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1)
+ stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey)
require.NoError(t, err)
encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey)
@@ -155,10 +155,10 @@ func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option)
// ECR auth bridge signing key: reuses the IAM master key to encrypt the
// signing key at rest in the same embedded JetStream KV, matching
// production's awsgw-keys wiring (services/awsgw/awsgw.go).
- signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey, 1)
+ signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey)
require.NoError(t, err)
- bedrockAccess := gateway_bedrock.NewModelAccessStore(js, 1)
+ bedrockAccess := gateway_bedrock.NewModelAccessStore(js)
cfg := &gateway.GatewayConfig{
DisableLogging: true,
diff --git a/tests/integration/iam_groups_test.go b/tests/integration/iam_groups_test.go
index b658b3be8..87c4c4cb2 100644
--- a/tests/integration/iam_groups_test.go
+++ b/tests/integration/iam_groups_test.go
@@ -36,7 +36,7 @@ func TestIAMGroupEnforcement(t *testing.T) {
// No policy anywhere yet: the active key authenticates, then default-denies.
_, err = memberCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
// Group + policy granting ec2:DescribeRegions, then join the user.
groupOut, err := gw.IAMClient(t).CreateGroup(&iam.CreateGroupInput{GroupName: aws.String("grp-enforce")})
@@ -71,7 +71,7 @@ func TestIAMGroupEnforcement(t *testing.T) {
require.NoError(t, err, "remove-user-from-group")
_, err = memberCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
}
// TestIAMGroupInlineEnforcement proves an inline policy embedded in a group
@@ -93,7 +93,7 @@ func TestIAMGroupInlineEnforcement(t *testing.T) {
// No grant anywhere yet: the active key authenticates, then default-denies.
_, err = memberCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
// Group with an inline policy granting ec2:DescribeRegions, then join the user.
groupOut, err := gw.IAMClient(t).CreateGroup(&iam.CreateGroupInput{GroupName: aws.String("grp-inline-enforce")})
@@ -126,7 +126,7 @@ func TestIAMGroupInlineEnforcement(t *testing.T) {
require.NoError(t, err, "delete-group-policy")
_, err = memberCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
}
// TestIAMUserInlineEnforcement proves the user-inline linchpin: a policy
@@ -150,7 +150,7 @@ func TestIAMUserInlineEnforcement(t *testing.T) {
// No grant anywhere yet: the active key authenticates, then default-denies.
_, err = userCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
// Put an inline policy granting ec2:DescribeRegions directly on the user.
const inlinePolicyName = "usr-inl-enf-describe-regions"
@@ -174,7 +174,7 @@ func TestIAMUserInlineEnforcement(t *testing.T) {
require.NoError(t, err, "delete-user-policy")
_, err = userCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
}
// Dedicated identifiers for the Groups lifecycle test, distinct from the
diff --git a/tests/integration/iam_roles_test.go b/tests/integration/iam_roles_test.go
index e9188942a..8df90a7be 100644
--- a/tests/integration/iam_roles_test.go
+++ b/tests/integration/iam_roles_test.go
@@ -108,10 +108,9 @@ func TestIAMRolesAndProfiles(t *testing.T) {
require.Equal(t, "updated", aws.StringValue(got.Role.Description))
require.Equal(t, int64(7200), aws.Int64Value(got.Role.MaxSessionDuration))
- // Server-side MaxSessionDuration range guard (900-43200) isn't reachable
- // via the AWS SDK: UpdateRoleInput carries min:"3600" so SDK.Validate()
- // blocks values < 3600 before dispatch. The gateway guard is covered by
- // handlers/iam/roles_test.go TestCreateRole_MaxSessionDuration_TooSmall.
+ // The MaxSessionDuration range guard (3600-43200) is unreachable here: the
+ // SDK refuses values below 3600 before dispatch. handlers/iam covers it in
+ // TestRole_MaxSessionDurationBounds.
// UpdateAssumeRolePolicy — swap document, no enforcement yet.
_, err = iamCli.UpdateAssumeRolePolicy(&iam.UpdateAssumeRolePolicyInput{
diff --git a/tests/integration/iam_test.go b/tests/integration/iam_test.go
index 28f20ab4c..e88d0d2a3 100644
--- a/tests/integration/iam_test.go
+++ b/tests/integration/iam_test.go
@@ -234,7 +234,7 @@ func TestIAMAccessKeyLifecycle(t *testing.T) {
// TestIAMUserAuthentication builds a scoped AWS client with alice's key and
// confirms signing is honoured (active key) / rejected (deactivated key, bad
// secret, bogus ID). Alice has no policy, so the authenticated call is
-// authz-denied — AccessDenied (not an authn error) proves the active key
+// authz-denied — UnauthorizedOperation (not an authn error) proves the active key
// signed and was accepted. The final root DescribeInstances call needs the
// EC2 daemon subjects stubbed since it actually dispatches over NATS; every
// other assertion here is rejected before dispatch.
@@ -252,11 +252,11 @@ func TestIAMUserAuthentication(t *testing.T) {
aliceSecret := aws.StringValue(k.AccessKey.SecretAccessKey)
// Active, correctly-signed key with no policy: the request authenticates
- // then default-denies. AccessDenied (vs the InvalidClientTokenId /
+ // then default-denies. UnauthorizedOperation (vs the InvalidClientTokenId /
// SignatureDoesNotMatch below) is the signal that authn succeeded.
aliceCli := gw.ClientsWithCreds(t, aliceKeyID, aliceSecret)
_, err = aliceCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = iamCli.UpdateAccessKey(&iam.UpdateAccessKeyInput{
UserName: aws.String(iamUserAlice),
@@ -487,7 +487,7 @@ func TestIAMPolicyAttachmentEnforcement(t *testing.T) {
// --- Enforcement ---
_, err = charlieCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = charlieCli.IAM.ListUsers(&iam.ListUsersInput{})
requireAWSErrorCode(t, err, "AccessDenied")
@@ -499,7 +499,7 @@ func TestIAMPolicyAttachmentEnforcement(t *testing.T) {
require.NoError(t, err, "alice iam:ListUsers")
_, err = aliceCli.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = aliceCli.IAM.CreateUser(&iam.CreateUserInput{UserName: aws.String("hack")})
requireAWSErrorCode(t, err, "AccessDenied")
@@ -508,7 +508,7 @@ func TestIAMPolicyAttachmentEnforcement(t *testing.T) {
_, err = bobCli.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
require.NoError(t, err, "bob ec2:DescribeKeyPairs")
_, err = bobCli.EC2.TerminateInstances(&ec2.TerminateInstancesInput{InstanceIds: []*string{aws.String("i-fake")}})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = bobCli.IAM.ListUsers(&iam.ListUsersInput{})
requireAWSErrorCode(t, err, "AccessDenied")
@@ -533,7 +533,7 @@ func TestIAMPolicyAttachmentEnforcement(t *testing.T) {
_, err = aliceCli.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
require.NoError(t, err, "alice ec2:DescribeKeyPairs (Describe*)")
_, err = aliceCli.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String("x")})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = aliceCli.IAM.ListUsers(&iam.ListUsersInput{})
requireAWSErrorCode(t, err, "AccessDenied")
@@ -583,7 +583,7 @@ func TestIAMPolicyLifecycle(t *testing.T) {
_, err = iamCli.DetachUserPolicy(&iam.DetachUserPolicyInput{UserName: aws.String(iamUserAlice), PolicyArn: aws.String(descAllArn)})
require.NoError(t, err, "detach EC2DescribeAll")
_, err = aliceCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = iamCli.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(denyArn)})
requireAWSErrorCode(t, err, "DeleteConflict")
diff --git a/tests/integration/predastore_test.go b/tests/integration/predastore_test.go
index 12ac2b6e0..feae0ce87 100644
--- a/tests/integration/predastore_test.go
+++ b/tests/integration/predastore_test.go
@@ -90,7 +90,7 @@ func predastoreS3Client(t *testing.T, fixture *testpredastore.Fixture) *s3.S3 {
t.Helper()
httpClient := &http.Client{
Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // self-signed cert generated by this same fixture, localhost-only
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
sess, err := session.NewSession(&aws.Config{
diff --git a/tests/integration/request_conformance_test.go b/tests/integration/request_conformance_test.go
index 9c222f172..c15e51f35 100644
--- a/tests/integration/request_conformance_test.go
+++ b/tests/integration/request_conformance_test.go
@@ -8,6 +8,7 @@ import (
"io"
"net"
"net/http"
+ "strings"
"testing"
"time"
@@ -50,8 +51,10 @@ func TestRequestConformance(t *testing.T) {
startRequestConformanceBackends(t, gw)
sender, err := newRequestSender(gw, service)
require.NoError(t, err)
+ options := awsmodel.RequestOptions{AccountID: gw.AccountID, Region: testRegion, AccessKeyID: testAccessKeyID}
+ options.Fixtures = createRequestFixtures(t, sender, service, gw.AccountID)
for _, operation := range coverage.Implemented {
- plan, err := awsmodel.GenerateRequests(service, operation, awsmodel.RequestOptions{AccountID: gw.AccountID, Region: testRegion})
+ plan, err := awsmodel.GenerateRequests(service, operation, options)
require.NoError(t, err, operation)
suiteRequestConformance.recordOperation(service, operation, plan)
judge := plan.NewJudge()
@@ -82,6 +85,56 @@ func startRequestConformanceBackends(t *testing.T, gw *Gateway) {
StartServiceDaemonLite(t, gw)
}
+// requestFixture is the name every fixture resource takes.
+const requestFixture = "conformance-fixture"
+
+// createRequestFixtures creates one resource of each kind the service's
+// generated requests name, so a case reaches its resource instead of ending
+// in not-found. It returns the members that name them.
+func createRequestFixtures(t *testing.T, sender *requestSender, service awsmodel.Service, accountID string) map[string]string {
+ t.Helper()
+ create := func(operation string, input map[string]any) {
+ t.Helper()
+ result, err := sender.send(awsmodel.RequestCase{Operation: operation, Input: input})
+ require.NoError(t, err, operation)
+ require.True(t, result.Status >= 200 && result.Status < 300, "create fixture: %s: %d %s: %s", operation, result.Status, result.Code, result.Message)
+ }
+ switch service {
+ case awsmodel.IAM:
+ const oidcHost = "fixture.example.com"
+ create("CreateRole", map[string]any{"RoleName": requestFixture, "AssumeRolePolicyDocument": `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`})
+ create("CreatePolicy", map[string]any{"PolicyName": requestFixture, "PolicyDocument": `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`})
+ create("CreateOpenIDConnectProvider", map[string]any{"Url": "https://" + oidcHost, "ClientIDList": []any{"sts.amazonaws.com"}, "ThumbprintList": []any{strings.Repeat("a", 40)}})
+ create("CreateInstanceProfile", map[string]any{"InstanceProfileName": requestFixture})
+ create("CreateGroup", map[string]any{"GroupName": requestFixture})
+ create("CreateUser", map[string]any{"UserName": requestFixture})
+ return map[string]string{
+ "RoleName": requestFixture,
+ "PolicyArn": "arn:aws:iam::" + accountID + ":policy/" + requestFixture,
+ "OpenIDConnectProviderArn": "arn:aws:iam::" + accountID + ":oidc-provider/" + oidcHost,
+ "InstanceProfileName": requestFixture,
+ "GroupName": requestFixture,
+ "UserName": requestFixture,
+ }
+ case awsmodel.ECS:
+ create("CreateCluster", map[string]any{"clusterName": requestFixture})
+ return map[string]string{
+ "cluster": requestFixture,
+ "clusters": requestFixture,
+ "resourceArn": "arn:aws:ecs:" + testRegion + ":" + accountID + ":cluster/" + requestFixture,
+ }
+ case awsmodel.ECR:
+ create("CreateRepository", map[string]any{"repositoryName": requestFixture})
+ return map[string]string{
+ "repositoryName": requestFixture,
+ "repositoryNames": requestFixture,
+ "resourceArn": "arn:aws:ecr:" + testRegion + ":" + accountID + ":repository/" + requestFixture,
+ }
+ default:
+ return nil
+ }
+}
+
type requestSender struct {
endpoint string
service awsmodel.Service
diff --git a/tests/integration/routetable_test.go b/tests/integration/routetable_test.go
index c07b3e377..fa45cfcb1 100644
--- a/tests/integration/routetable_test.go
+++ b/tests/integration/routetable_test.go
@@ -390,7 +390,7 @@ func TestReplaceRouteConvergence(t *testing.T) {
DestinationCidrBlock: aws.String("198.51.100.0/24"),
GatewayId: aws.String(igw2),
})
- requireAWSErrorCode(t, err, "InvalidRoute.NotFound")
+ requireAWSErrorCode(t, err, "InvalidParameterValue")
// Local route is immutable.
_, err = c.ReplaceRoute(&ec2.ReplaceRouteInput{
@@ -413,7 +413,7 @@ func TestReplaceRouteConvergence(t *testing.T) {
DestinationCidrBlock: aws.String(dest),
GatewayId: aws.String("igw-00000000000000000"),
})
- requireAWSErrorCode(t, err, "InvalidInternetGatewayID.NotFound")
+ requireAWSErrorCode(t, err, "InvalidGatewayID.NotFound")
// IGW attached to a different VPC than the route table.
_, err = c.ReplaceRoute(&ec2.ReplaceRouteInput{
diff --git a/tests/integration/run_instances_passrole_test.go b/tests/integration/run_instances_passrole_test.go
index 76a65497f..c313d561f 100644
--- a/tests/integration/run_instances_passrole_test.go
+++ b/tests/integration/run_instances_passrole_test.go
@@ -114,7 +114,7 @@ func testInstanceProfilePassRole(t *testing.T, path string) {
// No PassRole grant: denied before any NATS dispatch, so no daemon stub
// is needed for this call to resolve promptly.
_, err = sessionCli.EC2.RunInstances(runInput)
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
// An explicit path-scoped deny must override an otherwise unrestricted grant.
denyPolicy, err := iamCli.CreatePolicy(&iam.CreatePolicyInput{
@@ -131,12 +131,12 @@ func testInstanceProfilePassRole(t *testing.T, path string) {
})
require.NoError(t, err)
_, err = sessionCli.EC2.RunInstances(runInput)
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = sessionCli.EC2.AssociateIamInstanceProfile(&ec2.AssociateIamInstanceProfileInput{
InstanceId: aws.String("i-0123456789abcdef0"),
IamInstanceProfile: runInput.IamInstanceProfile,
})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
_, err = iamCli.DetachRolePolicy(&iam.DetachRolePolicyInput{
RoleName: callerRoleOut.Role.RoleName,
PolicyArn: denyPolicy.Policy.Arn,
diff --git a/tests/integration/service_daemon_lite.go b/tests/integration/service_daemon_lite.go
index 7315f0d61..7beb854fe 100644
--- a/tests/integration/service_daemon_lite.go
+++ b/tests/integration/service_daemon_lite.go
@@ -3,6 +3,7 @@
package integration
import (
+ "path/filepath"
"testing"
"github.com/mulgadc/spinifex/spinifex/config"
@@ -27,6 +28,11 @@ func StartServiceDaemonLite(t *testing.T, gw *Gateway) {
acm, err := handlers_acm.NewACMServiceImplWithNATS(t.Context(), cfg, nc, masterKey)
require.NoError(t, err, "construct ACM service")
+ // Without a tenant CA no mode can issue, so every RequestCertificate would
+ // be refused. The generated requests name example.com domains.
+ caDir := t.TempDir()
+ acm.TenantCA, err = handlers_acm.LoadOrCreateTenantCA(filepath.Join(caDir, "ca.pem"), filepath.Join(caDir, "ca.key"), []string{"example.com"})
+ require.NoError(t, err, "create ACM tenant CA")
subscribeServiceMethods(t, nc, "acm", acm)
elbv2, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(cfg, nc, masterKey)
@@ -35,7 +41,7 @@ func StartServiceDaemonLite(t *testing.T, gw *Gateway) {
subscribeServiceMethods(t, nc, "elbv2", elbv2)
eks, err := handlers_eks.NewEKSServiceImpl(handlers_eks.EKSServiceDeps{
- Config: cfg, NATSConn: nc, MasterKey: masterKey, Region: testRegion, HolderID: "integration-test-node", ClusterSize: 1,
+ Config: cfg, NATSConn: nc, MasterKey: masterKey, Region: testRegion, HolderID: "integration-test-node",
})
require.NoError(t, err, "construct EKS service")
t.Cleanup(eks.Shutdown)
diff --git a/tests/integration/shared_nats.go b/tests/integration/shared_nats.go
index ac285f2f8..0a3345609 100644
--- a/tests/integration/shared_nats.go
+++ b/tests/integration/shared_nats.go
@@ -11,6 +11,7 @@ import (
"testing"
"time"
+ "github.com/mulgadc/spinifex/spinifex/clustersize"
"github.com/mulgadc/spinifex/tests/fixtures/scratch"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
@@ -43,6 +44,14 @@ const natsStorePrefix = "spinifex-integration-nats-"
// startSharedNATS boots the one embedded, JetStream-enabled NATS server the
// whole package's tests connect into.
func startSharedNATS() (*sharedNATS, error) {
+ // One embedded, non-clustered server, so every bucket this binary creates is
+ // on one replica. Bucket creation refuses until it is told, and this package
+ // builds its gateway by hand rather than through config.LoadConfig, which is
+ // where a real process declares it.
+ if err := clustersize.Declare(1); err != nil {
+ return nil, err
+ }
+
// The store cannot be a t.TempDir(): the server outlives every individual
// test, so the first test to finish would delete it out from under it.
scratch.SweepAbandoned(os.TempDir(), natsStorePrefix, scratch.DefaultMaxAge)
diff --git a/tests/integration/sts_test.go b/tests/integration/sts_test.go
index f2d6952b4..c4baecca3 100644
--- a/tests/integration/sts_test.go
+++ b/tests/integration/sts_test.go
@@ -176,7 +176,7 @@ func TestSTSAssumeRoleAndGetCallerIdentity(t *testing.T) {
// permissions. The granted-policy positive case lives in
// TestAssumedRoleControlPlaneEnforcement.
_, err = sessionCli.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
- requireAWSErrorCode(t, err, "AccessDenied")
+ requireAWSErrorCode(t, err, "UnauthorizedOperation")
// GetSessionToken is user-only: an assumed-role (ASIA) session must NOT
// be able to mint a user session.