Compare commits

...

6 Commits

23 changed files with 317 additions and 97 deletions
+9
View File
@@ -134,6 +134,9 @@ func marshalEC2Response(action string, output any) ([]byte, error) {
var ec2Actions = map[string]ec2Action{
"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, c *call, accountID string) (any, error) {
if err := ec2instanceapi.ValidateDescribeInstancesInput(input); err != nil {
return nil, err
}
out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, c.NATS, 0, c.NodeIDs, accountID)
if err != nil {
return out, err
@@ -257,6 +260,9 @@ var ec2Actions = map[string]ec2Action{
}
}),
"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, c *call, accountID string) (any, error) {
if err := ec2imageapi.ValidateDescribeImagesInput(input); err != nil {
return nil, err
}
return ec2imageapi.DescribeImages(ctx, input, c.NATS, accountID)
}),
"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, c *call, accountID string) (any, error) {
@@ -287,6 +293,9 @@ var ec2Actions = map[string]ec2Action{
return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
}),
"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
if err := ec2volumeapi.ValidateDescribeVolumesInput(input); err != nil {
return nil, err
}
return ec2volumeapi.DescribeVolumes(ctx, input, c.NATS, accountID)
}),
"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, c *call, accountID string) (any, error) {
@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -30,3 +31,27 @@ func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
"Expected no URL-encoded padding remaining")
}
// A well-prefixed ID of the wrong shape is refused before any service is
// asked, so it can never come back as NotFound.
func TestDescribe_MalformedIDRejectedBeforeLookup(t *testing.T) {
tests := []struct {
action, param, id, code, msg string
}{
{"DescribeInstances", "InstanceId.1", "i-zzz", awserrors.ErrorInvalidInstanceIDMalformed, `Invalid id: "i-zzz"`},
{"DescribeImages", "ImageId.1", "ami-zzz", awserrors.ErrorInvalidAMIIDMalformed, `Invalid id: "ami-zzz"`},
{"DescribeVolumes", "VolumeId.1", "vol-zzz", awserrors.ErrorInvalidParameterValue, "Value (vol-zzz) for parameter volumes is invalid. Expected: 'vol-...'."},
}
for _, tt := range tests {
t.Run(tt.action, func(t *testing.T) {
h := ec2Actions[tt.action]
input, err := h.parse(map[string]string{tt.param: tt.id})
require.NoError(t, err)
_, err = h.dispatch(t.Context(), tt.action, input, &call{Handler: NewHandler(Deps{})}, "123456789012")
code, msg, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, tt.code, code)
assert.Equal(t, tt.msg, msg)
})
}
}
@@ -5,6 +5,7 @@ package image
import (
"context"
"errors"
"regexp"
"strings"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,12 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry without the ami-
// prefix with InvalidAMIID.Malformed.
// imageIDSuffixPattern is the AWS AMI ID shape after ami-: 8 (legacy) or 17
// lowercase hex characters.
var imageIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry that is not
// ami- followed by 8 or 17 lowercase hex characters with InvalidAMIID.Malformed.
func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
if input == nil {
return nil
@@ -22,9 +27,16 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
if input.ImageIds != nil {
for _, imageId := range input.ImageIds {
if imageId != nil && !strings.HasPrefix(*imageId, "ami-") {
if imageId == nil {
continue
}
suffix, ok := strings.CutPrefix(*imageId, "ami-")
if !ok {
return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
}
if !imageIDSuffixPattern.MatchString(suffix) {
return awserrors.Errorf(awserrors.ErrorInvalidAMIIDMalformed, "Invalid id: %q", *imageId)
}
}
}
@@ -34,11 +46,6 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
// DescribeImages implements the EC2 DescribeImages action, returning the AMIs visible to
// accountID from the NATS image service.
func DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, natsConn *nats.Conn, accountID string) (output ec2.DescribeImagesOutput, err error) {
err = ValidateDescribeImagesInput(input)
if err != nil {
return output, err
}
imageService := ec2image.NewNATSImageService(natsConn, 0)
result, err := imageService.DescribeImages(ctx, input, accountID)
if err != nil {
@@ -53,8 +53,8 @@ func TestValidateDescribeImagesInput(t *testing.T) {
name: "MultipleValidImageIds",
input: &ec2.DescribeImagesInput{
ImageIds: []*string{
aws.String("ami-111"),
aws.String("ami-222"),
aws.String("ami-1a2b3c4d"),
aws.String("ami-0123456789abcdef1"),
},
},
wantErr: false,
@@ -63,7 +63,7 @@ func TestValidateDescribeImagesInput(t *testing.T) {
name: "MixedValidAndInvalidImageIds",
input: &ec2.DescribeImagesInput{
ImageIds: []*string{
aws.String("ami-valid"),
aws.String("ami-0123456789abcdef0"),
aws.String("invalid-id"),
},
},
@@ -85,11 +85,24 @@ func TestValidateDescribeImagesInput(t *testing.T) {
err := ValidateDescribeImagesInput(tt.input)
if tt.wantErr {
assert.Error(t, err)
assert.Equal(t, tt.errMsg, err.Error())
code, ok := awserrors.ResolveErrorCode(err)
assert.True(t, ok)
assert.Equal(t, tt.errMsg, code)
} else {
assert.NoError(t, err)
}
})
}
}
// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
// characters with Malformed, not NotFound.
func TestValidateDescribeImagesInput_MalformedSuffix(t *testing.T) {
for _, id := range []string{"ami-zzz", "ami-0123456789ABCDEF0", "ami-0123456789abcdef", "ami-"} {
err := ValidateDescribeImagesInput(&ec2.DescribeImagesInput{ImageIds: []*string{aws.String(id)}})
code, msg, ok := awserrors.ResolveErrorDetail(err)
assert.True(t, ok, id)
assert.Equal(t, awserrors.ErrorInvalidAMIIDMalformed, code, id)
assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
}
}
@@ -39,13 +39,6 @@ func TestCreateImage_NilNATS(t *testing.T) {
assert.Error(t, err)
}
func TestDescribeImages_ValidationErrors(t *testing.T) {
_, err := DescribeImages(context.Background(), &ec2.DescribeImagesInput{
ImageIds: []*string{aws.String("bad-id")},
}, nil, "")
assert.EqualError(t, err, awserrors.ErrorInvalidAMIIDMalformed)
}
func TestDescribeImages_NilNATS(t *testing.T) {
_, err := DescribeImages(context.Background(), nil, nil, "acct-123")
assert.Error(t, err)
@@ -89,6 +89,12 @@ func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, n
return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
}
// ValidateDescribeInstancesInput rejects any InstanceIds entry that is not a
// well-formed instance ID with InvalidInstanceID.Malformed.
func ValidateDescribeInstancesInput(input *ec2.DescribeInstancesInput) error {
return validateInstanceIDFormat(input.InstanceIds)
}
// DescribeInstancesChecked is the customer-facing variant served by the
// gateway's DescribeInstances action. On top of DescribeInstances' aggregation,
// it asserts InvalidInstanceID.NotFound for any explicitly named instance ID
@@ -25,7 +25,7 @@ var instanceIDPattern = regexp.MustCompile(`^i-([0-9a-f]{8}|[0-9a-f]{17})$`)
func validateInstanceIDFormat(ids []*string) error {
for _, id := range ids {
if id != nil && !instanceIDPattern.MatchString(*id) {
return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
return awserrors.Errorf(awserrors.ErrorInvalidInstanceIDMalformed, "Invalid id: %q", *id)
}
}
return nil
@@ -57,8 +57,10 @@ func TestValidateInstanceIDFormat(t *testing.T) {
}
for _, id := range malformed {
err := validateInstanceIDFormat([]*string{aws.String(id)})
require.Error(t, err, id)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), id)
code, msg, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok, id)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, id)
assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
}
err := validateInstanceIDFormat([]*string{aws.String("i-0123456789abcdef0"), aws.String("i-abc")})
require.Error(t, err, "one malformed id fails the whole batch")
@@ -100,8 +102,9 @@ func TestLifecycleVerbs_MalformedIDRejectedBeforeNATS(t *testing.T) {
}
for name, call := range calls {
err := call()
require.Error(t, err, name)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), name)
code, ok := awserrors.ResolveErrorCode(err)
require.True(t, ok, name)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, name)
}
require.NoError(t, nc.Flush())
assert.Zero(t, requests.Load(), "no NATS request may be sent for a batch holding a malformed id")
@@ -5,6 +5,7 @@ package volume
import (
"context"
"errors"
"regexp"
"strings"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,13 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateDescribeVolumesInput accepts a nil input and rejects any VolumeIds entry without the
// vol- prefix with InvalidVolumeID.Malformed.
// volumeIDSuffixPattern is the AWS volume ID shape after vol-: 8 (legacy) or
// 17 lowercase hex characters.
var volumeIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
// ValidateDescribeVolumesInput accepts a nil input, rejects any VolumeIds entry without the
// vol- prefix with InvalidVolumeID.Malformed, and any with a malformed suffix with
// InvalidParameterValue, as AWS does.
func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
if input == nil {
return nil
@@ -22,9 +28,17 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
if input.VolumeIds != nil {
for _, volumeId := range input.VolumeIds {
if volumeId != nil && !strings.HasPrefix(*volumeId, "vol-") {
if volumeId == nil {
continue
}
suffix, ok := strings.CutPrefix(*volumeId, "vol-")
if !ok {
return errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
}
if !volumeIDSuffixPattern.MatchString(suffix) {
return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
"Value (%s) for parameter volumes is invalid. Expected: 'vol-...'.", *volumeId)
}
}
}
@@ -35,11 +49,6 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
func DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, natsConn *nats.Conn, accountID string) (ec2.DescribeVolumesOutput, error) {
var output ec2.DescribeVolumesOutput
err := ValidateDescribeVolumesInput(input)
if err != nil {
return output, err
}
volumeService := ec2volume.NewNATSVolumeService(natsConn)
result, err := volumeService.DescribeVolumes(ctx, input, accountID)
if err != nil {
@@ -5,6 +5,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/stretchr/testify/assert"
)
@@ -36,9 +37,9 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
name: "ValidMultipleVolumeIds",
input: &ec2.DescribeVolumesInput{
VolumeIds: []*string{
aws.String("vol-abc123"),
aws.String("vol-def456"),
aws.String("vol-ghi789"),
aws.String("vol-0123456789abcdef1"),
aws.String("vol-1a2b3c4d"),
aws.String("vol-0123456789abcdef2"),
},
},
wantErr: false,
@@ -71,7 +72,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
name: "MixedValidAndInvalid",
input: &ec2.DescribeVolumesInput{
VolumeIds: []*string{
aws.String("vol-valid123"),
aws.String("vol-0123456789abcdef3"),
aws.String("invalid-id"),
},
},
@@ -81,7 +82,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
{
name: "NilVolumeIdInList",
input: &ec2.DescribeVolumesInput{
VolumeIds: []*string{nil, aws.String("vol-abc123")},
VolumeIds: []*string{nil, aws.String("vol-0123456789abcdef1")},
},
wantErr: false,
},
@@ -100,3 +101,15 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
})
}
}
// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
// characters with InvalidParameterValue, not NotFound.
func TestValidateDescribeVolumesInput_MalformedSuffix(t *testing.T) {
for _, id := range []string{"vol-zzz", "vol-0123456789ABCDEF0", "vol-0123456789abcdef", "vol-"} {
err := ValidateDescribeVolumesInput(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}})
code, msg, ok := awserrors.ResolveErrorDetail(err)
assert.True(t, ok, id)
assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, id)
assert.Equal(t, "Value ("+id+") for parameter volumes is invalid. Expected: 'vol-...'.", msg, id)
}
}
@@ -96,13 +96,6 @@ func TestModifyVolume_NilNATS(t *testing.T) {
assert.Error(t, err)
}
func TestDescribeVolumes_ValidationErrors(t *testing.T) {
_, err := DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
VolumeIds: []*string{aws.String("bad-id")},
}, nil, "")
assert.EqualError(t, err, awserrors.ErrorInvalidVolumeIDMalformed)
}
func TestDescribeVolumes_NilNATS(t *testing.T) {
_, err := DescribeVolumes(context.Background(), nil, nil, "acct-123")
assert.Error(t, err)
+22 -5
View File
@@ -224,7 +224,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
groupNames := make(map[string]string)
s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
return &ec2.CreateNetworkInterfaceOutput{
NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames),
NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)),
}, nil
}
@@ -595,7 +595,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
}
s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames))
enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)))
}
// If specific ENI IDs were requested but not found, return error
@@ -1076,9 +1076,26 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
}
}
// vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
// unreadable VPC reads as off, so its interfaces go unnamed.
func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
if err != nil {
slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
return false
}
var rec VPCRecord
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
slog.WarnContext(ctx, "vpcDNSHostnames: VPC unmarshal failed", "vpcId", vpcID, "err", err)
return false
}
return rec.EnableDnsHostnames
}
// eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface, naming
// each security group from groupNames.
func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string) *ec2.NetworkInterface {
// each security group from groupNames. AWS names the interface only while its
// VPC has enableDnsHostnames on.
func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string, dnsHostnames bool) *ec2.NetworkInterface {
// ENIs with spinifex:managed-by tag are system-managed (e.g. by ELBv2)
requesterManaged := record.Tags["spinifex:managed-by"] != ""
@@ -1116,7 +1133,7 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, gro
eni.Groups = groups
}
if s.privateDNSName != nil && record.PrivateIpAddress != "" {
if dnsHostnames && s.privateDNSName != nil && record.PrivateIpAddress != "" {
if name := s.privateDNSName(record.PrivateIpAddress); name != "" {
eni.PrivateDnsName = aws.String(name)
eni.PrivateIpAddresses[0].PrivateDnsName = aws.String(name)
+56 -13
View File
@@ -55,15 +55,11 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
sgId := createTestSG(t, svc, vpcId, "web")
setHostnames(t, svc, vpcId, true)
// Without a namer an interface carries no private name.
unnamed := createTestENI(t, svc, subnetId)
desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
NetworkInterfaceIds: []*string{aws.String(unnamed)},
}, testAccountID)
require.NoError(t, err)
require.Len(t, desc.NetworkInterfaces, 1)
assert.Nil(t, desc.NetworkInterfaces[0].PrivateDnsName)
assert.Nil(t, describeTestENI(t, svc, unnamed).PrivateDnsName)
svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
@@ -72,13 +68,8 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
desc, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
NetworkInterfaceIds: []*string{out.NetworkInterface.NetworkInterfaceId},
}, testAccountID)
require.NoError(t, err)
require.Len(t, desc.NetworkInterfaces, 1)
for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc.NetworkInterfaces[0]} {
desc := describeTestENI(t, svc, *out.NetworkInterface.NetworkInterfaceId)
for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc} {
want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName), name)
require.Len(t, eni.PrivateIpAddresses, 1, name)
@@ -89,6 +80,58 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
}
}
func TestNetworkInterface_PrivateDnsNameFollowsVpcDnsHostnames(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
vpcId := createTestVPC(t, svc, "10.0.0.0/16")
subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
// A new VPC has enableDnsHostnames off, so its interfaces go unnamed.
out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetId),
}, testAccountID)
require.NoError(t, err)
eniId := *out.NetworkInterface.NetworkInterfaceId
for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": describeTestENI(t, svc, eniId)} {
assert.Nil(t, eni.PrivateDnsName, name)
require.Len(t, eni.PrivateIpAddresses, 1, name)
assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName, name)
}
// Turning the flag on names the existing interface.
setHostnames(t, svc, vpcId, true)
eni := describeTestENI(t, svc, eniId)
want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName))
assert.Equal(t, want, aws.StringValue(eni.PrivateIpAddresses[0].PrivateDnsName))
// And turning it off again drops the name.
setHostnames(t, svc, vpcId, false)
eni = describeTestENI(t, svc, eniId)
assert.Nil(t, eni.PrivateDnsName)
assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName)
}
func setHostnames(t *testing.T, svc *VPCServiceImpl, vpcId string, on bool) {
t.Helper()
_, err := svc.ModifyVpcAttribute(context.Background(), &ec2.ModifyVpcAttributeInput{
VpcId: aws.String(vpcId),
EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(on)},
}, testAccountID)
require.NoError(t, err)
}
func describeTestENI(t *testing.T, svc *VPCServiceImpl, eniId string) *ec2.NetworkInterface {
t.Helper()
desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
NetworkInterfaceIds: []*string{aws.String(eniId)},
}, testAccountID)
require.NoError(t, err)
require.Len(t, desc.NetworkInterfaces, 1)
return desc.NetworkInterfaces[0]
}
func TestCreateNetworkInterface_SequentialIPs(t *testing.T) {
t.Parallel()
svc := setupTestVPCService(t)
+14 -3
View File
@@ -288,7 +288,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
names = []string{defaultCluster}
}
includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
out := &ecs.DescribeClustersOutput{}
out := &ecs.DescribeClustersOutput{Clusters: []*ecs.Cluster{}, Failures: []*ecs.Failure{}}
for _, name := range names {
name = ClusterShortName(name)
var rec ClusterRecord
@@ -458,7 +458,7 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
if family == "" {
return nil, errors.New(awserrors.ErrorECSInvalidParameter)
}
if err := validateContainerDefs(input.ContainerDefinitions); err != nil {
if err := validateContainerDefs(input.ContainerDefinitions, aws.StringValue(input.Memory)); err != nil {
return nil, err
}
warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions)
@@ -497,12 +497,23 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
// The test here is per value, not per field: an empty list or a false/unset
// flag requests nothing and is accepted, because there is nothing to enforce
// or refuse. Only a value that asks for the restriction is rejected.
func validateContainerDefs(defs []*ecs.ContainerDefinition) error {
//
// Ahead of those, AWS's own rules: every container needs an image, and without
// a task-level memory every container needs memory or memoryReservation.
func validateContainerDefs(defs []*ecs.ContainerDefinition, taskMemory string) error {
for _, c := range defs {
if c == nil {
continue
}
name := aws.StringValue(c.Name)
if aws.StringValue(c.Image) == "" {
return awserrors.Errorf(awserrors.ErrorECSClientException,
"Container.image should not be null or empty.")
}
if taskMemory == "" && c.Memory == nil && c.MemoryReservation == nil {
return awserrors.Errorf(awserrors.ErrorECSClientException,
"Invalid setting for container '%s'. At least one of 'memory' or 'memoryReservation' must be specified.", name)
}
if len(c.Secrets) > 0 {
return errors.New(awserrors.ErrorECSInvalidParameter)
}
+24
View File
@@ -812,6 +812,12 @@ func TestService_ListTasks_EmptyIsPresentNotAbsent(t *testing.T) {
assert.Empty(t, out.TaskArns)
}
func TestService_ListTasks_ClusterNotFound(t *testing.T) {
svc, _ := newTestService(t)
_, err := svc.ListTasks(context.Background(), &ecs.ListTasksInput{Cluster: aws.String("ghost")}, testAccountID)
require.EqualError(t, err, awserrors.ErrorECSClusterNotFound)
}
func TestService_ListContainerInstances_EmptyIsPresentNotAbsent(t *testing.T) {
svc, _ := newTestService(t)
_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
@@ -829,3 +835,21 @@ func TestAccountIDFromBucket(t *testing.T) {
_, ok = accountIDFromBucket("some-other-bucket")
assert.False(t, ok)
}
func TestService_DescribeClusters_ListsAlwaysPresent(t *testing.T) {
svc, _ := newTestService(t)
_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
require.NoError(t, err)
found, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
require.NoError(t, err)
require.Len(t, found.Clusters, 1)
assert.NotNil(t, found.Failures)
assert.Empty(t, found.Failures)
miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
require.NoError(t, err)
require.Len(t, miss.Failures, 1)
assert.NotNil(t, miss.Clusters)
assert.Empty(t, miss.Clusters)
}
+2 -2
View File
@@ -313,7 +313,7 @@ func TestService_CreateService_ProjectsNetworkConfigurationAndManagedTags(t *tes
Family: aws.String("app"),
NetworkMode: aws.String(NetworkModeAwsvpc),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -352,7 +352,7 @@ func TestService_UpdateService_ChangesNetworkConfiguration(t *testing.T) {
Family: aws.String("app"),
NetworkMode: aws.String(NetworkModeAwsvpc),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -16,6 +16,8 @@ import (
// agent to reach anything other than PENDING.
func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ecsdomain.TaskRecord) {
t.Helper()
_, err := svc.CreateCluster(t.Context(), &ecs.CreateClusterInput{ClusterName: aws.String(cluster)}, ecsdomain.TestAccountID)
require.NoError(t, err)
kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
require.NoError(t, err)
for _, r := range recs {
+57 -18
View File
@@ -19,7 +19,7 @@ func TestRegisterTaskDefinition_RejectsSecrets(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
Secrets: []*ecs.Secret{{
Name: aws.String("DB_PASSWORD"), ValueFrom: aws.String("arn:aws:ssm:::parameter/db"),
}},
@@ -37,7 +37,7 @@ func TestRegisterTaskDefinition_AcceptsUnsupportedLogDriver(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
LogConfiguration: &ecs.LogConfiguration{
LogDriver: aws.String("awslogs"),
Options: map[string]*string{"awslogs-group": aws.String("/ecs/app")},
@@ -67,7 +67,7 @@ func TestRegisterTaskDefinition_EchoesRequiresCompatibilities(t *testing.T) {
Family: aws.String("app"),
RequiresCompatibilities: aws.StringSlice([]string{"EC2"}),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -91,7 +91,7 @@ func TestRegisterTaskDefinition_EchoesRuntimePlatform(t *testing.T) {
OperatingSystemFamily: aws.String("LINUX"),
},
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -113,7 +113,7 @@ func TestRegisterTaskDefinition_RuntimePlatformOmittedWhenUnset(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestRegisterTaskDefinition_EchoesRuntimeFields(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
User: aws.String("1000:1000"),
ReadonlyRootFilesystem: aws.Bool(true),
Privileged: aws.Bool(false),
@@ -310,7 +310,7 @@ func TestRegisterTaskDefinition_RuntimeFieldsOmittedWhenUnset(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
}},
}, testAccountID)
require.NoError(t, err)
@@ -347,7 +347,7 @@ func TestRegisterTaskDefinition_EchoesEmptyCollections(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
Environment: []*ecs.KeyValuePair{},
MountPoints: []*ecs.MountPoint{},
VolumesFrom: []*ecs.VolumeFrom{},
@@ -386,7 +386,7 @@ func TestRegisterTaskDefinition_FillsAWSDefaults(t *testing.T) {
Family: aws.String("app"),
NetworkMode: aws.String("awsvpc"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(80)}},
}},
}, testAccountID)
@@ -420,7 +420,7 @@ func TestRegisterTaskDefinition_BridgeLeavesHostPortUnset(t *testing.T) {
Family: aws.String("app"),
NetworkMode: aws.String("bridge"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(53), Protocol: aws.String("udp")}},
}},
}, testAccountID)
@@ -435,7 +435,7 @@ func TestRegisterTaskDefinition_AcceptsEmptyRuntimeFields(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
MountPoints: []*ecs.MountPoint{},
VolumesFrom: []*ecs.VolumeFrom{},
SystemControls: []*ecs.SystemControl{},
@@ -455,7 +455,7 @@ func TestRegisterTaskDefinition_RejectsMountPoints(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
MountPoints: []*ecs.MountPoint{{
SourceVolume: aws.String("data"), ContainerPath: aws.String("/data"),
}},
@@ -476,7 +476,7 @@ func TestRegisterTaskDefinition_RejectsVolumesFrom(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
VolumesFrom: []*ecs.VolumeFrom{{SourceContainer: aws.String("other")}},
}},
}, testAccountID)
@@ -494,7 +494,7 @@ func TestRegisterTaskDefinition_RejectsLinuxParametersDevices(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
LinuxParameters: &ecs.LinuxParameters{
Devices: []*ecs.Device{{HostPath: aws.String("/dev/fuse")}},
},
@@ -514,7 +514,7 @@ func TestRegisterTaskDefinition_RejectsInitProcessEnabled(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
LinuxParameters: &ecs.LinuxParameters{InitProcessEnabled: aws.Bool(true)},
}},
}, testAccountID)
@@ -533,7 +533,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("shm"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
LinuxParameters: &ecs.LinuxParameters{SharedMemorySize: aws.Int64(64)},
}},
}, testAccountID)
@@ -545,7 +545,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("tmpfs"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
LinuxParameters: &ecs.LinuxParameters{
Tmpfs: []*ecs.Tmpfs{{ContainerPath: aws.String("/tmp"), Size: aws.Int64(64)}},
},
@@ -567,7 +567,7 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"),
ContainerDefinitions: []*ecs.ContainerDefinition{{
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
User: aws.String("1000"),
ReadonlyRootFilesystem: aws.Bool(true),
Privileged: aws.Bool(true),
@@ -608,3 +608,42 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
assert.Equal(t, "net.core.somaxconn", ac.SystemControls[0].Namespace)
assert.Equal(t, []string{"SYS_PTRACE"}, ac.CapAdd)
}
// AWS rejects a container with no image before checking its memory, and
// without task-level memory requires memory or memoryReservation per container.
func TestRegisterTaskDefinition_RequiresImageAndMemory(t *testing.T) {
tests := []struct {
name string
taskMemory *string
container *ecs.ContainerDefinition
wantErr string
}{
{"no image or memory", nil, &ecs.ContainerDefinition{Name: aws.String("web")},
"Container.image should not be null or empty."},
{"empty image", aws.String("512"), &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("")},
"Container.image should not be null or empty."},
{"no memory", nil, &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("registry/web:1")},
"Invalid setting for container 'web'. At least one of 'memory' or 'memoryReservation' must be specified."},
{"memoryReservation only", nil, &ecs.ContainerDefinition{
Name: aws.String("web"), Image: aws.String("registry/web:1"), MemoryReservation: aws.Int64(128)}, ""},
{"task-level memory", aws.String("512"), &ecs.ContainerDefinition{
Name: aws.String("web"), Image: aws.String("registry/web:1")}, ""},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
svc, _ := newTestService(t)
_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
Family: aws.String("app"), Memory: tc.taskMemory,
ContainerDefinitions: []*ecs.ContainerDefinition{tc.container},
}, testAccountID)
if tc.wantErr == "" {
require.NoError(t, err)
return
}
code, msg, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorECSClientException, code)
assert.Equal(t, tc.wantErr, msg)
})
}
}
+8
View File
@@ -303,6 +303,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
if err != nil {
return nil, err
}
var clusterRec ClusterRecord
found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
if err != nil {
return nil, err
}
if !found {
return nil, errors.New(awserrors.ErrorECSClusterNotFound)
}
keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
if err != nil {
return nil, err
+8 -4
View File
@@ -3503,14 +3503,18 @@ func (s *ELBv2ServiceImpl) tgRecordToSDK(r *TargetGroupRecord, lbArns []string)
HealthCheckEnabled: aws.Bool(r.HealthCheck.Enabled),
HealthCheckProtocol: aws.String(r.HealthCheck.Protocol),
HealthCheckPort: aws.String(r.HealthCheck.Port),
HealthCheckPath: aws.String(r.HealthCheck.Path),
HealthCheckIntervalSeconds: aws.Int64(r.HealthCheck.IntervalSeconds),
HealthCheckTimeoutSeconds: aws.Int64(r.HealthCheck.TimeoutSeconds),
HealthyThresholdCount: aws.Int64(r.HealthCheck.HealthyThreshold),
UnhealthyThresholdCount: aws.Int64(r.HealthCheck.UnhealthyThreshold),
Matcher: &elbv2.Matcher{
HttpCode: aws.String(r.HealthCheck.Matcher),
},
}
// AWS omits both keys on TCP target groups, which have no path or matcher.
if r.HealthCheck.Path != "" {
tg.HealthCheckPath = aws.String(r.HealthCheck.Path)
}
if r.HealthCheck.Matcher != "" {
tg.Matcher = &elbv2.Matcher{HttpCode: aws.String(r.HealthCheck.Matcher)}
}
// Omitted for NLB (TCP/UDP/TLS) target groups, matching AWS, which returns
+5 -4
View File
@@ -459,11 +459,12 @@ func TestCreateTargetGroup_TCPProtocol(t *testing.T) {
assert.Equal(t, int64(5432), *tg.Port)
// NLB health check defaults: TCP protocol, no path, no matcher.
assert.Equal(t, "TCP", *tg.HealthCheckProtocol)
assert.Empty(t, *tg.HealthCheckPath)
assert.Empty(t, *tg.Matcher.HttpCode)
assert.Nil(t, tg.HealthCheckPath)
assert.Nil(t, tg.Matcher)
assert.Equal(t, int64(30), *tg.HealthCheckIntervalSeconds)
assert.Equal(t, int64(10), *tg.HealthCheckTimeoutSeconds)
assert.Equal(t, int64(3), *tg.HealthyThresholdCount)
assert.Equal(t, int64(3), *tg.UnhealthyThresholdCount)
assert.Equal(t, int64(5), *tg.HealthyThresholdCount)
assert.Equal(t, int64(2), *tg.UnhealthyThresholdCount)
}
func TestCreateTargetGroup_NLBProtocols(t *testing.T) {
+2 -2
View File
@@ -99,8 +99,8 @@ const (
// Default health check values (NLB).
DefaultNLBHealthCheckInterval = 30
DefaultNLBHealthCheckTimeout = 10
DefaultNLBHealthyThreshold = 3
DefaultNLBUnhealthyThreshold = 3
DefaultNLBHealthyThreshold = 5
DefaultNLBUnhealthyThreshold = 2
DefaultNLBHealthCheckProtocol = ProtocolTCP
DefaultNLBHealthCheckPort = "traffic-port"
+1 -1
View File
@@ -1268,7 +1268,7 @@ var ErrorLookup = map[string]ErrorMessage{
ErrorEKSResourceInUse: {HTTPCode: 409, Message: "A cluster with this name already exists in this account and Region. Delete the existing cluster, or choose a different name."},
ErrorEKSResourceNotFound: {HTTPCode: 404, Message: "The specified cluster could not be found. You can view your available clusters with ListClusters. Amazon EKS clusters are Region specific."},
ErrorECSClientException: {HTTPCode: 400, Message: "Unable to describe task definition."},
ErrorECSClusterNotFound: {HTTPCode: 400, Message: "The specified cluster was not found. You can view your available clusters with ListClusters. Amazon ECS clusters are Region specific."},
ErrorECSClusterNotFound: {HTTPCode: 400, Message: "Cluster not found."},
ErrorECSInvalidParameter: {HTTPCode: 400, Message: "The specified parameter is not valid. Review the available parameters for the API request."},
ErrorECSServiceNotFound: {HTTPCode: 400, Message: "The specified service was not found. You can view your available services with ListServices. Amazon ECS services are cluster specific and Region specific."},
ErrorRetryableError: {HTTPCode: 400, Message: "A request submitted by an AWS service on your behalf could not be completed. The requesting service might automatically retry the request."},