diff --git a/Makefile b/Makefile index 5b7cd7282..70bcf696f 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ else _SECQ = 2>&1 | tee endif -build: go_build build-installer build-lb-agent +build: go_build build-installer build-lb-agent generate-aws-model-coverage # Build spinifex-ui frontend (requires pnpm) build-ui: @@ -146,9 +146,22 @@ test-harness: # In-process integration tier: the real gateway router against embedded NATS # JetStream, with only the daemon-side NATS subjects stubbed. +AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt +AWS_MODEL_CONFORMANCE_MODE ?= fail +AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md test-integration: @echo -e "\n....Running in-process integration tests...." - $(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ) + $(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ) + @cat $(AWS_MODEL_CONFORMANCE_REPORT) + @$(MAKE) --no-print-directory generate-aws-model-coverage + @echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)" + +generate-aws-model-coverage: + @mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT)) + @go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT) + +aws-model-coverage: generate-aws-model-coverage + @cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT) # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which @@ -342,7 +355,7 @@ distro-arm64: distro-clean: rm -rf dist/ -.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \ +.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck nilaway \ diff --git a/cmd/aws-model-coverage/main.go b/cmd/aws-model-coverage/main.go new file mode 100644 index 000000000..e77d13cf1 --- /dev/null +++ b/cmd/aws-model-coverage/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "os" + + "github.com/mulgadc/spinifex/internal/awsmodel" + "github.com/mulgadc/spinifex/spinifex/gateway" +) + +func main() { + dispatch := gateway.AWSOperationInventory() + coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services())) + for _, service := range awsmodel.Services() { + inventory, ok := dispatch[string(service)] + var modelInventory awsmodel.DispatchInventory + switch { + case ok: + modelInventory = awsmodel.DispatchInventory{ + Registered: inventory.Registered, + Stubbed: inventory.Stubbed, + Unsupported: inventory.Unsupported, + } + case service == awsmodel.S3: + modelInventory = awsmodel.DispatchInventory{ + Opaque: true, + Note: "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.", + } + default: + fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service) + os.Exit(1) + } + + coverage, err := awsmodel.CompareOperations(service, modelInventory) + if err != nil { + fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err) + os.Exit(1) + } + coverages = append(coverages, coverage) + } + + fmt.Print(awsmodel.RenderCoverageMarkdown(coverages)) +} diff --git a/docs/README.md b/docs/README.md index 457c407a3..82367522c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,7 @@ # Spinifex Documentation -This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/) +This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/) + +The [AWS model operation coverage](aws-model-operation-coverage.md) report is +generated from Spinifex's dispatch tables and the pinned AWS API models. Run +`make aws-model-coverage` to regenerate and print it. diff --git a/docs/aws-model-operation-coverage.md b/docs/aws-model-operation-coverage.md new file mode 100644 index 000000000..8e1e60b16 --- /dev/null +++ b/docs/aws-model-operation-coverage.md @@ -0,0 +1,500 @@ +# AWS model operation coverage + +Generated from the cached `aws-sdk-go v1.55.8` `api-2.json` models and Spinifex's authoritative gateway dispatch tables. + +Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance. + +| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 | +| ec2 | 2016-11-15 | 625 | 119 | 119 | 0 | 0 | 506 | 0 | +| ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 | +| ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 | +| elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 | +| iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 | +| s3 | 2006-03-01 | 99 | — | — | — | — | — | — | +| sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 | + +## acm + +Implements **9 of 15** modelled operations (60.0%). + +
Implemented (9) + +`AddTagsToCertificate`, `DeleteCertificate`, `DescribeCertificate`, `GetCertificate`, `ImportCertificate`, `ListCertificates`, `ListTagsForCertificate`, `RemoveTagsFromCertificate` + +`RequestCertificate` + + +
+ +
Missing from dispatch (6) + +`ExportCertificate`, `GetAccountConfiguration`, `PutAccountConfiguration`, `RenewCertificate`, `ResendValidationEmail`, `UpdateCertificateOptions` + + +
+ +
Registered stubs (0) + +None. + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (0) + +None. + +
+ +## ec2 + +Implements **119 of 625** modelled operations (19.0%). + +
Implemented (119) + +`AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress` + +`AuthorizeSecurityGroupIngress`, `CancelCapacityReservation`, `CancelSpotInstanceRequests`, `CopyImage`, `CopySnapshot`, `CreateCapacityReservation`, `CreateEgressOnlyInternetGateway`, `CreateImage` + +`CreateInternetGateway`, `CreateKeyPair`, `CreateLaunchTemplate`, `CreateLaunchTemplateVersion`, `CreateNatGateway`, `CreateNetworkInterface`, `CreatePlacementGroup`, `CreateRoute` + +`CreateRouteTable`, `CreateSecurityGroup`, `CreateSnapshot`, `CreateSubnet`, `CreateTags`, `CreateVolume`, `CreateVpc`, `DeleteEgressOnlyInternetGateway` + +`DeleteInternetGateway`, `DeleteKeyPair`, `DeleteLaunchTemplate`, `DeleteLaunchTemplateVersions`, `DeleteNatGateway`, `DeleteNetworkInterface`, `DeletePlacementGroup`, `DeleteRoute` + +`DeleteRouteTable`, `DeleteSecurityGroup`, `DeleteSnapshot`, `DeleteSubnet`, `DeleteTags`, `DeleteVolume`, `DeleteVpc`, `DeregisterImage` + +`DescribeAccountAttributes`, `DescribeAddresses`, `DescribeAddressesAttribute`, `DescribeAvailabilityZones`, `DescribeCapacityReservations`, `DescribeEgressOnlyInternetGateways`, `DescribeIamInstanceProfileAssociations`, `DescribeImageAttribute` + +`DescribeImages`, `DescribeInstanceAttribute`, `DescribeInstanceCreditSpecifications`, `DescribeInstanceStatus`, `DescribeInstanceTypes`, `DescribeInstances`, `DescribeInternetGateways`, `DescribeKeyPairs` + +`DescribeLaunchTemplateVersions`, `DescribeLaunchTemplates`, `DescribeNatGateways`, `DescribeNetworkInterfaces`, `DescribePlacementGroups`, `DescribeRegions`, `DescribeRouteTables`, `DescribeSecurityGroupRules` + +`DescribeSecurityGroups`, `DescribeSnapshots`, `DescribeSpotInstanceRequests`, `DescribeSubnets`, `DescribeTags`, `DescribeVolumeStatus`, `DescribeVolumes`, `DescribeVolumesModifications` + +`DescribeVpcAttribute`, `DescribeVpcs`, `DetachInternetGateway`, `DetachNetworkInterface`, `DetachVolume`, `DisableEbsEncryptionByDefault`, `DisableSerialConsoleAccess`, `DisassociateAddress` + +`DisassociateIamInstanceProfile`, `DisassociateRouteTable`, `EnableEbsEncryptionByDefault`, `EnableSerialConsoleAccess`, `GetConsoleOutput`, `GetEbsEncryptionByDefault`, `GetPasswordData`, `GetSerialConsoleAccessStatus` + +`ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume` + +`ModifyVpcAttribute`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`, `RequestSpotInstances` + +`ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances` + + +
+ +
Missing from dispatch (506) + +`AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr` + +`AllocateHosts`, `AllocateIpamPoolCidr`, `ApplySecurityGroupsToClientVpnTargetNetwork`, `AssignIpv6Addresses`, `AssignPrivateIpAddresses`, `AssignPrivateNatGatewayAddress`, `AssociateClientVpnTargetNetwork`, `AssociateDhcpOptions` + +`AssociateEnclaveCertificateIamRole`, `AssociateInstanceEventWindow`, `AssociateIpamByoasn`, `AssociateIpamResourceDiscovery`, `AssociateNatGatewayAddress`, `AssociateSubnetCidrBlock`, `AssociateTransitGatewayMulticastDomain`, `AssociateTransitGatewayPolicyTable` + +`AssociateTransitGatewayRouteTable`, `AssociateTrunkInterface`, `AssociateVpcCidrBlock`, `AttachClassicLinkVpc`, `AttachVerifiedAccessTrustProvider`, `AttachVpnGateway`, `AuthorizeClientVpnIngress`, `BundleInstance` + +`CancelBundleTask`, `CancelCapacityReservationFleets`, `CancelConversionTask`, `CancelExportTask`, `CancelImageLaunchPermission`, `CancelImportTask`, `CancelReservedInstancesListing`, `CancelSpotFleetRequests` + +`ConfirmProductInstance`, `CopyFpgaImage`, `CreateCapacityReservationFleet`, `CreateCarrierGateway`, `CreateClientVpnEndpoint`, `CreateClientVpnRoute`, `CreateCoipCidr`, `CreateCoipPool` + +`CreateCustomerGateway`, `CreateDefaultSubnet`, `CreateDefaultVpc`, `CreateDhcpOptions`, `CreateFleet`, `CreateFlowLogs`, `CreateFpgaImage`, `CreateInstanceConnectEndpoint` + +`CreateInstanceEventWindow`, `CreateInstanceExportTask`, `CreateIpam`, `CreateIpamExternalResourceVerificationToken`, `CreateIpamPool`, `CreateIpamResourceDiscovery`, `CreateIpamScope`, `CreateLocalGatewayRoute` + +`CreateLocalGatewayRouteTable`, `CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `CreateLocalGatewayRouteTableVpcAssociation`, `CreateManagedPrefixList`, `CreateNetworkAcl`, `CreateNetworkAclEntry`, `CreateNetworkInsightsAccessScope`, `CreateNetworkInsightsPath` + +`CreateNetworkInterfacePermission`, `CreatePublicIpv4Pool`, `CreateReplaceRootVolumeTask`, `CreateReservedInstancesListing`, `CreateRestoreImageTask`, `CreateSnapshots`, `CreateSpotDatafeedSubscription`, `CreateStoreImageTask` + +`CreateSubnetCidrReservation`, `CreateTrafficMirrorFilter`, `CreateTrafficMirrorFilterRule`, `CreateTrafficMirrorSession`, `CreateTrafficMirrorTarget`, `CreateTransitGateway`, `CreateTransitGatewayConnect`, `CreateTransitGatewayConnectPeer` + +`CreateTransitGatewayMulticastDomain`, `CreateTransitGatewayPeeringAttachment`, `CreateTransitGatewayPolicyTable`, `CreateTransitGatewayPrefixListReference`, `CreateTransitGatewayRoute`, `CreateTransitGatewayRouteTable`, `CreateTransitGatewayRouteTableAnnouncement`, `CreateTransitGatewayVpcAttachment` + +`CreateVerifiedAccessEndpoint`, `CreateVerifiedAccessGroup`, `CreateVerifiedAccessInstance`, `CreateVerifiedAccessTrustProvider`, `CreateVpcEndpoint`, `CreateVpcEndpointConnectionNotification`, `CreateVpcEndpointServiceConfiguration`, `CreateVpcPeeringConnection` + +`CreateVpnConnection`, `CreateVpnConnectionRoute`, `CreateVpnGateway`, `DeleteCarrierGateway`, `DeleteClientVpnEndpoint`, `DeleteClientVpnRoute`, `DeleteCoipCidr`, `DeleteCoipPool` + +`DeleteCustomerGateway`, `DeleteDhcpOptions`, `DeleteFleets`, `DeleteFlowLogs`, `DeleteFpgaImage`, `DeleteInstanceConnectEndpoint`, `DeleteInstanceEventWindow`, `DeleteIpam` + +`DeleteIpamExternalResourceVerificationToken`, `DeleteIpamPool`, `DeleteIpamResourceDiscovery`, `DeleteIpamScope`, `DeleteLocalGatewayRoute`, `DeleteLocalGatewayRouteTable`, `DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `DeleteLocalGatewayRouteTableVpcAssociation` + +`DeleteManagedPrefixList`, `DeleteNetworkAcl`, `DeleteNetworkAclEntry`, `DeleteNetworkInsightsAccessScope`, `DeleteNetworkInsightsAccessScopeAnalysis`, `DeleteNetworkInsightsAnalysis`, `DeleteNetworkInsightsPath`, `DeleteNetworkInterfacePermission` + +`DeletePublicIpv4Pool`, `DeleteQueuedReservedInstances`, `DeleteSpotDatafeedSubscription`, `DeleteSubnetCidrReservation`, `DeleteTrafficMirrorFilter`, `DeleteTrafficMirrorFilterRule`, `DeleteTrafficMirrorSession`, `DeleteTrafficMirrorTarget` + +`DeleteTransitGateway`, `DeleteTransitGatewayConnect`, `DeleteTransitGatewayConnectPeer`, `DeleteTransitGatewayMulticastDomain`, `DeleteTransitGatewayPeeringAttachment`, `DeleteTransitGatewayPolicyTable`, `DeleteTransitGatewayPrefixListReference`, `DeleteTransitGatewayRoute` + +`DeleteTransitGatewayRouteTable`, `DeleteTransitGatewayRouteTableAnnouncement`, `DeleteTransitGatewayVpcAttachment`, `DeleteVerifiedAccessEndpoint`, `DeleteVerifiedAccessGroup`, `DeleteVerifiedAccessInstance`, `DeleteVerifiedAccessTrustProvider`, `DeleteVpcEndpointConnectionNotifications` + +`DeleteVpcEndpointServiceConfigurations`, `DeleteVpcEndpoints`, `DeleteVpcPeeringConnection`, `DeleteVpnConnection`, `DeleteVpnConnectionRoute`, `DeleteVpnGateway`, `DeprovisionByoipCidr`, `DeprovisionIpamByoasn` + +`DeprovisionIpamPoolCidr`, `DeprovisionPublicIpv4PoolCidr`, `DeregisterInstanceEventNotificationAttributes`, `DeregisterTransitGatewayMulticastGroupMembers`, `DeregisterTransitGatewayMulticastGroupSources`, `DescribeAddressTransfers`, `DescribeAggregateIdFormat`, `DescribeAwsNetworkPerformanceMetricSubscriptions` + +`DescribeBundleTasks`, `DescribeByoipCidrs`, `DescribeCapacityBlockOfferings`, `DescribeCapacityReservationFleets`, `DescribeCarrierGateways`, `DescribeClassicLinkInstances`, `DescribeClientVpnAuthorizationRules`, `DescribeClientVpnConnections` + +`DescribeClientVpnEndpoints`, `DescribeClientVpnRoutes`, `DescribeClientVpnTargetNetworks`, `DescribeCoipPools`, `DescribeConversionTasks`, `DescribeCustomerGateways`, `DescribeDhcpOptions`, `DescribeElasticGpus` + +`DescribeExportImageTasks`, `DescribeExportTasks`, `DescribeFastLaunchImages`, `DescribeFastSnapshotRestores`, `DescribeFleetHistory`, `DescribeFleetInstances`, `DescribeFleets`, `DescribeFlowLogs` + +`DescribeFpgaImageAttribute`, `DescribeFpgaImages`, `DescribeHostReservationOfferings`, `DescribeHostReservations`, `DescribeHosts`, `DescribeIdFormat`, `DescribeIdentityIdFormat`, `DescribeImportImageTasks` + +`DescribeImportSnapshotTasks`, `DescribeInstanceConnectEndpoints`, `DescribeInstanceEventNotificationAttributes`, `DescribeInstanceEventWindows`, `DescribeInstanceTopology`, `DescribeInstanceTypeOfferings`, `DescribeIpamByoasn`, `DescribeIpamExternalResourceVerificationTokens` + +`DescribeIpamPools`, `DescribeIpamResourceDiscoveries`, `DescribeIpamResourceDiscoveryAssociations`, `DescribeIpamScopes`, `DescribeIpams`, `DescribeIpv6Pools`, `DescribeLocalGatewayRouteTableVirtualInterfaceGroupAssociations`, `DescribeLocalGatewayRouteTableVpcAssociations` + +`DescribeLocalGatewayRouteTables`, `DescribeLocalGatewayVirtualInterfaceGroups`, `DescribeLocalGatewayVirtualInterfaces`, `DescribeLocalGateways`, `DescribeLockedSnapshots`, `DescribeMacHosts`, `DescribeManagedPrefixLists`, `DescribeMovingAddresses` + +`DescribeNetworkAcls`, `DescribeNetworkInsightsAccessScopeAnalyses`, `DescribeNetworkInsightsAccessScopes`, `DescribeNetworkInsightsAnalyses`, `DescribeNetworkInsightsPaths`, `DescribeNetworkInterfaceAttribute`, `DescribeNetworkInterfacePermissions`, `DescribePrefixLists` + +`DescribePrincipalIdFormat`, `DescribePublicIpv4Pools`, `DescribeReplaceRootVolumeTasks`, `DescribeReservedInstances`, `DescribeReservedInstancesListings`, `DescribeReservedInstancesModifications`, `DescribeReservedInstancesOfferings`, `DescribeScheduledInstanceAvailability` + +`DescribeScheduledInstances`, `DescribeSecurityGroupReferences`, `DescribeSnapshotAttribute`, `DescribeSnapshotTierStatus`, `DescribeSpotDatafeedSubscription`, `DescribeSpotFleetInstances`, `DescribeSpotFleetRequestHistory`, `DescribeSpotFleetRequests` + +`DescribeSpotPriceHistory`, `DescribeStaleSecurityGroups`, `DescribeStoreImageTasks`, `DescribeTrafficMirrorFilterRules`, `DescribeTrafficMirrorFilters`, `DescribeTrafficMirrorSessions`, `DescribeTrafficMirrorTargets`, `DescribeTransitGatewayAttachments` + +`DescribeTransitGatewayConnectPeers`, `DescribeTransitGatewayConnects`, `DescribeTransitGatewayMulticastDomains`, `DescribeTransitGatewayPeeringAttachments`, `DescribeTransitGatewayPolicyTables`, `DescribeTransitGatewayRouteTableAnnouncements`, `DescribeTransitGatewayRouteTables`, `DescribeTransitGatewayVpcAttachments` + +`DescribeTransitGateways`, `DescribeTrunkInterfaceAssociations`, `DescribeVerifiedAccessEndpoints`, `DescribeVerifiedAccessGroups`, `DescribeVerifiedAccessInstanceLoggingConfigurations`, `DescribeVerifiedAccessInstances`, `DescribeVerifiedAccessTrustProviders`, `DescribeVolumeAttribute` + +`DescribeVpcClassicLink`, `DescribeVpcClassicLinkDnsSupport`, `DescribeVpcEndpointConnectionNotifications`, `DescribeVpcEndpointConnections`, `DescribeVpcEndpointServiceConfigurations`, `DescribeVpcEndpointServicePermissions`, `DescribeVpcEndpointServices`, `DescribeVpcEndpoints` + +`DescribeVpcPeeringConnections`, `DescribeVpnConnections`, `DescribeVpnGateways`, `DetachClassicLinkVpc`, `DetachVerifiedAccessTrustProvider`, `DetachVpnGateway`, `DisableAddressTransfer`, `DisableAwsNetworkPerformanceMetricSubscription` + +`DisableFastLaunch`, `DisableFastSnapshotRestores`, `DisableImage`, `DisableImageBlockPublicAccess`, `DisableImageDeprecation`, `DisableImageDeregistrationProtection`, `DisableIpamOrganizationAdminAccount`, `DisableSnapshotBlockPublicAccess` + +`DisableTransitGatewayRouteTablePropagation`, `DisableVgwRoutePropagation`, `DisableVpcClassicLink`, `DisableVpcClassicLinkDnsSupport`, `DisassociateClientVpnTargetNetwork`, `DisassociateEnclaveCertificateIamRole`, `DisassociateInstanceEventWindow`, `DisassociateIpamByoasn` + +`DisassociateIpamResourceDiscovery`, `DisassociateNatGatewayAddress`, `DisassociateSubnetCidrBlock`, `DisassociateTransitGatewayMulticastDomain`, `DisassociateTransitGatewayPolicyTable`, `DisassociateTransitGatewayRouteTable`, `DisassociateTrunkInterface`, `DisassociateVpcCidrBlock` + +`EnableAddressTransfer`, `EnableAwsNetworkPerformanceMetricSubscription`, `EnableFastLaunch`, `EnableFastSnapshotRestores`, `EnableImage`, `EnableImageBlockPublicAccess`, `EnableImageDeprecation`, `EnableImageDeregistrationProtection` + +`EnableIpamOrganizationAdminAccount`, `EnableReachabilityAnalyzerOrganizationSharing`, `EnableSnapshotBlockPublicAccess`, `EnableTransitGatewayRouteTablePropagation`, `EnableVgwRoutePropagation`, `EnableVolumeIO`, `EnableVpcClassicLink`, `EnableVpcClassicLinkDnsSupport` + +`ExportClientVpnClientCertificateRevocationList`, `ExportClientVpnClientConfiguration`, `ExportImage`, `ExportTransitGatewayRoutes`, `GetAssociatedEnclaveCertificateIamRoles`, `GetAssociatedIpv6PoolCidrs`, `GetAwsNetworkPerformanceData`, `GetCapacityReservationUsage` + +`GetCoipPoolUsage`, `GetConsoleScreenshot`, `GetDefaultCreditSpecification`, `GetEbsDefaultKmsKeyId`, `GetFlowLogsIntegrationTemplate`, `GetGroupsForCapacityReservation`, `GetHostReservationPurchasePreview`, `GetImageBlockPublicAccessState` + +`GetInstanceMetadataDefaults`, `GetInstanceTpmEkPub`, `GetInstanceTypesFromInstanceRequirements`, `GetInstanceUefiData`, `GetIpamAddressHistory`, `GetIpamDiscoveredAccounts`, `GetIpamDiscoveredPublicAddresses`, `GetIpamDiscoveredResourceCidrs` + +`GetIpamPoolAllocations`, `GetIpamPoolCidrs`, `GetIpamResourceCidrs`, `GetLaunchTemplateData`, `GetManagedPrefixListAssociations`, `GetManagedPrefixListEntries`, `GetNetworkInsightsAccessScopeAnalysisFindings`, `GetNetworkInsightsAccessScopeContent` + +`GetReservedInstancesExchangeQuote`, `GetSecurityGroupsForVpc`, `GetSnapshotBlockPublicAccessState`, `GetSpotPlacementScores`, `GetSubnetCidrReservations`, `GetTransitGatewayAttachmentPropagations`, `GetTransitGatewayMulticastDomainAssociations`, `GetTransitGatewayPolicyTableAssociations` + +`GetTransitGatewayPolicyTableEntries`, `GetTransitGatewayPrefixListReferences`, `GetTransitGatewayRouteTableAssociations`, `GetTransitGatewayRouteTablePropagations`, `GetVerifiedAccessEndpointPolicy`, `GetVerifiedAccessGroupPolicy`, `GetVpnConnectionDeviceSampleConfiguration`, `GetVpnConnectionDeviceTypes` + +`GetVpnTunnelReplacementStatus`, `ImportClientVpnClientCertificateRevocationList`, `ImportImage`, `ImportInstance`, `ImportSnapshot`, `ImportVolume`, `ListImagesInRecycleBin`, `ListSnapshotsInRecycleBin` + +`LockSnapshot`, `ModifyAddressAttribute`, `ModifyAvailabilityZoneGroup`, `ModifyCapacityReservation`, `ModifyCapacityReservationFleet`, `ModifyClientVpnEndpoint`, `ModifyDefaultCreditSpecification`, `ModifyEbsDefaultKmsKeyId` + +`ModifyFleet`, `ModifyFpgaImageAttribute`, `ModifyHosts`, `ModifyIdFormat`, `ModifyIdentityIdFormat`, `ModifyInstanceCapacityReservationAttributes`, `ModifyInstanceCreditSpecification`, `ModifyInstanceEventStartTime` + +`ModifyInstanceEventWindow`, `ModifyInstanceMaintenanceOptions`, `ModifyInstanceMetadataDefaults`, `ModifyInstancePlacement`, `ModifyIpam`, `ModifyIpamPool`, `ModifyIpamResourceCidr`, `ModifyIpamResourceDiscovery` + +`ModifyIpamScope`, `ModifyLocalGatewayRoute`, `ModifyManagedPrefixList`, `ModifyPrivateDnsNameOptions`, `ModifyReservedInstances`, `ModifySecurityGroupRules`, `ModifySnapshotAttribute`, `ModifySnapshotTier` + +`ModifySpotFleetRequest`, `ModifyTrafficMirrorFilterNetworkServices`, `ModifyTrafficMirrorFilterRule`, `ModifyTrafficMirrorSession`, `ModifyTransitGateway`, `ModifyTransitGatewayPrefixListReference`, `ModifyTransitGatewayVpcAttachment`, `ModifyVerifiedAccessEndpoint` + +`ModifyVerifiedAccessEndpointPolicy`, `ModifyVerifiedAccessGroup`, `ModifyVerifiedAccessGroupPolicy`, `ModifyVerifiedAccessInstance`, `ModifyVerifiedAccessInstanceLoggingConfiguration`, `ModifyVerifiedAccessTrustProvider`, `ModifyVolumeAttribute`, `ModifyVpcEndpoint` + +`ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions` + +`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MonitorInstances`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr` + +`ProvisionPublicIpv4PoolCidr`, `PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources` + +`RejectTransitGatewayMulticastDomainAssociations`, `RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation` + +`ReplaceNetworkAclEntry`, `ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute` + +`ResetInstanceAttribute`, `ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier` + +`RevokeClientVpnIngress`, `RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis` + +`StartVpcEndpointServicePrivateDnsVerification`, `TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UnmonitorInstances`, `UpdateSecurityGroupRuleDescriptionsEgress` + +`UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr` + + +
+ +
Registered stubs (0) + +None. + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (0) + +None. + +
+ +## ecr + +Implements **19 of 47** modelled operations (40.4%). + +
Implemented (19) + +`BatchDeleteImage`, `BatchGetImage`, `CreateRepository`, `DeleteLifecyclePolicy`, `DeleteRepository`, `DeleteRepositoryPolicy`, `DescribeImages`, `DescribeRepositories` + +`GetAuthorizationToken`, `GetLifecyclePolicy`, `GetLifecyclePolicyPreview`, `GetRepositoryPolicy`, `ListImages`, `ListTagsForResource`, `PutImage`, `PutImageTagMutability` + +`PutLifecyclePolicy`, `SetRepositoryPolicy`, `StartLifecyclePolicyPreview` + + +
+ +
Missing from dispatch (11) + +`CreatePullThroughCacheRule`, `CreateRepositoryCreationTemplate`, `DeletePullThroughCacheRule`, `DeleteRegistryPolicy`, `DeleteRepositoryCreationTemplate`, `DescribeImageReplicationStatus`, `DescribePullThroughCacheRules`, `DescribeRepositoryCreationTemplates` + +`UpdatePullThroughCacheRule`, `UpdateRepositoryCreationTemplate`, `ValidatePullThroughCacheRule` + + +
+ +
Registered stubs (13) + +`BatchCheckLayerAvailability`, `CompleteLayerUpload`, `DescribeRegistry`, `GetDownloadUrlForLayer`, `GetRegistryPolicy`, `InitiateLayerUpload`, `ListRepositories`, `PutRegistryPolicy` + +`PutReplicationConfiguration`, `ReplicateImage`, `TagResource`, `UntagResource`, `UploadLayerPart` + + +
+ +
Deliberately unsupported (7) + +`BatchGetRepositoryScanningConfiguration`, `DescribeImageScanFindings`, `GetImageScanningConfiguration`, `GetRegistryScanningConfiguration`, `PutImageScanningConfiguration`, `PutRegistryScanningConfiguration`, `StartImageScan` + + +
+ +
Registered outside the pinned model (3) + +`GetImageScanningConfiguration`, `ListRepositories`, `ReplicateImage` + + +
+ +## ecs + +Implements **31 of 56** modelled operations (55.4%). + +
Implemented (31) + +`CreateCapacityProvider`, `CreateCluster`, `CreateService`, `DeleteCapacityProvider`, `DeleteCluster`, `DeleteService`, `DeregisterContainerInstance`, `DeregisterTaskDefinition` + +`DescribeCapacityProviders`, `DescribeClusters`, `DescribeContainerInstances`, `DescribeServices`, `DescribeTaskDefinition`, `DescribeTasks`, `ListClusters`, `ListContainerInstances` + +`ListServices`, `ListTagsForResource`, `ListTaskDefinitions`, `ListTasks`, `PutClusterCapacityProviders`, `RegisterContainerInstance`, `RegisterTaskDefinition`, `RunTask` + +`StartTask`, `StopTask`, `SubmitTaskStateChange`, `TagResource`, `UntagResource`, `UpdateContainerInstancesState`, `UpdateService` + + +
+ +
Missing from dispatch (20) + +`CreateTaskSet`, `DeleteAccountSetting`, `DeleteAttributes`, `DeleteTaskDefinitions`, `DeleteTaskSet`, `DescribeTaskSets`, `DiscoverPollEndpoint`, `ExecuteCommand` + +`GetTaskProtection`, `ListAttributes`, `PutAccountSettingDefault`, `PutAttributes`, `SubmitAttachmentStateChanges`, `SubmitContainerStateChange`, `UpdateCapacityProvider`, `UpdateClusterSettings` + +`UpdateContainerAgent`, `UpdateServicePrimaryTaskSet`, `UpdateTaskProtection`, `UpdateTaskSet` + + +
+ +
Registered stubs (5) + +`ListAccountSettings`, `ListServicesByNamespace`, `ListTaskDefinitionFamilies`, `PutAccountSetting`, `UpdateCluster` + + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (3) + +`PollAssignments`, `ProvisionCapacity`, `ReportTaskGPU` + + +
+ +## elasticloadbalancingv2 + +Implements **33 of 46** modelled operations (71.7%). + +
Implemented (33) + +`AddListenerCertificates`, `AddTags`, `CreateListener`, `CreateLoadBalancer`, `CreateRule`, `CreateTargetGroup`, `DeleteListener`, `DeleteLoadBalancer` + +`DeleteRule`, `DeleteTargetGroup`, `DeregisterTargets`, `DescribeListenerCertificates`, `DescribeListeners`, `DescribeLoadBalancerAttributes`, `DescribeLoadBalancers`, `DescribeRules` + +`DescribeSSLPolicies`, `DescribeTags`, `DescribeTargetGroupAttributes`, `DescribeTargetGroups`, `DescribeTargetHealth`, `ModifyListener`, `ModifyLoadBalancerAttributes`, `ModifyRule` + +`ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `RegisterTargets`, `RemoveListenerCertificates`, `RemoveTags`, `SetIpAddressType`, `SetRulePriorities`, `SetSecurityGroups` + +`SetSubnets` + + +
+ +
Missing from dispatch (13) + +`AddTrustStoreRevocations`, `CreateTrustStore`, `DeleteSharedTrustStoreAssociation`, `DeleteTrustStore`, `DescribeAccountLimits`, `DescribeTrustStoreAssociations`, `DescribeTrustStoreRevocations`, `DescribeTrustStores` + +`GetResourcePolicy`, `GetTrustStoreCaCertificatesBundle`, `GetTrustStoreRevocationContent`, `ModifyTrustStore`, `RemoveTrustStoreRevocations` + + +
+ +
Registered stubs (0) + +None. + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (4) + +`DescribeListenerAttributes`, `GetLBConfig`, `LBAgentHeartbeat`, `ModifyListenerAttributes` + + +
+ +## iam + +Implements **75 of 159** modelled operations (47.2%). + +
Implemented (75) + +`AddRoleToInstanceProfile`, `AddUserToGroup`, `AttachGroupPolicy`, `AttachRolePolicy`, `AttachUserPolicy`, `CreateAccessKey`, `CreateGroup`, `CreateInstanceProfile` + +`CreateOpenIDConnectProvider`, `CreatePolicy`, `CreateRole`, `CreateUser`, `DeleteAccessKey`, `DeleteGroup`, `DeleteGroupPolicy`, `DeleteInstanceProfile` + +`DeleteOpenIDConnectProvider`, `DeletePolicy`, `DeleteRole`, `DeleteRolePolicy`, `DeleteUser`, `DeleteUserPolicy`, `DetachGroupPolicy`, `DetachRolePolicy` + +`DetachUserPolicy`, `GetAccountSummary`, `GetGroup`, `GetGroupPolicy`, `GetInstanceProfile`, `GetOpenIDConnectProvider`, `GetPolicy`, `GetPolicyVersion` + +`GetRole`, `GetRolePolicy`, `GetUser`, `GetUserPolicy`, `ListAccessKeys`, `ListAttachedGroupPolicies`, `ListAttachedRolePolicies`, `ListAttachedUserPolicies` + +`ListGroupPolicies`, `ListGroups`, `ListGroupsForUser`, `ListInstanceProfileTags`, `ListInstanceProfiles`, `ListInstanceProfilesForRole`, `ListOpenIDConnectProviderTags`, `ListOpenIDConnectProviders` + +`ListPolicies`, `ListPolicyTags`, `ListPolicyVersions`, `ListRolePolicies`, `ListRoleTags`, `ListRoles`, `ListUserPolicies`, `ListUserTags` + +`ListUsers`, `PutGroupPolicy`, `PutRolePolicy`, `PutUserPolicy`, `RemoveRoleFromInstanceProfile`, `RemoveUserFromGroup`, `TagInstanceProfile`, `TagOpenIDConnectProvider` + +`TagPolicy`, `TagRole`, `TagUser`, `UntagInstanceProfile`, `UntagOpenIDConnectProvider`, `UntagPolicy`, `UntagRole`, `UntagUser` + +`UpdateAccessKey`, `UpdateAssumeRolePolicy`, `UpdateRole` + + +
+ +
Missing from dispatch (84) + +`AddClientIDToOpenIDConnectProvider`, `ChangePassword`, `CreateAccountAlias`, `CreateLoginProfile`, `CreatePolicyVersion`, `CreateSAMLProvider`, `CreateServiceLinkedRole`, `CreateServiceSpecificCredential` + +`CreateVirtualMFADevice`, `DeactivateMFADevice`, `DeleteAccountAlias`, `DeleteAccountPasswordPolicy`, `DeleteLoginProfile`, `DeletePolicyVersion`, `DeleteRolePermissionsBoundary`, `DeleteSAMLProvider` + +`DeleteSSHPublicKey`, `DeleteServerCertificate`, `DeleteServiceLinkedRole`, `DeleteServiceSpecificCredential`, `DeleteSigningCertificate`, `DeleteUserPermissionsBoundary`, `DeleteVirtualMFADevice`, `EnableMFADevice` + +`GenerateCredentialReport`, `GenerateOrganizationsAccessReport`, `GenerateServiceLastAccessedDetails`, `GetAccessKeyLastUsed`, `GetAccountAuthorizationDetails`, `GetAccountPasswordPolicy`, `GetContextKeysForCustomPolicy`, `GetContextKeysForPrincipalPolicy` + +`GetCredentialReport`, `GetLoginProfile`, `GetMFADevice`, `GetOrganizationsAccessReport`, `GetSAMLProvider`, `GetSSHPublicKey`, `GetServerCertificate`, `GetServiceLastAccessedDetails` + +`GetServiceLastAccessedDetailsWithEntities`, `GetServiceLinkedRoleDeletionStatus`, `ListAccountAliases`, `ListEntitiesForPolicy`, `ListMFADeviceTags`, `ListMFADevices`, `ListPoliciesGrantingServiceAccess`, `ListSAMLProviderTags` + +`ListSAMLProviders`, `ListSSHPublicKeys`, `ListServerCertificateTags`, `ListServerCertificates`, `ListServiceSpecificCredentials`, `ListSigningCertificates`, `ListVirtualMFADevices`, `PutRolePermissionsBoundary` + +`PutUserPermissionsBoundary`, `RemoveClientIDFromOpenIDConnectProvider`, `ResetServiceSpecificCredential`, `ResyncMFADevice`, `SetDefaultPolicyVersion`, `SetSecurityTokenServicePreferences`, `SimulateCustomPolicy`, `SimulatePrincipalPolicy` + +`TagMFADevice`, `TagSAMLProvider`, `TagServerCertificate`, `UntagMFADevice`, `UntagSAMLProvider`, `UntagServerCertificate`, `UpdateAccountPasswordPolicy`, `UpdateGroup` + +`UpdateLoginProfile`, `UpdateOpenIDConnectProviderThumbprint`, `UpdateRoleDescription`, `UpdateSAMLProvider`, `UpdateSSHPublicKey`, `UpdateServerCertificate`, `UpdateServiceSpecificCredential`, `UpdateSigningCertificate` + +`UpdateUser`, `UploadSSHPublicKey`, `UploadServerCertificate`, `UploadSigningCertificate` + + +
+ +
Registered stubs (0) + +None. + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (0) + +None. + +
+ +## s3 + +Operation coverage is not enumerable: Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically. + +## sts + +Implements **4 of 8** modelled operations (50.0%). + +
Implemented (4) + +`AssumeRole`, `AssumeRoleWithWebIdentity`, `GetCallerIdentity`, `GetSessionToken` + + +
+ +
Missing from dispatch (4) + +`AssumeRoleWithSAML`, `DecodeAuthorizationMessage`, `GetAccessKeyInfo`, `GetFederationToken` + + +
+ +
Registered stubs (0) + +None. + +
+ +
Deliberately unsupported (0) + +None. + +
+ +
Registered outside the pinned model (0) + +None. + +
diff --git a/internal/awsmodel/coverage.go b/internal/awsmodel/coverage.go new file mode 100644 index 000000000..e94612af3 --- /dev/null +++ b/internal/awsmodel/coverage.go @@ -0,0 +1,187 @@ +package awsmodel + +import ( + "fmt" + "sort" + "strings" +) + +// DispatchInventory is the gateway-side operation registration state supplied +// to CompareOperations. +type DispatchInventory struct { + Registered []string + Stubbed []string + Unsupported []string + Opaque bool + Note string +} + +// OperationCoverage is a deterministic model-versus-dispatch comparison. +// Implemented contains only modelled operations with a real handler; Extra +// contains registered operations absent from the pinned model. +type OperationCoverage struct { + Service Service + APIVersion string + Modelled []string + Registered []string + Implemented []string + Stubbed []string + Unsupported []string + Missing []string + Extra []string + Opaque bool + Note string +} + +// CompareOperations compares one embedded service model with an authoritative +// gateway dispatch inventory. +func CompareOperations(service Service, dispatch DispatchInventory) (OperationCoverage, error) { + model, err := Load(service) + if err != nil { + return OperationCoverage{}, err + } + coverage := OperationCoverage{ + Service: service, + APIVersion: model.Metadata().APIVersion, + Modelled: model.Operations(), + Opaque: dispatch.Opaque, + Note: dispatch.Note, + } + if dispatch.Opaque { + return coverage, nil + } + + registered, err := uniqueSorted("registered", dispatch.Registered) + if err != nil { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err) + } + stubbed, err := uniqueSorted("stubbed", dispatch.Stubbed) + if err != nil { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err) + } + unsupported, err := uniqueSorted("unsupported", dispatch.Unsupported) + if err != nil { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err) + } + registeredSet := toSet(registered) + stubbedSet := toSet(stubbed) + unsupportedSet := toSet(unsupported) + for operation := range stubbedSet { + if !registeredSet[operation] { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s stubbed operation %q is not registered", service, operation) + } + if unsupportedSet[operation] { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s operation %q is both stubbed and unsupported", service, operation) + } + } + for operation := range unsupportedSet { + if !registeredSet[operation] { + return OperationCoverage{}, fmt.Errorf("awsmodel: %s unsupported operation %q is not registered", service, operation) + } + } + + modelledSet := toSet(coverage.Modelled) + for _, operation := range coverage.Modelled { + if !registeredSet[operation] { + coverage.Missing = append(coverage.Missing, operation) + continue + } + if !stubbedSet[operation] && !unsupportedSet[operation] { + coverage.Implemented = append(coverage.Implemented, operation) + } + } + for _, operation := range registered { + if !modelledSet[operation] { + coverage.Extra = append(coverage.Extra, operation) + } + } + coverage.Registered = registered + coverage.Stubbed = stubbed + coverage.Unsupported = unsupported + return coverage, nil +} + +func uniqueSorted(label string, values []string) ([]string, error) { + result := append([]string(nil), values...) + sort.Strings(result) + for index, value := range result { + if value == "" { + return nil, fmt.Errorf("%s operation is empty", label) + } + if index > 0 && result[index-1] == value { + return nil, fmt.Errorf("%s operation %q is duplicated", label, value) + } + } + return result, nil +} + +func toSet(values []string) map[string]bool { + set := make(map[string]bool, len(values)) + for _, value := range values { + set[value] = true + } + return set +} + +// RenderCoverageMarkdown renders a reproducible divergence report suitable for +// publishing directly or checking as a generated artefact. +func RenderCoverageMarkdown(coverages []OperationCoverage) string { + ordered := append([]OperationCoverage(nil), coverages...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Service < ordered[j].Service }) + + var report strings.Builder + fmt.Fprintf(&report, "# AWS model operation coverage\n\n") + fmt.Fprintf(&report, "Generated from the cached `aws-sdk-go %s` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.\n\n", SourceSDKVersion) + report.WriteString("Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.\n\n") + report.WriteString("| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |\n") + report.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|\n") + for _, coverage := range ordered { + if coverage.Opaque { + fmt.Fprintf(&report, "| %s | %s | %d | — | — | — | — | — | — |\n", + coverage.Service, coverage.APIVersion, len(coverage.Modelled)) + continue + } + fmt.Fprintf(&report, "| %s | %s | %d | %d | %d | %d | %d | %d | %d |\n", + coverage.Service, coverage.APIVersion, len(coverage.Modelled), len(coverage.Registered), + len(coverage.Implemented), len(coverage.Stubbed), len(coverage.Unsupported), len(coverage.Missing), len(coverage.Extra)) + } + + for _, coverage := range ordered { + fmt.Fprintf(&report, "\n## %s\n\n", coverage.Service) + if coverage.Opaque { + fmt.Fprintf(&report, "Operation coverage is not enumerable: %s\n", coverage.Note) + continue + } + percentage := float64(0) + if len(coverage.Modelled) != 0 { + percentage = 100 * float64(len(coverage.Implemented)) / float64(len(coverage.Modelled)) + } + fmt.Fprintf(&report, "Implements **%d of %d** modelled operations (%.1f%%).\n", len(coverage.Implemented), len(coverage.Modelled), percentage) + writeOperationList(&report, "Implemented", coverage.Implemented) + writeOperationList(&report, "Missing from dispatch", coverage.Missing) + writeOperationList(&report, "Registered stubs", coverage.Stubbed) + writeOperationList(&report, "Deliberately unsupported", coverage.Unsupported) + writeOperationList(&report, "Registered outside the pinned model", coverage.Extra) + if coverage.Note != "" { + fmt.Fprintf(&report, "\nNote: %s\n", coverage.Note) + } + } + return report.String() +} + +func writeOperationList(report *strings.Builder, title string, operations []string) { + fmt.Fprintf(report, "\n
%s (%d)\n\n", title, len(operations)) + if len(operations) == 0 { + report.WriteString("None.\n") + } else { + for start := 0; start < len(operations); start += 8 { + end := min(start+8, len(operations)) + quoted := make([]string, 0, end-start) + for _, operation := range operations[start:end] { + quoted = append(quoted, "`"+operation+"`") + } + report.WriteString(strings.Join(quoted, ", ") + "\n\n") + } + } + report.WriteString("\n
\n") +} diff --git a/internal/awsmodel/coverage_test.go b/internal/awsmodel/coverage_test.go new file mode 100644 index 000000000..79c103419 --- /dev/null +++ b/internal/awsmodel/coverage_test.go @@ -0,0 +1,51 @@ +package awsmodel + +import ( + "strings" + "testing" +) + +func TestCompareOperations(t *testing.T) { + coverage, err := CompareOperations(STS, DispatchInventory{ + Registered: []string{"AssumeRole", "ExtraOperation", "GetCallerIdentity", "GetSessionToken"}, + Stubbed: []string{"GetSessionToken"}, + Unsupported: []string{"GetCallerIdentity"}, + }) + if err != nil { + t.Fatal(err) + } + if len(coverage.Implemented) != 1 || coverage.Implemented[0] != "AssumeRole" { + t.Fatalf("implemented = %v, want [AssumeRole]", coverage.Implemented) + } + if len(coverage.Missing) != 5 { + t.Fatalf("missing count = %d, want 5: %v", len(coverage.Missing), coverage.Missing) + } + if len(coverage.Extra) != 1 || coverage.Extra[0] != "ExtraOperation" { + t.Fatalf("extra = %v, want [ExtraOperation]", coverage.Extra) + } +} + +func TestCompareOperationsRejectsInvalidInventory(t *testing.T) { + _, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}, Stubbed: []string{"Missing"}}) + if err == nil || !strings.Contains(err.Error(), `stubbed operation "Missing" is not registered`) { + t.Fatalf("error = %v", err) + } +} + +func TestRenderCoverageMarkdown(t *testing.T) { + coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}}) + if err != nil { + t.Fatal(err) + } + report := RenderCoverageMarkdown([]OperationCoverage{coverage}) + for _, want := range []string{ + "# AWS model operation coverage", + "| sts | 2011-06-15 | 8 | 1 | 1 | 0 | 0 | 7 | 0 |", + "Implements **1 of 8** modelled operations (12.5%).", + "Missing from dispatch (7)", + } { + if !strings.Contains(report, want) { + t.Errorf("report does not contain %q:\n%s", want, report) + } + } +} diff --git a/internal/awsmodel/ec2error.go b/internal/awsmodel/ec2error.go new file mode 100644 index 000000000..f0e13da2d --- /dev/null +++ b/internal/awsmodel/ec2error.go @@ -0,0 +1,153 @@ +package awsmodel + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strings" + "sync" +) + +const ec2ErrorCatalogPath = "models/ec2/error-codes.json" + +type ec2ErrorClass string + +const ( + ec2ErrorClassClient ec2ErrorClass = "client" + ec2ErrorClassServer ec2ErrorClass = "server" +) + +type ec2ErrorCatalogFile struct { + Source string `json:"source"` + VerifiedOn string `json:"verifiedOn"` + CommonClient []string `json:"commonClient"` + SpecificClient []string `json:"specificClient"` + Server []string `json:"server"` +} + +type ec2ErrorCatalog struct { + metadata ec2ErrorCatalogFile + codes map[string]ec2ErrorClass +} + +var loadEC2ErrorCatalog = sync.OnceValues(func() (*ec2ErrorCatalog, error) { + contents, err := modelFS.ReadFile(ec2ErrorCatalogPath) + if err != nil { + return nil, fmt.Errorf("awsmodel: read EC2 error catalog: %w", err) + } + + decoder := json.NewDecoder(bytes.NewReader(contents)) + decoder.DisallowUnknownFields() + var file ec2ErrorCatalogFile + if err := decoder.Decode(&file); err != nil { + return nil, fmt.Errorf("awsmodel: parse EC2 error catalog: %w", err) + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return nil, fmt.Errorf("awsmodel: parse EC2 error catalog: %w", err) + } + if file.Source == "" || file.VerifiedOn == "" { + return nil, errors.New("awsmodel: EC2 error catalog requires source and verifiedOn") + } + + catalog := &ec2ErrorCatalog{metadata: file, codes: make(map[string]ec2ErrorClass)} + add := func(label string, values []string, class ec2ErrorClass) error { + if !slices.IsSorted(values) { + return fmt.Errorf("awsmodel: EC2 error catalog %s codes are not sorted", label) + } + for _, code := range values { + if code == "" { + return fmt.Errorf("awsmodel: EC2 error catalog %s contains an empty code", label) + } + if previous, exists := catalog.codes[code]; exists { + return fmt.Errorf("awsmodel: EC2 error catalog code %q is duplicated (%s and %s)", code, previous, label) + } + catalog.codes[code] = class + } + return nil + } + if err := add("commonClient", file.CommonClient, ec2ErrorClassClient); err != nil { + return nil, err + } + if err := add("specificClient", file.SpecificClient, ec2ErrorClassClient); err != nil { + return nil, err + } + if err := add("server", file.Server, ec2ErrorClassServer); err != nil { + return nil, err + } + return catalog, nil +}) + +// ValidateEC2ErrorResponse validates an EC2 Query API error envelope against +// the checked-in catalog curated from AWS's EC2 error-code reference. The AWS +// api-2.json model declares no operation errors, so this is intentionally a +// separate oracle. +func ValidateEC2ErrorResponse(status int, body []byte) ([]Violation, error) { + root, err := parseXML(body) + if err != nil { + return nil, fmt.Errorf("awsmodel: decode EC2 error response: %w", err) + } + if root.name != "Response" { + return nil, fmt.Errorf("awsmodel: decode EC2 error response: root element is %q, want %q", root.name, "Response") + } + errorsNode := directXMLChild(root, "Errors") + if errorsNode == nil { + return nil, errors.New("awsmodel: decode EC2 error response: Errors element is missing") + } + errorNode := directXMLChild(errorsNode, "Error") + if errorNode == nil { + return nil, errors.New("awsmodel: decode EC2 error response: Error element is missing") + } + code := directXMLText(errorNode, "Code") + if code == "" { + return nil, errors.New("awsmodel: decode EC2 error response: Code element is missing") + } + + catalog, err := loadEC2ErrorCatalog() + if err != nil { + return nil, err + } + class, known := catalog.codes[code] + if !known { + return []Violation{{ + Rule: RuleErrorCode, + Path: "$error.Code", + Message: fmt.Sprintf("error code %q is not in the curated EC2 catalog", code), + }}, nil + } + + wantStatusClass := 4 + if class == ec2ErrorClassServer { + wantStatusClass = 5 + } + if status/100 != wantStatusClass { + return []Violation{{ + Rule: RuleHTTPStatus, + Path: "$response.status", + Message: fmt.Sprintf("status %d is not in the %dxx class required for EC2 %s error %q", status, wantStatusClass, class, code), + }}, nil + } + return nil, nil +} + +func directXMLChild(node *xmlNode, name string) *xmlNode { + for _, child := range node.children { + if child.name == name { + return child + } + } + return nil +} + +func directXMLText(node *xmlNode, name string) string { + child := directXMLChild(node, name) + if child == nil { + return "" + } + return strings.TrimSpace(child.text.String()) +} diff --git a/internal/awsmodel/ec2error_test.go b/internal/awsmodel/ec2error_test.go new file mode 100644 index 000000000..01bbfbbda --- /dev/null +++ b/internal/awsmodel/ec2error_test.go @@ -0,0 +1,93 @@ +package awsmodel + +import ( + "reflect" + "strings" + "testing" +) + +func TestValidateEC2ErrorResponseAllowsCuratedCodes(t *testing.T) { + tests := []struct { + name string + status int + code string + }{ + {name: "common client", status: 403, code: "UnauthorizedOperation"}, + {name: "specific client", status: 404, code: "InvalidInstanceID.NotFound"}, + {name: "server", status: 500, code: "InternalError"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + violations, err := ValidateEC2ErrorResponse(test.status, ec2ErrorBody(test.code)) + if err != nil { + t.Fatal(err) + } + if len(violations) != 0 { + t.Fatalf("violations = %#v, want none", violations) + } + }) + } +} + +func TestValidateEC2ErrorResponseRejectsUnverifiedCode(t *testing.T) { + violations, err := ValidateEC2ErrorResponse(403, ec2ErrorBody("ExpiredToken")) + if err != nil { + t.Fatal(err) + } + want := []Violation{{ + Rule: RuleErrorCode, + Path: "$error.Code", + Message: `error code "ExpiredToken" is not in the curated EC2 catalog`, + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateEC2ErrorResponseRejectsWrongStatusClass(t *testing.T) { + violations, err := ValidateEC2ErrorResponse(400, ec2ErrorBody("InsufficientInstanceCapacity")) + if err != nil { + t.Fatal(err) + } + want := []Violation{{ + Rule: RuleHTTPStatus, + Path: "$response.status", + Message: `status 400 is not in the 5xx class required for EC2 server error "InsufficientInstanceCapacity"`, + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateEC2ErrorResponseRejectsMalformedEnvelope(t *testing.T) { + _, err := ValidateEC2ErrorResponse(400, []byte(`InvalidAction`)) + if err == nil || !strings.Contains(err.Error(), `root element is "ErrorResponse", want "Response"`) { + t.Fatalf("error = %v", err) + } +} + +func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) { + catalog, err := loadEC2ErrorCatalog() + if err != nil { + t.Fatal(err) + } + if catalog.metadata.Source != "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html" { + t.Fatalf("source = %q", catalog.metadata.Source) + } + if catalog.metadata.VerifiedOn == "" { + t.Fatal("verifiedOn is empty") + } + if len(catalog.codes) != 112 { + t.Fatalf("catalog code count = %d, want 112", len(catalog.codes)) + } + if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient { + t.Fatal("InvalidParameterValue is not classified as client") + } + if catalog.codes["ServerInternal"] != ec2ErrorClassServer { + t.Fatal("ServerInternal is not classified as server") + } +} + +func ec2ErrorBody(code string) []byte { + return []byte(`` + code + `testrequest-1`) +} diff --git a/internal/awsmodel/embed.go b/internal/awsmodel/embed.go new file mode 100644 index 000000000..0d21ef315 --- /dev/null +++ b/internal/awsmodel/embed.go @@ -0,0 +1,21 @@ +package awsmodel + +import "embed" + +// modelFS contains only the curated EC2 error catalog. The large upstream +// api-2.json files are resolved from the pinned aws-sdk-go module cache at +// runtime; they are intentionally not source-controlled here. +// +//go:embed models/ec2/error-codes.json +var modelFS embed.FS + +var modelFiles = map[Service]string{ + ACM: "models/apis/acm/2015-12-08/api-2.json", + EC2: "models/apis/ec2/2016-11-15/api-2.json", + ECR: "models/apis/ecr/2015-09-21/api-2.json", + ECS: "models/apis/ecs/2014-11-13/api-2.json", + ElasticLoadBalancingV2: "models/apis/elasticloadbalancingv2/2015-12-01/api-2.json", + IAM: "models/apis/iam/2010-05-08/api-2.json", + S3: "models/apis/s3/2006-03-01/api-2.json", + STS: "models/apis/sts/2011-06-15/api-2.json", +} diff --git a/internal/awsmodel/error.go b/internal/awsmodel/error.go new file mode 100644 index 000000000..041264d26 --- /dev/null +++ b/internal/awsmodel/error.go @@ -0,0 +1,142 @@ +package awsmodel + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" +) + +// ValidateErrorResponse decodes an AWS error envelope and checks its code +// against the errors declared for operation. Modelled is false when AWS's +// operation definition declares no errors, as is the case for some STS calls. +func ValidateErrorResponse(service Service, operationName string, body []byte) (violations []Violation, modelled bool, err error) { + model, err := Load(service) + if err != nil { + return nil, false, err + } + operation, ok := model.Operation(operationName) + if !ok { + return nil, false, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName) + } + + code, err := model.decodeErrorCode(body) + if err != nil { + return nil, false, fmt.Errorf("awsmodel: decode %s %s error response: %w", service, operationName, err) + } + if isUnmodelledCommonError(service, code) { + return nil, false, nil + } + codes := model.operationErrorCodes(operation) + if len(codes) == 0 { + return nil, false, nil + } + if slices.Contains(codes, code) { + return nil, true, nil + } + return []Violation{{ + Rule: RuleErrorCode, + Path: "$error.Code", + Message: fmt.Sprintf("error code %q is not declared; allowed [%s]", code, strings.Join(codes, ", ")), + }}, true, nil +} + +// isUnmodelledCommonError identifies service/protocol errors AWS does not put +// in operation error lists. They remain counted as unmodelled by the harness; +// they are not treated as proof that an operation-specific code conforms. +func isUnmodelledCommonError(service Service, code string) bool { + switch service { + case IAM, ElasticLoadBalancingV2: + return isUnmodelledQueryError(code) + case STS: + // STS also uses these two runtime input errors even though api-2.json + // does not declare them on AssumeRole/GetSessionToken. + return isUnmodelledQueryError(code) || code == "ValidationError" || code == "InvalidParameterValue" + case ECS: + switch code { + case "AccessDeniedException", "IncompleteSignatureException", "InvalidClientTokenIdException", + "MissingAuthenticationTokenException", "RequestExpiredException", "SignatureDoesNotMatchException", + "ThrottlingException": + return true + } + default: + } + return false +} + +func isUnmodelledQueryError(code string) bool { + switch code { + case "AccessDenied", "IncompleteSignature", "InvalidClientTokenId", "MissingAuthenticationToken", + "RequestExpired", "SignatureDoesNotMatch", "Throttling": + return true + default: + return false + } +} + +func (m *Model) operationErrorCodes(operation *Operation) []string { + codes := make([]string, 0, len(operation.Errors)) + for _, ref := range operation.Errors { + code := ref.Shape + if shape := m.shapes[ref.Shape]; shape != nil && shape.Error != nil && shape.Error.Code != "" { + code = shape.Error.Code + } + codes = append(codes, code) + } + sort.Strings(codes) + return codes +} + +func (m *Model) decodeErrorCode(body []byte) (string, error) { + switch m.metadata.Protocol { + case "json", "rest-json": + var envelope struct { + Type string `json:"__type"` + Code string `json:"code"` + } + decoder := json.NewDecoder(bytes.NewReader(body)) + if err := decoder.Decode(&envelope); err != nil { + return "", err + } + code := envelope.Type + if code == "" { + code = envelope.Code + } + if separator := strings.LastIndex(code, "#"); separator >= 0 { + code = code[separator+1:] + } + if separator := strings.Index(code, ":"); separator >= 0 { + code = code[:separator] + } + if code == "" { + return "", fmt.Errorf("JSON error envelope has no code") + } + return code, nil + case "ec2", "query": + root, err := parseXML(body) + if err != nil { + return "", err + } + code := findXMLText(root, "Code") + if code == "" { + return "", fmt.Errorf("XML error envelope has no Code element") + } + return code, nil + default: + return "", fmt.Errorf("error decoding for protocol %q is not implemented", m.metadata.Protocol) + } +} + +func findXMLText(node *xmlNode, name string) string { + if node.name == name { + return strings.TrimSpace(node.text.String()) + } + for _, child := range node.children { + if value := findXMLText(child, name); value != "" { + return value + } + } + return "" +} diff --git a/internal/awsmodel/error_test.go b/internal/awsmodel/error_test.go new file mode 100644 index 000000000..abf47ca12 --- /dev/null +++ b/internal/awsmodel/error_test.go @@ -0,0 +1,119 @@ +package awsmodel + +import ( + "reflect" + "testing" +) + +func TestValidateErrorResponseQueryAllowsDeclaredCode(t *testing.T) { + body := []byte(`NoSuchEntitymissing`) + violations, modelled, err := ValidateErrorResponse(IAM, "GetUser", body) + if err != nil { + t.Fatal(err) + } + if !modelled { + t.Fatal("GetUser errors should be modelled") + } + if len(violations) != 0 { + t.Fatalf("violations = %#v, want none", violations) + } +} + +func TestValidateErrorResponseRejectsUndeclaredCode(t *testing.T) { + body := []byte(`InventedFailure`) + violations, modelled, err := ValidateErrorResponse(IAM, "GetUser", body) + if err != nil { + t.Fatal(err) + } + if !modelled { + t.Fatal("GetUser errors should be modelled") + } + want := []Violation{{ + Rule: RuleErrorCode, + Path: "$error.Code", + Message: `error code "InventedFailure" is not declared; allowed [NoSuchEntity, ServiceFailure]`, + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateErrorResponseJSONAllowsDeclaredShapeName(t *testing.T) { + body := []byte(`{"__type":"com.amazonaws.ecs#ClusterNotFoundException","message":"missing"}`) + violations, modelled, err := ValidateErrorResponse(ECS, "DescribeTasks", body) + if err != nil { + t.Fatal(err) + } + if !modelled || len(violations) != 0 { + t.Fatalf("modelled = %t, violations = %#v, want modelled and conforming", modelled, violations) + } +} + +func TestValidateErrorResponseELBAllowsDeclaredCode(t *testing.T) { + body := []byte(`LoadBalancerNotFound`) + violations, modelled, err := ValidateErrorResponse(ElasticLoadBalancingV2, "DescribeLoadBalancers", body) + if err != nil { + t.Fatal(err) + } + if !modelled || len(violations) != 0 { + t.Fatalf("modelled = %t, violations = %#v, want modelled and conforming", modelled, violations) + } +} + +func TestValidateErrorResponseRejectsMalformedEnvelope(t *testing.T) { + _, _, err := ValidateErrorResponse(ECS, "DescribeTasks", []byte(`{"message":"missing code"}`)) + if err == nil { + t.Fatal("expected missing error code to fail decoding") + } +} + +func TestValidateErrorResponseOperationWithoutDeclaredErrors(t *testing.T) { + body := []byte(`AccessDenied`) + violations, modelled, err := ValidateErrorResponse(STS, "GetCallerIdentity", body) + if err != nil { + t.Fatal(err) + } + if modelled || len(violations) != 0 { + t.Fatalf("modelled = %t, violations = %#v, want unmodelled and no violations", modelled, violations) + } +} + +func TestValidateErrorResponseClassifiesCommonErrorsAsUnmodelled(t *testing.T) { + tests := []struct { + name string + service Service + operation string + body string + }{ + { + name: "query authorization", + service: IAM, + operation: "CreateUser", + body: `AccessDenied`, + }, + { + name: "STS runtime validation", + service: STS, + operation: "AssumeRole", + body: `ValidationError`, + }, + { + name: "JSON authorization", + service: ECS, + operation: "DescribeTasks", + body: `{"__type":"AccessDeniedException"}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + violations, modelled, err := ValidateErrorResponse(test.service, test.operation, []byte(test.body)) + if err != nil { + t.Fatal(err) + } + if modelled || len(violations) != 0 { + t.Fatalf("modelled = %t, violations = %#v, want unmodelled and no violations", modelled, violations) + } + }) + } +} diff --git a/internal/awsmodel/model.go b/internal/awsmodel/model.go new file mode 100644 index 000000000..01ba0e1cf --- /dev/null +++ b/internal/awsmodel/model.go @@ -0,0 +1,296 @@ +// Package awsmodel loads the AWS api-2.json service definitions used by the +// conformance suite. +package awsmodel + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "os/user" + "path/filepath" + "slices" + "sync" +) + +// SourceSDKVersion is the aws-sdk-go release whose module-cache models are +// used by the conformance suite. +const SourceSDKVersion = "v1.55.8" + +const sourceSDKModule = "github.com/aws/aws-sdk-go" + +// Service identifies an AWS service model loaded from the SDK module cache. +type Service string + +const ( + ACM Service = "acm" + EC2 Service = "ec2" + ECR Service = "ecr" + ECS Service = "ecs" + ElasticLoadBalancingV2 Service = "elasticloadbalancingv2" + IAM Service = "iam" + S3 Service = "s3" + STS Service = "sts" +) + +// Metadata describes the service and wire protocol represented by a model. +type Metadata struct { + APIVersion string `json:"apiVersion"` + EndpointPrefix string `json:"endpointPrefix"` + Protocol string `json:"protocol"` + Protocols []string `json:"protocols"` + ServiceAbbreviation string `json:"serviceAbbreviation"` + ServiceFullName string `json:"serviceFullName"` + ServiceID string `json:"serviceId"` + SignatureVersion string `json:"signatureVersion"` + UID string `json:"uid"` +} + +// Operation describes an AWS API operation and the shapes used by its input, +// output and declared errors. +type Operation struct { + Name string `json:"name"` + HTTP HTTP `json:"http"` + Input *ShapeRef `json:"input"` + Output *ShapeRef `json:"output"` + Errors []ShapeRef `json:"errors"` +} + +// HTTP describes an operation's HTTP binding. +type HTTP struct { + Method string `json:"method"` + RequestURI string `json:"requestUri"` + ResponseCode int `json:"responseCode"` +} + +// Shape describes a value in an AWS service model. Depending on Type, it can +// refer to structure members, a list member, or map keys and values. +type Shape struct { + Type string `json:"type"` + Required []string `json:"required"` + Members map[string]ShapeRef `json:"members"` + Member *ShapeRef `json:"member"` + Key *ShapeRef `json:"key"` + Value *ShapeRef `json:"value"` + Enum []string `json:"enum"` + Min *float64 `json:"min"` + Max *float64 `json:"max"` + Pattern string `json:"pattern"` + LocationName string `json:"locationName"` + TimestampFormat string `json:"timestampFormat"` + Payload string `json:"payload"` + Flattened bool `json:"flattened"` + Sensitive bool `json:"sensitive"` + Exception bool `json:"exception"` + Fault bool `json:"fault"` + Error *ErrorInfo `json:"error"` +} + +// ErrorInfo describes an error shape's code and HTTP classification. +type ErrorInfo struct { + Code string `json:"code"` + HTTPStatusCode int `json:"httpStatusCode"` + SenderFault bool `json:"senderFault"` +} + +// ShapeRef names another shape and carries any wire binding specific to the +// place where it is referenced. +type ShapeRef struct { + Shape string `json:"shape"` + ResultWrapper string `json:"resultWrapper"` + Location string `json:"location"` + LocationName string `json:"locationName"` + QueryName string `json:"queryName"` + TimestampFormat string `json:"timestampFormat"` + Flattened bool `json:"flattened"` + Streaming bool `json:"streaming"` + XMLAttribute bool `json:"xmlAttribute"` +} + +// Model is an indexed AWS service definition. +type Model struct { + service Service + metadata Metadata + operations map[string]*Operation + shapes map[string]*Shape +} + +type modelDocument struct { + Version string `json:"version"` + Metadata Metadata `json:"metadata"` + Operations map[string]*Operation `json:"operations"` + Shapes map[string]*Shape `json:"shapes"` +} + +type loadResult struct { + model *Model + err error +} + +var modelCache sync.Map + +var modelRoot = sync.OnceValues(resolveModelRoot) + +func resolveModelRoot() (string, error) { + if root := os.Getenv("GOMODCACHE"); root != "" { + return filepath.Join(root, sourceSDKModule+"@"+SourceSDKVersion), nil + } + if output, err := exec.Command("go", "env", "GOMODCACHE").Output(); err == nil { + if root := string(output); root != "" { + return filepath.Join(stringTrimSpace(root), sourceSDKModule+"@"+SourceSDKVersion), nil + } + } + currentUser, err := user.Current() + if err != nil { + return "", fmt.Errorf("awsmodel: resolve Go module cache: %w", err) + } + return filepath.Join(currentUser.HomeDir, "go", "pkg", "mod", sourceSDKModule+"@"+SourceSDKVersion), nil +} + +func stringTrimSpace(value string) string { + for len(value) > 0 && (value[len(value)-1] == '\n' || value[len(value)-1] == '\r' || value[len(value)-1] == ' ' || value[len(value)-1] == '\t') { + value = value[:len(value)-1] + } + return value +} + +// Services returns the supported service identifiers in stable order. +func Services() []Service { + services := make([]Service, 0, len(modelFiles)) + for service := range modelFiles { + services = append(services, service) + } + slices.Sort(services) + return services +} + +// Load parses and indexes the cached SDK model for service. Each service is +// parsed at most once and the resulting Model is safe for concurrent reads. +func Load(service Service) (*Model, error) { + path, ok := modelFiles[service] + if !ok { + return nil, fmt.Errorf("awsmodel: unsupported service %q", service) + } + + loader, _ := modelCache.LoadOrStore(service, sync.OnceValue(func() loadResult { + root, err := modelRoot() + if err != nil { + return loadResult{err: err} + } + contents, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + return loadResult{err: fmt.Errorf("awsmodel: read cached %s model at %s: %w", service, filepath.Join(root, path), err)} + } + + var document modelDocument + if err := json.Unmarshal(contents, &document); err != nil { + return loadResult{err: fmt.Errorf("awsmodel: parse %s model: %w", service, err)} + } + if document.Version != "2.0" { + return loadResult{err: fmt.Errorf("awsmodel: %s model version is %q, want %q", service, document.Version, "2.0")} + } + if len(document.Operations) == 0 || len(document.Shapes) == 0 { + return loadResult{err: fmt.Errorf("awsmodel: %s model has no operations or shapes", service)} + } + + model := &Model{ + service: service, + metadata: document.Metadata, + operations: document.Operations, + shapes: document.Shapes, + } + if err := model.validateReferences(); err != nil { + return loadResult{err: err} + } + return loadResult{model: model} + })) + load, ok := loader.(func() loadResult) + if !ok { + return nil, fmt.Errorf("awsmodel: invalid cache entry for service %q", service) + } + result := load() + return result.model, result.err +} + +// Service returns the identifier used to load the model. +func (m *Model) Service() Service { return m.service } + +// Metadata returns the service metadata from api-2.json. +func (m *Model) Metadata() Metadata { return m.metadata } + +// Operation resolves an operation by its model key, such as DescribeInstances. +func (m *Model) Operation(name string) (*Operation, bool) { + operation, ok := m.operations[name] + return operation, ok +} + +// Shape resolves a shape by name. +func (m *Model) Shape(name string) (*Shape, bool) { + shape, ok := m.shapes[name] + return shape, ok +} + +// Operations returns all operation keys in stable order. +func (m *Model) Operations() []string { return sortedKeys(m.operations) } + +// Shapes returns all shape names in stable order. +func (m *Model) Shapes() []string { return sortedKeys(m.shapes) } + +func sortedKeys[T any](values map[string]T) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func (m *Model) validateReferences() error { + validate := func(owner string, ref *ShapeRef) error { + if ref == nil { + return nil + } + if _, ok := m.shapes[ref.Shape]; !ok { + return fmt.Errorf("awsmodel: %s model %s references unknown shape %q", m.service, owner, ref.Shape) + } + return nil + } + + for name, operation := range m.operations { + if operation == nil { + return fmt.Errorf("awsmodel: %s model operation %q is null", m.service, name) + } + if err := validate("operation "+name+" input", operation.Input); err != nil { + return err + } + if err := validate("operation "+name+" output", operation.Output); err != nil { + return err + } + for i := range operation.Errors { + if err := validate(fmt.Sprintf("operation %s error %d", name, i), &operation.Errors[i]); err != nil { + return err + } + } + } + + for name, shape := range m.shapes { + if shape == nil { + return fmt.Errorf("awsmodel: %s model shape %q is null", m.service, name) + } + for memberName, member := range shape.Members { + if err := validate("shape "+name+" member "+memberName, &member); err != nil { + return err + } + } + for label, ref := range map[string]*ShapeRef{ + "member": shape.Member, + "key": shape.Key, + "value": shape.Value, + } { + if err := validate("shape "+name+" "+label, ref); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/awsmodel/model_test.go b/internal/awsmodel/model_test.go new file mode 100644 index 000000000..f3580c2cd --- /dev/null +++ b/internal/awsmodel/model_test.go @@ -0,0 +1,138 @@ +package awsmodel + +import ( + "reflect" + "strings" + "sync" + "testing" +) + +func TestServices(t *testing.T) { + want := []Service{ACM, EC2, ECR, ECS, ElasticLoadBalancingV2, IAM, S3, STS} + if got := Services(); !reflect.DeepEqual(got, want) { + t.Fatalf("Services() = %v, want %v", got, want) + } +} + +func TestLoadAllModels(t *testing.T) { + wantSizes := map[Service]struct { + apiVersion string + operations int + shapes int + }{ + ACM: {"2015-12-08", 15, 102}, + EC2: {"2016-11-15", 625, 3198}, + ECR: {"2015-09-21", 47, 330}, + ECS: {"2014-11-13", 56, 400}, + ElasticLoadBalancingV2: {"2015-12-01", 46, 357}, + IAM: {"2010-05-08", 159, 497}, + S3: {"2006-03-01", 99, 604}, + STS: {"2011-06-15", 8, 76}, + } + + for service, want := range wantSizes { + t.Run(string(service), func(t *testing.T) { + model, err := Load(service) + if err != nil { + t.Fatalf("Load(%q): %v", service, err) + } + if model.Service() != service { + t.Errorf("Service() = %q, want %q", model.Service(), service) + } + if model.Metadata().APIVersion != want.apiVersion || model.Metadata().Protocol == "" { + t.Errorf("metadata is incomplete: %+v", model.Metadata()) + } + if got := len(model.Operations()); got != want.operations { + t.Errorf("operation count = %d, want %d", got, want.operations) + } + if got := len(model.Shapes()); got != want.shapes { + t.Errorf("shape count = %d, want %d", got, want.shapes) + } + }) + } +} + +func TestLoadEC2IndexesOperationsAndShapes(t *testing.T) { + model, err := Load(EC2) + if err != nil { + t.Fatal(err) + } + + operation, ok := model.Operation("DescribeInstances") + if !ok { + t.Fatal("DescribeInstances operation was not indexed") + } + if operation.Output == nil || operation.Output.Shape != "DescribeInstancesResult" { + t.Fatalf("DescribeInstances output = %+v, want DescribeInstancesResult", operation.Output) + } + + shape, ok := model.Shape(operation.Output.Shape) + if !ok { + t.Fatalf("output shape %q was not indexed", operation.Output.Shape) + } + if reservationSet, ok := shape.Members["Reservations"]; !ok || reservationSet.Shape != "ReservationList" { + t.Errorf("Reservations member = %+v, want ReservationList", reservationSet) + } +} + +func TestLoadPreservesWrappersAndErrorMetadata(t *testing.T) { + model, err := Load(IAM) + if err != nil { + t.Fatal(err) + } + + getUser, ok := model.Operation("GetUser") + if !ok || getUser.Output == nil || getUser.Output.ResultWrapper != "GetUserResult" { + t.Fatalf("GetUser output = %+v, want GetUserResult wrapper", getUser) + } + + createUser, ok := model.Operation("CreateUser") + if !ok || len(createUser.Errors) == 0 { + t.Fatalf("CreateUser operation has no modelled errors: %+v", createUser) + } + errorShape, ok := model.Shape(createUser.Errors[0].Shape) + if !ok || errorShape.Error == nil || errorShape.Error.Code == "" || errorShape.Error.HTTPStatusCode == 0 { + t.Fatalf("CreateUser error metadata was not loaded: %+v", errorShape) + } +} + +func TestLoadRejectsUnknownService(t *testing.T) { + _, err := Load("not-a-service") + if err == nil || !strings.Contains(err.Error(), `unsupported service "not-a-service"`) { + t.Fatalf("Load(unknown) error = %v", err) + } +} + +func TestLoadCachesModelAcrossConcurrentCalls(t *testing.T) { + const callers = 16 + models := make(chan *Model, callers) + errors := make(chan error, callers) + + var wait sync.WaitGroup + for range callers { + wait.Go(func() { + model, err := Load(STS) + models <- model + errors <- err + }) + } + wait.Wait() + close(models) + close(errors) + + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + var first *Model + for model := range models { + if first == nil { + first = model + continue + } + if model != first { + t.Fatal("Load returned different cached model pointers") + } + } +} diff --git a/internal/awsmodel/models/README.md b/internal/awsmodel/models/README.md new file mode 100644 index 000000000..dd421f334 --- /dev/null +++ b/internal/awsmodel/models/README.md @@ -0,0 +1,13 @@ +# AWS model cache + +The conformance suite reads `api-2.json` from the Go module cache for the +pinned `github.com/aws/aws-sdk-go` version `v1.55.8`. Running Go tests or +`go mod download` populates that cache; the large upstream model files are +intentionally not tracked in this repository. + +`ec2/error-codes.json` is separate from the SDK models because EC2's +`api-2.json` declares no operation errors. It is a curated subset of the +official [EC2 error-code reference](https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html): +all documented common and server codes, plus the action-specific codes that +Spinifex currently emits. The catalog records its verification date and must be +reviewed against that source when it changes. diff --git a/internal/awsmodel/models/ec2/error-codes.json b/internal/awsmodel/models/ec2/error-codes.json new file mode 100644 index 000000000..d3e4d4778 --- /dev/null +++ b/internal/awsmodel/models/ec2/error-codes.json @@ -0,0 +1,122 @@ +{ + "source": "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html", + "verifiedOn": "2026-08-05", + "commonClient": [ + "AuthFailure", + "Blocked", + "DryRunOperation", + "IdempotentParameterMismatch", + "IncompleteSignature", + "InvalidAction", + "InvalidCharacter", + "InvalidClientTokenId", + "InvalidPaginationToken", + "InvalidParameter", + "InvalidParameterCombination", + "InvalidParameterDependency", + "InvalidParameterValue", + "InvalidQueryParameter", + "MalformedQueryString", + "MissingAction", + "MissingAuthenticationToken", + "MissingParameter", + "OptInRequired", + "PendingVerification", + "RequestExpired", + "TagPolicyViolation", + "UnauthorizedOperation", + "UnknownParameter", + "UnsupportedInstanceAttribute", + "UnsupportedOperation", + "UnsupportedProtocol", + "ValidationError" + ], + "specificClient": [ + "CannotDelete", + "DependencyViolation", + "Gateway.NotAttached", + "IncorrectInstanceState", + "IncorrectState", + "InvalidAMIID.Malformed", + "InvalidAMIID.NotFound", + "InvalidAMIID.Unavailable", + "InvalidAMIName.Duplicate", + "InvalidAMIName.Malformed", + "InvalidAddress.Locked", + "InvalidAllocationID.NotFound", + "InvalidAssociationID.NotFound", + "InvalidAttachmentID.NotFound", + "InvalidAvailabilityZone", + "InvalidCapacityReservationId.Malformed", + "InvalidCapacityReservationId.NotFound", + "InvalidEgressOnlyInternetGatewayId.NotFound", + "InvalidGroup.Duplicate", + "InvalidGroup.NotFound", + "InvalidGroup.Reserved", + "InvalidInstanceAttributeValue", + "InvalidInstanceID.Malformed", + "InvalidInstanceID.NotFound", + "InvalidInstanceType", + "InvalidInternetGatewayID.NotFound", + "InvalidKey.Format", + "InvalidKeyPair.Duplicate", + "InvalidKeyPair.Format", + "InvalidKeyPair.NotFound", + "InvalidLaunchTemplateId.Malformed", + "InvalidLaunchTemplateId.NotFound", + "InvalidLaunchTemplateId.VersionNotFound", + "InvalidLaunchTemplateName.AlreadyExistsException", + "InvalidLaunchTemplateName.MalformedException", + "InvalidLaunchTemplateName.NotFoundException", + "InvalidNatGatewayID.NotFound", + "InvalidNetworkInterface.InUse", + "InvalidNetworkInterface.NotFound", + "InvalidNetworkInterfaceID.NotFound", + "InvalidPermission.Duplicate", + "InvalidPermission.NotFound", + "InvalidPlacementGroup.Duplicate", + "InvalidPlacementGroup.InUse", + "InvalidPlacementGroup.Unknown", + "InvalidRoute.NotFound", + "InvalidRouteTableID.NotFound", + "InvalidSecurityGroupRuleId.Malformed", + "InvalidSecurityGroupRuleId.NotFound", + "InvalidSnapshot.InUse", + "InvalidSnapshot.NotFound", + "InvalidSnapshotID.Malformed", + "InvalidSpotInstanceRequestID.NotFound", + "InvalidSubnet.Conflict", + "InvalidSubnet.Range", + "InvalidSubnetID.NotFound", + "InvalidVolume.NotFound", + "InvalidVolumeID.Malformed", + "InvalidVpcID.NotFound", + "InvalidVpcRange", + "OperationNotPermitted", + "ReservationCapacityExceeded", + "Resource.AlreadyAssociated", + "ResourceLimitExceeded", + "RouteAlreadyExists", + "RulesPerSecurityGroupLimitExceeded", + "SecurityGroupsPerInterfaceLimitExceeded", + "SignatureDoesNotMatch", + "UnknownVolumeType", + "Unsupported", + "VolumeInUse" + ], + "server": [ + "BandwidthLimitExceeded", + "InsufficientAddressCapacity", + "InsufficientCapacity", + "InsufficientHostCapacity", + "InsufficientInstanceCapacity", + "InsufficientReservedInstanceCapacity", + "InsufficientVolumeCapacity", + "InternalError", + "InternalFailure", + "RequestLimitExceeded", + "ServerInternal", + "ServiceUnavailable", + "Unavailable" + ] +} diff --git a/internal/awsmodel/response.go b/internal/awsmodel/response.go new file mode 100644 index 000000000..365777c51 --- /dev/null +++ b/internal/awsmodel/response.go @@ -0,0 +1,251 @@ +package awsmodel + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "strings" +) + +// ValidateResponse decodes an AWS wire response and validates it against the +// operation's output shape. JSON services are decoded directly; EC2 and Query +// XML responses are normalized from their wire names and result wrappers to +// api-2.json member names first. +func ValidateResponse(service Service, operationName string, body []byte) ([]Violation, error) { + model, err := Load(service) + if err != nil { + return nil, err + } + operation, ok := model.Operation(operationName) + if !ok { + return nil, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName) + } + var document any + switch model.metadata.Protocol { + case "json", "rest-json": + document, err = decodeJSONResponse(body) + case "ec2", "query": + if operation.Output == nil { + err = validateEmptyXMLResponse(operationName, body) + } else { + document, err = model.decodeXMLResponse(operationName, operation.Output, body) + } + default: + return nil, fmt.Errorf("awsmodel: response decoding for %s protocol %q is not implemented", service, model.metadata.Protocol) + } + if err != nil { + return nil, fmt.Errorf("awsmodel: decode %s %s response: %w", service, operationName, err) + } + if operation.Output == nil { + // Some Query/EC2 operations deliberately have no modelled output. There + // is no shape to inspect, but decoding above still verifies that the + // response is syntactically valid and has the correct XML envelope. + return nil, nil + } + return Validate(service, operationName, document) +} + +func decodeJSONResponse(body []byte) (any, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + var document any + if err := decoder.Decode(&document); err != nil { + return nil, err + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + if err == nil { + return nil, errors.New("multiple JSON values") + } + return nil, err + } + return document, nil +} + +type xmlNode struct { + name string + text strings.Builder + children []*xmlNode +} + +func parseXML(body []byte) (*xmlNode, error) { + decoder := xml.NewDecoder(bytes.NewReader(body)) + var root *xmlNode + var stack []*xmlNode + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + switch token := token.(type) { + case xml.StartElement: + node := &xmlNode{name: token.Name.Local} + if len(stack) == 0 { + if root != nil { + return nil, errors.New("multiple XML roots") + } + root = node + } else { + parent := stack[len(stack)-1] + parent.children = append(parent.children, node) + } + stack = append(stack, node) + case xml.CharData: + if len(stack) != 0 { + stack[len(stack)-1].text.Write([]byte(token)) + } + case xml.EndElement: + if len(stack) == 0 { + return nil, errors.New("unexpected XML end element") + } + stack = stack[:len(stack)-1] + } + } + if root == nil { + return nil, errors.New("empty XML response") + } + return root, nil +} + +func validateEmptyXMLResponse(operationName string, body []byte) error { + root, err := parseXML(body) + if err != nil { + return err + } + wantRoot := operationName + "Response" + if root.name != wantRoot { + return fmt.Errorf("root element is %q, want %q", root.name, wantRoot) + } + return nil +} + +func (m *Model) decodeXMLResponse(operationName string, output *ShapeRef, body []byte) (any, error) { + root, err := parseXML(body) + if err != nil { + return nil, err + } + wantRoot := operationName + "Response" + if root.name != wantRoot { + return nil, fmt.Errorf("root element is %q, want %q", root.name, wantRoot) + } + + outputNode := root + if output.ResultWrapper != "" { + outputNode = firstXMLChild(root, output.ResultWrapper) + if outputNode == nil { + return nil, fmt.Errorf("result wrapper %q is missing", output.ResultWrapper) + } + } + return m.normalizeXMLShape(output.Shape, []*xmlNode{outputNode}, false), nil +} + +func (m *Model) normalizeXMLShape(shapeName string, nodes []*xmlNode, flattened bool) any { + shape := m.shapes[shapeName] + if shape == nil || len(nodes) == 0 { + return nil + } + + switch shape.Type { + case "structure": + return m.normalizeXMLStructure(shape, nodes[0]) + case "list": + itemNodes := nodes + if !flattened && !shape.Flattened { + itemName := shape.Member.LocationName + if itemName == "" { + if m.metadata.Protocol == "ec2" { + itemName = "item" + } else { + itemName = "member" + } + } + itemNodes = xmlChildren(nodes[0], itemName) + } + items := make([]any, 0, len(itemNodes)) + for _, node := range itemNodes { + items = append(items, m.normalizeXMLShape(shape.Member.Shape, []*xmlNode{node}, false)) + } + return items + case "map": + entries := nodes + if !flattened && !shape.Flattened { + entries = xmlChildren(nodes[0], "entry") + } + result := make(map[string]any, len(entries)) + for _, entry := range entries { + keyNode := firstXMLChild(entry, xmlRefName("key", *shape.Key)) + valueNode := firstXMLChild(entry, xmlRefName("value", *shape.Value)) + if keyNode != nil && valueNode != nil { + key := strings.TrimSpace(keyNode.text.String()) + result[key] = m.normalizeXMLShape(shape.Value.Shape, []*xmlNode{valueNode}, false) + } + } + return result + default: + return strings.TrimSpace(nodes[0].text.String()) + } +} + +func (m *Model) normalizeXMLStructure(shape *Shape, node *xmlNode) map[string]any { + document := make(map[string]any) + wireMembers := make(map[string]string, len(shape.Members)) + for name, ref := range shape.Members { + wireMembers[xmlRefName(name, ref)] = name + } + + grouped := make(map[string][]*xmlNode) + for _, child := range node.children { + if isTransportMetadata(child.name) { + continue + } + grouped[child.name] = append(grouped[child.name], child) + } + for wireName, children := range grouped { + name, ok := wireMembers[wireName] + if !ok { + document[wireName] = strings.TrimSpace(children[0].text.String()) + continue + } + ref := shape.Members[name] + target := m.shapes[ref.Shape] + flattened := ref.Flattened || (target != nil && target.Flattened) + document[name] = m.normalizeXMLShape(ref.Shape, children, flattened) + } + return document +} + +func xmlRefName(memberName string, ref ShapeRef) string { + if ref.LocationName != "" { + return ref.LocationName + } + return memberName +} + +func firstXMLChild(node *xmlNode, name string) *xmlNode { + for _, child := range node.children { + if child.name == name { + return child + } + } + return nil +} + +func xmlChildren(node *xmlNode, name string) []*xmlNode { + children := make([]*xmlNode, 0) + for _, child := range node.children { + if child.name == name { + children = append(children, child) + } + } + return children +} + +func isTransportMetadata(name string) bool { + normalized := strings.ToLower(name) + return normalized == "requestid" || normalized == "responsemetadata" +} diff --git a/internal/awsmodel/response_test.go b/internal/awsmodel/response_test.go new file mode 100644 index 000000000..c721a6c22 --- /dev/null +++ b/internal/awsmodel/response_test.go @@ -0,0 +1,86 @@ +package awsmodel + +import ( + "reflect" + "strings" + "testing" +) + +func TestValidateResponseEC2XML(t *testing.T) { + body := []byte(` + req-1 + + 16paused + surprise + +`) + + violations, err := ValidateResponse(EC2, "DescribeInstances", body) + if err != nil { + t.Fatal(err) + } + want := []Violation{ + {Rule: RuleUnknownField, Path: "$.Reservations[0].Instances[0].inventedField", Message: "field is not present in the model shape"}, + {Rule: RuleEnum, Path: "$.Reservations[0].Instances[0].State.Name", Message: `value "paused" is not one of [pending, running, shutting-down, terminated, stopping, stopped]`}, + } + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateResponseQueryXML(t *testing.T) { + body := []byte(` + /AIDAEXAMPLE + arn:aws:iam::123456789012:user/alice + 2026-08-05T00:00:00Z +`) + + violations, err := ValidateResponse(IAM, "GetUser", body) + if err != nil { + t.Fatal(err) + } + want := []Violation{{Rule: RuleRequiredMember, Path: "$.User.UserName", Message: "required member is missing"}} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateResponseJSON(t *testing.T) { + body := []byte(`{"tasks":[{"launchType":"ON_PREMISES"}]}`) + violations, err := ValidateResponse(ECS, "DescribeTasks", body) + if err != nil { + t.Fatal(err) + } + if len(violations) != 1 || violations[0].Rule != RuleEnum || violations[0].Path != "$.tasks[0].launchType" { + t.Fatalf("violations = %#v, want launchType enum violation", violations) + } +} + +func TestValidateResponseConformingEmptyEC2List(t *testing.T) { + body := []byte(`req-1`) + violations, err := ValidateResponse(EC2, "DescribeInstances", body) + if err != nil { + t.Fatal(err) + } + if len(violations) != 0 { + t.Fatalf("violations = %#v, want none", violations) + } +} + +func TestValidateResponseOperationWithoutOutputShape(t *testing.T) { + body := []byte(`req-1`) + violations, err := ValidateResponse(EC2, "AttachInternetGateway", body) + if err != nil { + t.Fatal(err) + } + if len(violations) != 0 { + t.Fatalf("violations = %#v, want none", violations) + } +} + +func TestValidateResponseRejectsWrongWrapper(t *testing.T) { + _, err := ValidateResponse(IAM, "GetUser", []byte(``)) + if err == nil || !strings.Contains(err.Error(), `result wrapper "GetUserResult" is missing`) { + t.Fatalf("error = %v", err) + } +} diff --git a/internal/awsmodel/validate.go b/internal/awsmodel/validate.go new file mode 100644 index 000000000..7549f3790 --- /dev/null +++ b/internal/awsmodel/validate.go @@ -0,0 +1,140 @@ +package awsmodel + +import ( + "fmt" + "slices" + "strconv" + "strings" +) + +// Rule identifies a model constraint violated by a response document. +type Rule string + +const ( + RuleUnknownField Rule = "unknown_field" + RuleRequiredMember Rule = "required_member" + RuleEnum Rule = "enum" + RuleErrorCode Rule = "error_code" + RuleHTTPStatus Rule = "http_status" +) + +// Violation describes one response value that does not conform to its output +// shape. Path uses a JSONPath-like notation rooted at $. +type Violation struct { + Rule Rule + Path string + Message string +} + +func (v Violation) String() string { + return fmt.Sprintf("%s: %s (%s)", v.Path, v.Message, v.Rule) +} + +// Validate checks a normalized response document against an operation's +// output shape. Structure keys must use the member names from api-2.json; +// protocol-specific response middleware is responsible for translating wire +// names and wrappers to that representation. +func Validate(service Service, operationName string, document any) ([]Violation, error) { + model, err := Load(service) + if err != nil { + return nil, err + } + operation, ok := model.Operation(operationName) + if !ok { + return nil, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName) + } + if operation.Output == nil { + return nil, nil + } + + var violations []Violation + model.validateShape(operation.Output.Shape, "$", document, &violations) + return violations, nil +} + +func (m *Model) validateShape(shapeName, path string, value any, violations *[]Violation) { + shape := m.shapes[shapeName] + if shape == nil { + // Load verifies references, so this is unreachable for a loaded model. + return + } + + switch shape.Type { + case "structure": + m.validateStructure(shape, path, value, violations) + case "list": + values, ok := value.([]any) + if !ok { + return + } + for index, item := range values { + m.validateShape(shape.Member.Shape, fmt.Sprintf("%s[%d]", path, index), item, violations) + } + case "map": + values, ok := value.(map[string]any) + if !ok { + return + } + keys := sortedKeys(values) + for _, key := range keys { + keyPath := path + "[" + strconv.Quote(key) + "]" + m.validateShape(shape.Key.Shape, keyPath+".key", key, violations) + m.validateShape(shape.Value.Shape, keyPath, values[key], violations) + } + case "string": + stringValue, ok := value.(string) + if !ok || len(shape.Enum) == 0 { + return + } + if !slices.Contains(shape.Enum, stringValue) { + *violations = append(*violations, Violation{ + Rule: RuleEnum, + Path: path, + Message: fmt.Sprintf("value %q is not one of [%s]", stringValue, strings.Join(shape.Enum, ", ")), + }) + } + } +} + +func (m *Model) validateStructure(shape *Shape, path string, value any, violations *[]Violation) { + fields, ok := value.(map[string]any) + if !ok { + fields = map[string]any{} + } + + required := append([]string(nil), shape.Required...) + slices.Sort(required) + for _, name := range required { + if _, present := fields[name]; !present { + *violations = append(*violations, Violation{ + Rule: RuleRequiredMember, + Path: memberPath(path, name), + Message: "required member is missing", + }) + } + } + + fieldNames := sortedKeys(fields) + for _, name := range fieldNames { + if _, modelled := shape.Members[name]; !modelled { + *violations = append(*violations, Violation{ + Rule: RuleUnknownField, + Path: memberPath(path, name), + Message: "field is not present in the model shape", + }) + } + } + + memberNames := sortedKeys(shape.Members) + for _, name := range memberNames { + field, present := fields[name] + if !present { + continue + } + m.validateShape(shape.Members[name].Shape, memberPath(path, name), field, violations) + } +} + +func memberPath(parent, name string) string { + return parent + "." + name +} diff --git a/internal/awsmodel/validate_test.go b/internal/awsmodel/validate_test.go new file mode 100644 index 000000000..0181cf652 --- /dev/null +++ b/internal/awsmodel/validate_test.go @@ -0,0 +1,128 @@ +package awsmodel + +import ( + "reflect" + "strings" + "testing" +) + +func TestValidateUnknownField(t *testing.T) { + document := validIAMGetUserDocument() + document["Invented"] = true + + violations, err := Validate(IAM, "GetUser", document) + if err != nil { + t.Fatal(err) + } + want := []Violation{{ + Rule: RuleUnknownField, + Path: "$.Invented", + Message: "field is not present in the model shape", + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateRequiredMembers(t *testing.T) { + document := validIAMGetUserDocument() + delete(document["User"].(map[string]any), "UserName") + + violations, err := Validate(IAM, "GetUser", document) + if err != nil { + t.Fatal(err) + } + want := []Violation{{ + Rule: RuleRequiredMember, + Path: "$.User.UserName", + Message: "required member is missing", + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateEnum(t *testing.T) { + document := map[string]any{ + "tasks": []any{ + map[string]any{"launchType": "ON_PREMISES"}, + }, + } + + violations, err := Validate(ECS, "DescribeTasks", document) + if err != nil { + t.Fatal(err) + } + want := []Violation{{ + Rule: RuleEnum, + Path: "$.tasks[0].launchType", + Message: "value \"ON_PREMISES\" is not one of [EC2, FARGATE, EXTERNAL]", + }} + if !reflect.DeepEqual(violations, want) { + t.Fatalf("violations = %#v, want %#v", violations, want) + } +} + +func TestValidateConformingDocuments(t *testing.T) { + tests := []struct { + name string + service Service + operation string + document any + }{ + { + name: "IAM required members", + service: IAM, + operation: "GetUser", + document: validIAMGetUserDocument(), + }, + { + name: "ECS enum", + service: ECS, + operation: "DescribeTasks", + document: map[string]any{ + "tasks": []any{map[string]any{"launchType": "EC2"}}, + "failures": []any{}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + violations, err := Validate(test.service, test.operation, test.document) + if err != nil { + t.Fatal(err) + } + if len(violations) != 0 { + t.Fatalf("violations = %#v, want none", violations) + } + }) + } +} + +func TestValidateUnknownOperation(t *testing.T) { + _, err := Validate(STS, "InventedOperation", map[string]any{}) + if err == nil || !strings.Contains(err.Error(), `operation "InventedOperation" is not modelled`) { + t.Fatalf("Validate unknown operation error = %v", err) + } +} + +func TestViolationString(t *testing.T) { + violation := Violation{Rule: RuleEnum, Path: "$.status", Message: `value "broken" is invalid`} + want := `$.status: value "broken" is invalid (enum)` + if got := violation.String(); got != want { + t.Fatalf("String() = %q, want %q", got, want) + } +} + +func validIAMGetUserDocument() map[string]any { + return map[string]any{ + "User": map[string]any{ + "Path": "/", + "UserName": "alice", + "UserId": "AIDAEXAMPLE", + "Arn": "arn:aws:iam::123456789012:user/alice", + "CreateDate": "2026-08-05T00:00:00Z", + }, + } +} diff --git a/spinifex/gateway/ecrapi.go b/spinifex/gateway/ecrapi.go index ee7141f9a..6af45866a 100644 --- a/spinifex/gateway/ecrapi.go +++ b/spinifex/gateway/ecrapi.go @@ -11,6 +11,27 @@ import ( gateway_ecrapi "github.com/mulgadc/spinifex/spinifex/gateway/ecrapi" ) +type ecrInlineHandler func(*GatewayConfig, http.ResponseWriter, *http.Request) error + +// ecrInlineActions is the authoritative inventory of ECR operations handled +// directly by the gateway rather than relayed through gateway_ecrapi.Actions. +// Keeping dispatch and coverage on the same map prevents the generated +// operation report from classifying an inline implementation as a stub. +var ecrInlineActions = map[string]ecrInlineHandler{ + "GetAuthorizationToken": (*GatewayConfig).handleGetAuthorizationToken, + "DescribeRepositories": (*GatewayConfig).handleDescribeRepositories, + "CreateRepository": (*GatewayConfig).handleCreateRepository, + "DeleteRepository": (*GatewayConfig).handleDeleteRepository, + "PutImageTagMutability": (*GatewayConfig).handlePutImageTagMutability, + "ListImages": (*GatewayConfig).handleListImages, + "DescribeImages": (*GatewayConfig).handleDescribeImages, + "BatchGetImage": (*GatewayConfig).handleBatchGetImage, + "PutImage": (*GatewayConfig).handlePutImage, + "BatchDeleteImage": (*GatewayConfig).handleBatchDeleteImage, + "StartLifecyclePolicyPreview": (*GatewayConfig).handleStartLifecyclePolicyPreview, + "GetLifecyclePolicyPreview": (*GatewayConfig).handleGetLifecyclePolicyPreview, +} + // ecrActionFromTarget extracts the action suffix from an X-Amz-Target header. // Any "." or bare "" form is accepted. func ecrActionFromTarget(target string) string { @@ -39,58 +60,8 @@ func (gw *GatewayConfig) ECR_Request(w http.ResponseWriter, r *http.Request) err return err } - // GetAuthorizationToken mints a registry token from the SigV4 auth context; - // it is served inline rather than relayed onto a NATS subject. - if action == "GetAuthorizationToken" { - return gw.handleGetAuthorizationToken(w, r) - } - - // DescribeRepositories builds repository ARNs/URIs from the gateway region - // and internal suffix, which the relayed Handler signature does not carry, - // so it is served inline like GetAuthorizationToken. - if action == "DescribeRepositories" { - return gw.handleDescribeRepositories(w, r) - } - - // CreateRepository and DeleteRepository return the repository ARN/URI, built - // from the gateway region and internal suffix, so they are served inline. - if action == "CreateRepository" { - return gw.handleCreateRepository(w, r) - } - if action == "DeleteRepository" { - return gw.handleDeleteRepository(w, r) - } - - // PutImageTagMutability read-modify-writes the per-account RepoMeta record - // over NATS, like CreateRepository, so it is served inline. - if action == "PutImageTagMutability" { - return gw.handlePutImageTagMutability(w, r) - } - - // Image-management actions need the gateway-side predastore Store (manifest - // bytes) carried by the OCI Registry, which the relayed Handler signature - // does not reach, so they are served inline. - switch action { - case "ListImages": - return gw.handleListImages(w, r) - case "DescribeImages": - return gw.handleDescribeImages(w, r) - case "BatchGetImage": - return gw.handleBatchGetImage(w, r) - case "PutImage": - return gw.handlePutImage(w, r) - case "BatchDeleteImage": - return gw.handleBatchDeleteImage(w, r) - } - - // Lifecycle-policy preview evaluates the stored/override policy against the - // repo's current images via the gateway-side OCI registry, which the relayed - // Handler signature does not reach, so they are served inline. - switch action { - case "StartLifecyclePolicyPreview": - return gw.handleStartLifecyclePolicyPreview(w, r) - case "GetLifecyclePolicyPreview": - return gw.handleGetLifecyclePolicyPreview(w, r) + if inline, ok := ecrInlineActions[action]; ok { + return inline(gw, w, r) } accountID, _ := r.Context().Value(ctxAccountID).(string) diff --git a/spinifex/gateway/ecrapi/handler.go b/spinifex/gateway/ecrapi/handler.go index 8804c1b15..1c89ce014 100644 --- a/spinifex/gateway/ecrapi/handler.go +++ b/spinifex/gateway/ecrapi/handler.go @@ -13,6 +13,8 @@ import ( "errors" "log/slog" "net/http" + "reflect" + "sort" "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" "github.com/mulgadc/spinifex/spinifex/awserrors" @@ -102,6 +104,29 @@ var Actions = map[string]Handler{ "ListTagsForResource": ListTagsForResource, } +// StubbedActionNames returns the Actions entries still bound to NotImplemented. +func StubbedActionNames() []string { + return actionsWithHandler(NotImplemented) +} + +// UnsupportedActionNames returns actions that deliberately report the feature +// as unsupported rather than behaving as implemented operations. +func UnsupportedActionNames() []string { + return actionsWithHandler(ScanningNotSupported) +} + +func actionsWithHandler(target Handler) []string { + want := reflect.ValueOf(target).Pointer() + actions := make([]string, 0) + for name, handler := range Actions { + if reflect.ValueOf(handler).Pointer() == want { + actions = append(actions, name) + } + } + sort.Strings(actions) + return actions +} + // WriteJSONResponse serialises obj as a 200 AWS JSON 1.1 response using the // SDK's jsonutil marshaler (epoch-second time encoding), matching ACM/EKS. func WriteJSONResponse(w http.ResponseWriter, obj any) { diff --git a/spinifex/gateway/ecs/handler.go b/spinifex/gateway/ecs/handler.go index 31604b51d..b0f6668c4 100644 --- a/spinifex/gateway/ecs/handler.go +++ b/spinifex/gateway/ecs/handler.go @@ -14,6 +14,8 @@ import ( "errors" "log/slog" "net/http" + "reflect" + "sort" "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" "github.com/mulgadc/spinifex/spinifex/awserrors" @@ -109,6 +111,19 @@ var Actions = map[string]Handler{ "ListTagsForResource": ListTagsForResource, } +// StubbedActionNames returns the Actions entries still bound to NotImplemented. +func StubbedActionNames() []string { + stub := reflect.ValueOf(NotImplemented).Pointer() + actions := make([]string, 0) + for name, handler := range Actions { + if reflect.ValueOf(handler).Pointer() == stub { + actions = append(actions, name) + } + } + sort.Strings(actions) + return actions +} + // RawJSONActions encode their response with encoding/json instead of jsonutil. // These are internal agent↔gateway shapes (not AWS SDK types) whose payloads // carry RFC3339 time fields the on-VM agent decodes with encoding/json; the diff --git a/spinifex/gateway/operations.go b/spinifex/gateway/operations.go new file mode 100644 index 000000000..fa4d31b34 --- /dev/null +++ b/spinifex/gateway/operations.go @@ -0,0 +1,86 @@ +package gateway + +import ( + "sort" + + gateway_ecrapi "github.com/mulgadc/spinifex/spinifex/gateway/ecrapi" + gateway_ecs "github.com/mulgadc/spinifex/spinifex/gateway/ecs" +) + +// ServiceOperationInventory describes the authoritative dispatch state for an +// AWS-compatible service. Registered operations minus Stubbed and Unsupported +// are implemented handlers. +type ServiceOperationInventory struct { + Registered []string + Stubbed []string + Unsupported []string +} + +// AWSOperationInventory returns a fresh, stable snapshot of the gateway's +// operation dispatch tables. S3 is intentionally absent: Spinifex does not +// dispatch S3 operations and delegates that REST surface to Predastore. +func AWSOperationInventory() map[string]ServiceOperationInventory { + ecrInline := mapKeys(ecrInlineActions) + ecrRegistered := union(mapKeys(gateway_ecrapi.Actions), ecrInline) + ecrStubbed := without(gateway_ecrapi.StubbedActionNames(), ecrInline) + return map[string]ServiceOperationInventory{ + "acm": { + Registered: mapKeys(acmActions), + }, + "ec2": { + Registered: mapKeys(ec2Actions), + }, + "ecr": { + Registered: ecrRegistered, + Stubbed: ecrStubbed, + Unsupported: gateway_ecrapi.UnsupportedActionNames(), + }, + "ecs": { + Registered: mapKeys(gateway_ecs.Actions), + Stubbed: gateway_ecs.StubbedActionNames(), + }, + "elasticloadbalancingv2": { + Registered: mapKeys(elbv2Actions), + }, + "iam": { + Registered: mapKeys(iamActions), + }, + "sts": { + Registered: mapKeys(stsActions), + }, + } +} + +func mapKeys[K ~string, V any](values map[K]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, string(key)) + } + sort.Strings(keys) + return keys +} + +func without(values, excluded []string) []string { + exclude := make(map[string]bool, len(excluded)) + for _, value := range excluded { + exclude[value] = true + } + result := make([]string, 0, len(values)) + for _, value := range values { + if !exclude[value] { + result = append(result, value) + } + } + return result +} + +func union(left, right []string) []string { + set := make(map[string]bool, len(left)+len(right)) + for _, value := range left { + set[value] = true + } + for _, value := range right { + set[value] = true + } + return mapKeys(set) +} diff --git a/spinifex/gateway/operations_test.go b/spinifex/gateway/operations_test.go new file mode 100644 index 000000000..11621b84e --- /dev/null +++ b/spinifex/gateway/operations_test.go @@ -0,0 +1,51 @@ +package gateway + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAWSOperationInventoryClassifiesDispatchers(t *testing.T) { + inventory := AWSOperationInventory() + + require.Contains(t, inventory["ec2"].Registered, "DescribeInstances") + require.NotContains(t, inventory["ec2"].Stubbed, "DescribeInstances") + + require.Contains(t, inventory["ecs"].Registered, "DescribeClusters") + require.NotContains(t, inventory["ecs"].Stubbed, "DescribeClusters") + require.Contains(t, inventory["ecs"].Stubbed, "UpdateCluster") + + // These operations are NotImplemented in the relay table but intercepted + // by ECR_Request's authoritative inline dispatch map. + require.Contains(t, inventory["ecr"].Registered, "GetAuthorizationToken") + require.NotContains(t, inventory["ecr"].Stubbed, "GetAuthorizationToken") + require.NotContains(t, inventory["ecr"].Stubbed, "DescribeRepositories") + require.Contains(t, inventory["ecr"].Unsupported, "StartImageScan") + + _, hasS3 := inventory["s3"] + require.False(t, hasS3, "S3 is delegated to Predastore, not dispatched here") +} + +func TestAWSOperationInventoryStatusesAreDisjointSubsets(t *testing.T) { + for service, inventory := range AWSOperationInventory() { + t.Run(service, func(t *testing.T) { + registered := stringSet(inventory.Registered) + for _, action := range inventory.Stubbed { + require.True(t, registered[action], "stubbed action %q is not registered", action) + require.NotContains(t, inventory.Unsupported, action, "action cannot be stubbed and unsupported") + } + for _, action := range inventory.Unsupported { + require.True(t, registered[action], "unsupported action %q is not registered", action) + } + }) + } +} + +func stringSet(values []string) map[string]bool { + set := make(map[string]bool, len(values)) + for _, value := range values { + set[value] = true + } + return set +} diff --git a/tests/integration/conformance-known-findings.md b/tests/integration/conformance-known-findings.md new file mode 100644 index 000000000..521322b19 --- /dev/null +++ b/tests/integration/conformance-known-findings.md @@ -0,0 +1,85 @@ +# AWS model conformance ratchet + +The integration suite validates every successful modelled AWS response. Services +listed in `conformance-promoted-services.json` are blocking in the default +`fail` mode; findings from all other services remain warnings until triaged and +fixed. + +Run the ratchet with: + +```sh +make test-integration +``` + +Use `AWS_MODEL_CONFORMANCE_MODE=warn` only for investigation. The report is +written to `.cache/aws-model-conformance-report.txt` by default. + +## Promoted services + +- `sts` — promoted on 2026-08-05 after the integration baseline completed with + no STS violations or decoder errors. + +## Triaged warnings + +Baseline from 2026-08-05: 370 successful responses checked, three violations, +zero decoder errors. + +| Service / operation | Finding | Classification | Implementation cause | +|---|---|---|---| +| EC2 `DescribeImages` | `BootMode` is emitted as `""`, outside the model enum | Defect | Legacy AMI metadata is intentionally projected through `aws.String(amiMeta.BootMode)`, which emits an empty element instead of omitting the optional member. | +| IAM `ListAccessKeys` | Required `AccessKeyMetadata` missing | Defect | The empty result starts as a nil slice, so Query XML serialization omits the required list container. | +| IAM `ListGroups` | Required `Groups` missing | Defect | When the KV bucket has keys but none match the account, the result slice remains nil and Query XML serialization omits the required list container. | + +These warnings are not suppressed by a finding allowlist. They remain visible +on every run. Handler changes belong in focused follow-up work; once a service's +warnings are resolved, add that service to the promoted-services file in the +same change that proves its full integration baseline is clean. + +## Error-code coverage + +Phase 4 validates IAM, STS, ECS, and ELBv2 error envelopes against their service +models. The 2026-08-05 full integration baseline checked 46 operation-declared +errors (45 IAM, one STS) with no error-code violations. ECS and ELBv2 +decoding/model comparison have focused tests but their error paths are not +exercised by the current integration suite, so neither service is ready for +promotion on that evidence. + +Twelve responses are reported separately as `errors_unmodelled`: four IAM and +eight STS. These are protocol-level authentication/authorization/signature/ +throttling errors that AWS omits from operation error lists, plus STS's runtime +`ValidationError` and `InvalidParameterValue` responses. “Unmodelled” is not a +conformance pass; it means the operation model cannot judge the code. They stay +visible in the report rather than being silently counted as conforming. + +EC2's model declares no operation errors, so EC2 uses a separate checked-in +catalog curated from AWS's official EC2 error-code reference. It contains every +documented common and server error plus the documented action-specific codes +Spinifex currently emits. The validator checks the EC2 XML envelope, catalog +membership, and AWS's documented 4xx client / 5xx server classification. + +The same baseline examined 42 EC2 error responses and exposed 16 violations: + +| Finding | Count | Classification | Implementation cause | +|---|---:|---|---| +| EC2 authorization failures return `AccessDenied`, which is absent from the EC2 catalog; EC2 documents `UnauthorizedOperation` | 15 | Defect | The shared policy evaluator returns the Query/IAM-oriented `AccessDenied` code for EC2 requests. | +| `RequestSpotInstances` returns `InsufficientInstanceCapacity` with HTTP 400 instead of the documented server-error 5xx class | 1 | Defect | The global error lookup currently assigns HTTP 400 to `InsufficientInstanceCapacity`. | + +Six additional production-referenced values are deliberately not accepted by +the catalog because the chosen EC2 reference does not list them: `ExpiredToken`, +`IamInstanceProfileAlreadyAssociated`, `InvalidIamInstanceProfile.NotFound`, +`NoSuchAssociation`, `NoSuchEntity`, and `RequestEntityTooLarge`. A runtime +occurrence remains a visible violation until it is either backed by an +authoritative EC2 source or replaced with a documented EC2 code. + +## Operation coverage + +Phase 5 generates the public model-versus-dispatch inventory at +`docs/aws-model-operation-coverage.md`. Run `make aws-model-coverage` to +regenerate and print it. A normal `make build` and the integration target both +regenerate the document so handler-map changes cannot silently leave it stale. + +The inventory counts modelled operations bound to real handlers separately +from registered stubs and deliberately unsupported handlers. S3 is marked +opaque because Spinifex delegates its REST surface to Predastore rather than +using an operation-name dispatch table; no mechanical S3 coverage percentage +is claimed. diff --git a/tests/integration/conformance-promoted-services.json b/tests/integration/conformance-promoted-services.json new file mode 100644 index 000000000..13df2c714 --- /dev/null +++ b/tests/integration/conformance-promoted-services.json @@ -0,0 +1,5 @@ +{ + "promotedServices": [ + "sts" + ] +} diff --git a/tests/integration/conformance.go b/tests/integration/conformance.go new file mode 100644 index 000000000..374cfc725 --- /dev/null +++ b/tests/integration/conformance.go @@ -0,0 +1,409 @@ +//go:build integration + +package integration + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "sync" + + "github.com/mulgadc/spinifex/internal/awsmodel" +) + +type conformanceFinding struct { + service awsmodel.Service + operation string + violation awsmodel.Violation + count int +} + +type conformanceDiagnostic struct { + service awsmodel.Service + operation string + message string + count int +} + +// conformanceCollector is shared across the integration suite. Findings for +// services in the checked-in promotion policy become blocking in fail mode. +type conformanceCollector struct { + mu sync.Mutex + checked int + checkedByService map[awsmodel.Service]int + errorChecked int + errorCheckedByService map[awsmodel.Service]int + errorUnmodelled int + errorUnmodelledByService map[awsmodel.Service]int + findings map[string]*conformanceFinding + diagnostics map[string]*conformanceDiagnostic +} + +func newConformanceCollector() *conformanceCollector { + return &conformanceCollector{ + checkedByService: make(map[awsmodel.Service]int), + errorCheckedByService: make(map[awsmodel.Service]int), + errorUnmodelledByService: make(map[awsmodel.Service]int), + findings: make(map[string]*conformanceFinding), + diagnostics: make(map[string]*conformanceDiagnostic), + } +} + +var suiteConformance = newConformanceCollector() + +func (c *conformanceCollector) record(service awsmodel.Service, operation string, body []byte) { + violations, err := awsmodel.ValidateResponse(service, operation, body) + + c.mu.Lock() + defer c.mu.Unlock() + c.checked++ + c.checkedByService[service]++ + if err != nil { + key := string(service) + "\x00" + operation + "\x00" + err.Error() + diagnostic := c.diagnostics[key] + if diagnostic == nil { + diagnostic = &conformanceDiagnostic{service: service, operation: operation, message: err.Error()} + c.diagnostics[key] = diagnostic + } + diagnostic.count++ + return + } + c.recordViolationsLocked(service, operation, violations) +} + +func (c *conformanceCollector) recordError(service awsmodel.Service, operation string, status int, body []byte) { + var ( + violations []awsmodel.Violation + modelled bool + err error + ) + if service == awsmodel.EC2 { + violations, err = awsmodel.ValidateEC2ErrorResponse(status, body) + modelled = true + } else { + violations, modelled, err = awsmodel.ValidateErrorResponse(service, operation, body) + } + + c.mu.Lock() + defer c.mu.Unlock() + if err != nil { + key := string(service) + "\x00" + operation + "\x00error\x00" + err.Error() + diagnostic := c.diagnostics[key] + if diagnostic == nil { + diagnostic = &conformanceDiagnostic{service: service, operation: operation, message: err.Error()} + c.diagnostics[key] = diagnostic + } + diagnostic.count++ + return + } + if !modelled { + c.errorUnmodelled++ + c.errorUnmodelledByService[service]++ + return + } + c.errorChecked++ + c.errorCheckedByService[service]++ + c.recordViolationsLocked(service, operation, violations) +} + +func (c *conformanceCollector) recordViolationsLocked(service awsmodel.Service, operation string, violations []awsmodel.Violation) { + for _, violation := range violations { + key := strings.Join([]string{string(service), operation, string(violation.Rule), violation.Path, violation.Message}, "\x00") + finding := c.findings[key] + if finding == nil { + finding = &conformanceFinding{service: service, operation: operation, violation: violation} + c.findings[key] = finding + } + finding.count++ + } +} + +func (c *conformanceCollector) counts() (checked, violations, diagnostics int) { + c.mu.Lock() + defer c.mu.Unlock() + for _, finding := range c.findings { + violations += finding.count + } + for _, diagnostic := range c.diagnostics { + diagnostics += diagnostic.count + } + return c.checked, violations, diagnostics +} + +func (c *conformanceCollector) blocking(policy conformancePolicy, mode conformanceMode) int { + if mode != conformanceModeFail { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return c.blockingLocked(policy) +} + +func (c *conformanceCollector) blockingLocked(policy conformancePolicy) int { + blocking := 0 + for _, finding := range c.findings { + if policy.isPromoted(finding.service) { + blocking += finding.count + } + } + for _, diagnostic := range c.diagnostics { + if policy.isPromoted(diagnostic.service) { + blocking += diagnostic.count + } + } + return blocking +} + +func (c *conformanceCollector) report(policy conformancePolicy, mode conformanceMode) string { + c.mu.Lock() + defer c.mu.Unlock() + + violationCount := 0 + findings := make([]*conformanceFinding, 0, len(c.findings)) + for _, finding := range c.findings { + violationCount += finding.count + findings = append(findings, finding) + } + diagnosticCount := 0 + diagnostics := make([]*conformanceDiagnostic, 0, len(c.diagnostics)) + for _, diagnostic := range c.diagnostics { + diagnosticCount += diagnostic.count + diagnostics = append(diagnostics, diagnostic) + } + sort.Slice(findings, func(i, j int) bool { + left, right := findings[i], findings[j] + return fmt.Sprint(left.service, "\x00", left.operation, "\x00", left.violation.Path, "\x00", left.violation.Rule) < + fmt.Sprint(right.service, "\x00", right.operation, "\x00", right.violation.Path, "\x00", right.violation.Rule) + }) + sort.Slice(diagnostics, func(i, j int) bool { + left, right := diagnostics[i], diagnostics[j] + return fmt.Sprint(left.service, "\x00", left.operation, "\x00", left.message) < + fmt.Sprint(right.service, "\x00", right.operation, "\x00", right.message) + }) + + var report strings.Builder + blocking := 0 + if mode == conformanceModeFail { + blocking = c.blockingLocked(policy) + } + promoted := make([]string, 0, len(policy.promoted)) + for _, service := range policy.services() { + promoted = append(promoted, string(service)) + } + fmt.Fprintf(&report, "AWS model conformance (%s): checked=%d errors_checked=%d errors_unmodelled=%d violations=%d decode_errors=%d blocking=%d promoted=%s\n", + mode, c.checked, c.errorChecked, c.errorUnmodelled, violationCount, diagnosticCount, blocking, strings.Join(promoted, ",")) + + checkedServices := make(map[awsmodel.Service]bool, len(c.checkedByService)+len(c.errorCheckedByService)+len(c.errorUnmodelledByService)+len(policy.promoted)) + for service := range c.checkedByService { + checkedServices[service] = true + } + for service := range c.errorCheckedByService { + checkedServices[service] = true + } + for service := range c.errorUnmodelledByService { + checkedServices[service] = true + } + for service := range policy.promoted { + checkedServices[service] = true + } + services := make([]awsmodel.Service, 0, len(checkedServices)) + for service := range checkedServices { + services = append(services, service) + } + sort.Slice(services, func(i, j int) bool { return services[i] < services[j] }) + for _, service := range services { + fmt.Fprintf(&report, "CHECKED %s success=%d errors=%d unmodelled_errors=%d promoted=%t\n", + service, c.checkedByService[service], c.errorCheckedByService[service], c.errorUnmodelledByService[service], policy.isPromoted(service)) + } + + for _, finding := range findings { + severity := "WARN" + if mode == conformanceModeFail && policy.isPromoted(finding.service) { + severity = "FAIL" + } + fmt.Fprintf(&report, "%s %s %s %s %s count=%d: %s\n", + severity, + finding.service, finding.operation, finding.violation.Path, finding.violation.Rule, finding.count, finding.violation.Message) + } + for _, diagnostic := range diagnostics { + severity := "WARN" + if mode == conformanceModeFail && policy.isPromoted(diagnostic.service) { + severity = "FAIL" + } + fmt.Fprintf(&report, "%s %s %s decode_error count=%d: %s\n", + severity, + diagnostic.service, diagnostic.operation, diagnostic.count, diagnostic.message) + } + return strings.TrimSuffix(report.String(), "\n") +} + +func conformanceMiddleware(next http.Handler, collector *conformanceCollector) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + service, operation, ok := resolveConformanceRequest(r) + if !ok { + next.ServeHTTP(w, r) + return + } + + capture := &conformanceResponseWriter{ResponseWriter: w} + next.ServeHTTP(capture, r) + status := capture.status + if status == 0 { + status = http.StatusOK + } + if status >= 200 && status < 300 { + collector.record(service, operation, capture.body.Bytes()) + } else if errorCodeConformanceService(service) { + collector.recordError(service, operation, status, capture.body.Bytes()) + } + }) +} + +func errorCodeConformanceService(service awsmodel.Service) bool { + switch service { + case awsmodel.EC2, awsmodel.IAM, awsmodel.STS, awsmodel.ECS, awsmodel.ElasticLoadBalancingV2: + return true + default: + return false + } +} + +type conformanceResponseWriter struct { + http.ResponseWriter + status int + body bytes.Buffer +} + +func (w *conformanceResponseWriter) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *conformanceResponseWriter) Write(body []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + w.body.Write(body) + return w.ResponseWriter.Write(body) +} + +func (w *conformanceResponseWriter) Flush() { + _ = http.NewResponseController(w.ResponseWriter).Flush() +} + +func (w *conformanceResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +func resolveConformanceRequest(r *http.Request) (awsmodel.Service, string, bool) { + service, ok := conformanceService(r) + if !ok { + return "", "", false + } + + var operation string + if target := r.Header.Get("X-Amz-Target"); target != "" { + if separator := strings.LastIndex(target, "."); separator >= 0 { + operation = target[separator+1:] + } else { + operation = target + } + } else if service != awsmodel.S3 { + operation = queryAction(r) + } + if operation == "" { + return "", "", false + } + + model, err := awsmodel.Load(service) + if err != nil { + return "", "", false + } + if _, modelled := model.Operation(operation); !modelled { + return "", "", false + } + return service, operation, true +} + +func conformanceService(r *http.Request) (awsmodel.Service, bool) { + if target := r.Header.Get("X-Amz-Target"); target != "" { + prefix := target + if separator := strings.LastIndex(target, "."); separator >= 0 { + prefix = target[:separator] + } + switch prefix { + case "AmazonEC2ContainerServiceV20141113": + return awsmodel.ECS, true + case "AmazonEC2ContainerRegistry_V20150921": + return awsmodel.ECR, true + case "CertificateManager": + return awsmodel.ACM, true + } + } + + authorization := r.Header.Get("Authorization") + credentialAt := strings.Index(authorization, "Credential=") + if credentialAt >= 0 { + credential := authorization[credentialAt+len("Credential="):] + if end := strings.IndexAny(credential, " ,"); end >= 0 { + credential = credential[:end] + } + parts := strings.Split(credential, "/") + if len(parts) >= 5 && parts[len(parts)-1] == "aws4_request" { + return modelService(parts[len(parts)-2]) + } + } + + // AssumeRoleWithWebIdentity is intentionally unsigned and is the only + // anonymous modelled request in this harness. + if queryAction(r) == "AssumeRoleWithWebIdentity" { + return awsmodel.STS, true + } + return "", false +} + +func modelService(signatureService string) (awsmodel.Service, bool) { + switch signatureService { + case "acm": + return awsmodel.ACM, true + case "ec2": + return awsmodel.EC2, true + case "ecr": + return awsmodel.ECR, true + case "ecs": + return awsmodel.ECS, true + case "elasticloadbalancing": + return awsmodel.ElasticLoadBalancingV2, true + case "iam": + return awsmodel.IAM, true + case "s3": + return awsmodel.S3, true + case "sts": + return awsmodel.STS, true + default: + return "", false + } +} + +func queryAction(r *http.Request) string { + if action := r.URL.Query().Get("Action"); action != "" { + return action + } + if r.Body == nil { + return "" + } + body, err := io.ReadAll(r.Body) + if err != nil { + return "" + } + r.Body = io.NopCloser(bytes.NewReader(body)) + values, err := url.ParseQuery(string(body)) + if err != nil { + return "" + } + return values.Get("Action") +} diff --git a/tests/integration/conformance_policy.go b/tests/integration/conformance_policy.go new file mode 100644 index 000000000..ba6f6130f --- /dev/null +++ b/tests/integration/conformance_policy.go @@ -0,0 +1,91 @@ +//go:build integration + +package integration + +import ( + _ "embed" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + "github.com/mulgadc/spinifex/internal/awsmodel" +) + +type conformanceMode string + +const ( + conformanceModeWarn conformanceMode = "warn" + conformanceModeFail conformanceMode = "fail" +) + +type conformancePolicy struct { + promoted map[awsmodel.Service]bool +} + +type conformancePolicyFile struct { + PromotedServices []awsmodel.Service `json:"promotedServices"` +} + +//go:embed conformance-promoted-services.json +var conformancePolicyJSON []byte + +func loadConformancePolicy() (conformancePolicy, error) { + decoder := json.NewDecoder(strings.NewReader(string(conformancePolicyJSON))) + decoder.DisallowUnknownFields() + var file conformancePolicyFile + if err := decoder.Decode(&file); err != nil { + return conformancePolicy{}, fmt.Errorf("parse promoted-services policy: %w", err) + } + + known := make(map[awsmodel.Service]bool) + for _, service := range awsmodel.Services() { + known[service] = true + } + policy := conformancePolicy{promoted: make(map[awsmodel.Service]bool, len(file.PromotedServices))} + for _, service := range file.PromotedServices { + if !known[service] { + return conformancePolicy{}, fmt.Errorf("promoted-services policy contains unknown service %q", service) + } + if policy.promoted[service] { + return conformancePolicy{}, fmt.Errorf("promoted-services policy contains duplicate service %q", service) + } + policy.promoted[service] = true + } + return policy, nil +} + +func conformancePolicyFor(services ...awsmodel.Service) conformancePolicy { + policy := conformancePolicy{promoted: make(map[awsmodel.Service]bool, len(services))} + for _, service := range services { + policy.promoted[service] = true + } + return policy +} + +func (p conformancePolicy) isPromoted(service awsmodel.Service) bool { + return p.promoted[service] +} + +func (p conformancePolicy) services() []awsmodel.Service { + services := make([]awsmodel.Service, 0, len(p.promoted)) + for service := range p.promoted { + services = append(services, service) + } + sort.Slice(services, func(i, j int) bool { return services[i] < services[j] }) + return services +} + +func conformanceModeFromEnvironment() (conformanceMode, error) { + value := strings.ToLower(strings.TrimSpace(os.Getenv("AWS_MODEL_CONFORMANCE_MODE"))) + if value == "" { + return conformanceModeFail, nil + } + switch conformanceMode(value) { + case conformanceModeWarn, conformanceModeFail: + return conformanceMode(value), nil + default: + return "", fmt.Errorf("AWS_MODEL_CONFORMANCE_MODE must be %q or %q, got %q", conformanceModeWarn, conformanceModeFail, value) + } +} diff --git a/tests/integration/conformance_policy_test.go b/tests/integration/conformance_policy_test.go new file mode 100644 index 000000000..f220015b4 --- /dev/null +++ b/tests/integration/conformance_policy_test.go @@ -0,0 +1,46 @@ +//go:build integration + +package integration + +import ( + "testing" + + "github.com/mulgadc/spinifex/internal/awsmodel" + "github.com/stretchr/testify/require" +) + +func TestEmbeddedConformancePolicyPromotesSTS(t *testing.T) { + policy, err := loadConformancePolicy() + require.NoError(t, err) + require.Equal(t, []awsmodel.Service{awsmodel.STS}, policy.services()) + require.True(t, policy.isPromoted(awsmodel.STS)) + require.False(t, policy.isPromoted(awsmodel.EC2)) +} + +func TestConformanceModeFromEnvironment(t *testing.T) { + tests := []struct { + name string + value string + want conformanceMode + wantErr bool + }{ + {name: "default is fail", want: conformanceModeFail}, + {name: "explicit fail", value: "fail", want: conformanceModeFail}, + {name: "warn", value: "warn", want: conformanceModeWarn}, + {name: "case and whitespace", value: " WARN ", want: conformanceModeWarn}, + {name: "invalid", value: "off", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv("AWS_MODEL_CONFORMANCE_MODE", test.value) + got, err := conformanceModeFromEnvironment() + if test.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} diff --git a/tests/integration/conformance_test.go b/tests/integration/conformance_test.go new file mode 100644 index 000000000..d6417be03 --- /dev/null +++ b/tests/integration/conformance_test.go @@ -0,0 +1,162 @@ +//go:build integration + +package integration + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/internal/awsmodel" + "github.com/stretchr/testify/require" +) + +func TestConformanceMiddlewareDetectsStubbedNATSMalformedResponse(t *testing.T) { + collector := newConformanceCollector() + gw := startGateway(t, collector) + stubEmptyInstanceBuckets(t, gw) + gw.StubSubject(t, "ec2.DescribeInstances", mustMarshal(t, &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{ + Instances: []*ec2.Instance{{ + State: &ec2.InstanceState{Name: aws.String("paused")}, + }}, + }}, + })) + + _, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{}) + require.NoError(t, err) + + checked, violations, diagnostics := collector.counts() + require.Equal(t, 1, checked) + require.Equal(t, 1, violations) + require.Zero(t, diagnostics) + require.Contains(t, collector.report(conformancePolicy{}, conformanceModeWarn), "$.Reservations[0].Instances[0].State.Name enum") +} + +func TestConformanceRatchetBlocksPromotedService(t *testing.T) { + collector := newConformanceCollector() + collector.record(awsmodel.STS, "GetCallerIdentity", []byte(` + user-1123456789012 + arn:aws:iam::123456789012:user/alicetrue +`)) + policy := conformancePolicyFor(awsmodel.STS) + + require.Equal(t, 1, collector.blocking(policy, conformanceModeFail)) + report := collector.report(policy, conformanceModeFail) + require.Contains(t, report, "blocking=1 promoted=sts") + require.Contains(t, report, "CHECKED sts success=1 errors=0 unmodelled_errors=0 promoted=true") + require.Contains(t, report, "FAIL sts GetCallerIdentity $.Invented unknown_field") +} + +func TestConformanceRatchetWarnModeDoesNotBlock(t *testing.T) { + collector := newConformanceCollector() + collector.record(awsmodel.STS, "GetCallerIdentity", []byte(` + true +`)) + policy := conformancePolicyFor(awsmodel.STS) + + require.Zero(t, collector.blocking(policy, conformanceModeWarn)) + report := collector.report(policy, conformanceModeWarn) + require.Contains(t, report, "blocking=0 promoted=sts") + require.Contains(t, report, "WARN sts GetCallerIdentity $.Invented unknown_field") + require.NotContains(t, report, "FAIL sts") +} + +func TestConformanceRatchetDoesNotBlockUnpromotedService(t *testing.T) { + collector := newConformanceCollector() + collector.record(awsmodel.ECS, "DescribeTasks", []byte(`{"tasks":[{"launchType":"INVALID"}]}`)) + policy := conformancePolicyFor(awsmodel.STS) + + require.Zero(t, collector.blocking(policy, conformanceModeFail)) + require.Contains(t, collector.report(policy, conformanceModeFail), "WARN ecs DescribeTasks") +} + +func TestConformanceMiddlewareAcceptsStubbedNATSConformingResponse(t *testing.T) { + collector := newConformanceCollector() + gw := startGateway(t, collector) + stubEmptyInstanceBuckets(t, gw) + gw.StubSubject(t, "ec2.DescribeInstances", mustMarshal(t, &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{ + Instances: []*ec2.Instance{{ + State: &ec2.InstanceState{Name: aws.String("running")}, + }}, + }}, + })) + + _, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{}) + require.NoError(t, err) + + checked, violations, diagnostics := collector.counts() + require.Equal(t, 1, checked) + require.Zero(t, violations) + require.Zero(t, diagnostics) +} + +func TestResolveConformanceRequestPreservesQueryBody(t *testing.T) { + body := "Action=GetCallerIdentity&Version=2011-06-15" + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIATEST/20260805/us-east-1/sts/aws4_request, SignedHeaders=host, Signature=abc") + + service, operation, ok := resolveConformanceRequest(req) + require.True(t, ok) + require.Equal(t, awsmodel.STS, service) + require.Equal(t, "GetCallerIdentity", operation) + + preserved, err := io.ReadAll(req.Body) + require.NoError(t, err) + require.True(t, bytes.Equal([]byte(body), preserved)) +} + +func TestConformanceMiddlewareChecksDeclaredErrorResponses(t *testing.T) { + collector := newConformanceCollector() + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"__type":"InvalidParameterException"}`)) + }) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{}`)) + req.Header.Set("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.DescribeTasks") + + conformanceMiddleware(next, collector).ServeHTTP(httptest.NewRecorder(), req) + checked, violations, diagnostics := collector.counts() + require.Zero(t, checked) + require.Zero(t, violations) + require.Zero(t, diagnostics) + require.Contains(t, collector.report(conformancePolicy{}, conformanceModeWarn), + "CHECKED ecs success=0 errors=1 unmodelled_errors=0 promoted=false") +} + +func TestConformanceMiddlewareChecksCuratedEC2ErrorResponses(t *testing.T) { + collector := newConformanceCollector() + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`AccessDenieddeniedrequest-1`)) + }) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`Action=DescribeInstances&Version=2016-11-15`)) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIATEST/20260805/us-east-1/ec2/aws4_request, SignedHeaders=host, Signature=abc") + + conformanceMiddleware(next, collector).ServeHTTP(httptest.NewRecorder(), req) + checked, violations, diagnostics := collector.counts() + require.Zero(t, checked) + require.Equal(t, 1, violations) + require.Zero(t, diagnostics) + require.Contains(t, collector.report(conformancePolicy{}, conformanceModeWarn), + `WARN ec2 DescribeInstances $error.Code error_code count=1: error code "AccessDenied" is not in the curated EC2 catalog`) +} + +func TestConformanceRatchetBlocksUndeclaredErrorForPromotedService(t *testing.T) { + collector := newConformanceCollector() + collector.recordError(awsmodel.STS, "AssumeRole", http.StatusBadRequest, []byte(` + InventedFailurebroken +`)) + policy := conformancePolicyFor(awsmodel.STS) + + require.Equal(t, 1, collector.blocking(policy, conformanceModeFail)) + report := collector.report(policy, conformanceModeFail) + require.Contains(t, report, "CHECKED sts success=0 errors=1 unmodelled_errors=0 promoted=true") + require.Contains(t, report, "FAIL sts AssumeRole $error.Code error_code") +} diff --git a/tests/integration/harness.go b/tests/integration/harness.go index 240c94fbd..8919b53b6 100644 --- a/tests/integration/harness.go +++ b/tests/integration/harness.go @@ -119,6 +119,10 @@ type Option func(*gateway.GatewayConfig) // every other control-plane path (auth, routing, XML/JSON marshaling, IAM // policy evaluation) runs for real. func StartGateway(t *testing.T, opts ...Option) *Gateway { + return startGateway(t, suiteConformance, opts...) +} + +func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option) *Gateway { t.Helper() // Every test connects into its own dynamically provisioned NATS account @@ -174,7 +178,11 @@ func StartGateway(t *testing.T, opts ...Option) *Gateway { opt(cfg) } - srv := httptest.NewServer(cfg.SetupRoutes()) + handler := cfg.SetupRoutes() + if collector != nil { + handler = conformanceMiddleware(handler, collector) + } + srv := httptest.NewServer(handler) t.Cleanup(srv.Close) gw := &Gateway{ diff --git a/tests/integration/main_test.go b/tests/integration/main_test.go index 2e742bd61..7cde64b04 100644 --- a/tests/integration/main_test.go +++ b/tests/integration/main_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" "os" + "path/filepath" "testing" ) @@ -28,6 +29,17 @@ import ( // Per-test state isolation is preserved despite the shared server — see the // accountAuthenticator doc comment in shared_nats.go for how. func TestMain(m *testing.M) { + policy, err := loadConformancePolicy() + if err != nil { + fmt.Fprintln(os.Stderr, "tests/integration: load AWS conformance policy:", err) + os.Exit(1) + } + mode, err := conformanceModeFromEnvironment() + if err != nil { + fmt.Fprintln(os.Stderr, "tests/integration:", err) + os.Exit(1) + } + h, err := startSharedNATS() if err != nil { fmt.Fprintln(os.Stderr, "tests/integration: failed to start shared NATS server:", err) @@ -37,6 +49,22 @@ func TestMain(m *testing.M) { code := m.Run() + report := suiteConformance.report(policy, mode) + fmt.Fprintln(os.Stderr, report) + if blocking := suiteConformance.blocking(policy, mode); blocking != 0 { + fmt.Fprintf(os.Stderr, "tests/integration: %d blocking AWS model conformance violation(s)\n", blocking) + code = 1 + } + if reportPath := os.Getenv("AWS_MODEL_CONFORMANCE_REPORT"); reportPath != "" { + if err := os.MkdirAll(filepath.Dir(reportPath), 0o755); err != nil { + fmt.Fprintln(os.Stderr, "tests/integration: create conformance report directory:", err) + code = 1 + } else if err := os.WriteFile(reportPath, []byte(report+"\n"), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "tests/integration: write conformance report:", err) + code = 1 + } + } + // Connections are closed here, together, rather than per-test: closing // one early can kill the connection the clienttoken singleton latched // onto (see above), so every connection's lifetime is pinned to the