//go:build integration // Package integration is the in-process integration test tier for spinifex's // AWS-compatible control plane. Tests here start the REAL gateway router // (gateway.SetupRoutes) against an embedded NATS JetStream instance shared // across the whole package (see TestMain in main_test.go), with real IAM/STS // services wired in for authentic SigV4 authz — the only thing faked is the // daemon side of the wire: the NATS subjects a live spinifex daemon would // answer (ec2.*, ebs.*, ...) are stubbed per-test via StubSubject. // // Nothing is provisioned: no tofu, no docker, no Spinifex daemons, no fixed // ports. Every test connects into its own isolated NATS account on the // shared server (see TestMain), so tests stay independent in state despite // sharing the underlying process, and are safe to run concurrently on a // shared build host. // // This package is gated behind the "integration" build tag (see the // `test-integration` Makefile target) so it is never swept by the default // `go test ./spinifex/...`, `make test-cover`, or `make test-race`. package integration import ( "encoding/json" awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers" "net/http/httptest" "testing" "github.com/aws/aws-sdk-go/aws" awscreds "github.com/aws/aws-sdk-go/aws/credentials" awssession "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/bedrock" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1" acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi" ecrdomain "github.com/mulgadc/spinifex/spinifex/domains/ecr" ecrauth "github.com/mulgadc/spinifex/spinifex/domains/ecr/auth" awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi" ecrregistry "github.com/mulgadc/spinifex/spinifex/domains/ecr/registry" "github.com/mulgadc/spinifex/spinifex/gateway" gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts" awsdispatch "github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch" "github.com/mulgadc/spinifex/spinifex/providers/objectstore" "github.com/nats-io/nats.go" "github.com/stretchr/testify/require" ) const ( // testRegion/testAZ pin the gateway's SigV4 credential scope and AZ. // Every SDK client built via Gateway.EC2Client signs against testRegion, // so requests verify against the same scope the gateway expects. testRegion = "us-east-1" testAZ = "us-east-1a" // testAccessKeyID/testSecretAccessKey are the root credentials seeded // into the real IAM service for every harness-started gateway. AKIA // prefix routes SigV4 auth through the long-lived-key path (see // spinifex/gateway/auth.go resolveLongLivedAKID). testAccessKeyID = "AKIAINTEGRATIONTESTR" testSecretAccessKey = "integration-test-root-secret" // nodeDiscoverSubject is the fan-out GatewayConfig.DiscoverActiveNodes // publishes to (spinifex/gateway/gateway.go). StartGateway stubs it with // a single-node reply so any call path that triggers node discovery // resolves immediately instead of riding out the fan-out's 500ms timeout. nodeDiscoverSubject = "spinifex.nodes.discover" // testECRAudience is the audience claim every harness-issued ECR JWT is // minted for and verified against — arbitrary but stable within this tier, // mirroring production's "ecr.." shape without depending on // GatewayConfig.InternalSuffix (unset here; the in-process harness talks to // the gateway's httptest address directly rather than a registry hostname). testECRAudience = "ecr.integration-test" // testECRServicesDomain completes the AWS-shaped registry host ECR // responses advertise; no test resolves it. testECRServicesDomain = "integration.test" ) // Gateway is a running in-process instance of the real AWS gateway router, // backed by the package's shared embedded NATS JetStream server and real // IAM/STS services. Obtained via StartGateway; the httptest server and any // StubSubject responders registered on it are torn down automatically via // t.Cleanup. NATSConn is NOT closed at test end — see the TestMain doc // comment in main_test.go for why — TestMain closes every test's connection // together once the whole package's tests have finished. type Gateway struct { // Server hosts gateway.SetupRoutes() on an ephemeral httptest port. Server *httptest.Server // NATSConn is this test's isolated JetStream connection into the shared // server (see TestMain), which the gateway, IAMService, and STSService // all share. Tests use it directly with StubSubject to fake daemon-side // responders. NATSConn *nats.Conn // AccountID is the account the seeded root credentials belong to // (awsidentifiers.GlobalAccountID). AccountID string // Config is the GatewayConfig SetupRoutes was built from, exposed so // tests can inspect or further wire fields StartGateway did not set. Config *gateway.GatewayConfig } // Option customises a Gateway's GatewayConfig before SetupRoutes builds the // router. Applied after StartGateway's own defaults, so an Option can // override any of them (e.g. a different Region/AZ, or a nil IAMService to // exercise the pre-IAM-compatibility bypass paths). type Option func(*gateway.GatewayConfig) // StartGateway boots the real gateway router over the package's shared, // embedded NATS JetStream server — connected via a fresh, isolated NATS // account so this test's state cannot collide with any other's — and serves // it from an httptest.Server on an ephemeral port. It wires real // IAMService/STSService implementations (not a mock) and // seeds a root access key via SeedBootstrap, so SigV4 requests authenticate // and authorize exactly as they would against a live environment — root // bypasses IAM policy evaluation, matching gateway.evaluatePrincipalPolicy. // // ExpectedNodes is pinned to 1: natsmsg.Gather waits the FULL timeout on every // call when ExpectedNodes is 0, which would make every stubbed NATS // round-trip pathologically slow. // // Callers stub whichever NATS subjects their test needs via StubSubject; // every other control-plane path (auth, routing, XML/JSON marshaling, IAM // policy evaluation) runs for real. func StartGateway(t *testing.T, opts ...Option) *Gateway { return startGateway(t, suiteConformance, opts...) } func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option) *Gateway { t.Helper() // Every test connects into its own dynamically provisioned NATS account // on the one server this package's TestMain shares across the whole // binary, rather than booting a private embedded server per test — see // the TestMain doc comment in main_test.go for why, and how state stays // isolated regardless. nc, js, err := sharedNATSHarness.connectIsolated(accountNameFor(t)) require.NoError(t, err) masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey) require.NoError(t, err) stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, handlers_eks.NewOIDCJWKSReader(js)) require.NoError(t, err) encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey) require.NoError(t, err) require.NoError(t, iamSvc.SeedBootstrap(&handlers_iam.BootstrapData{ AccessKeyID: testAccessKeyID, EncryptedSecret: encryptedSecret, AccountID: awsidentifiers.GlobalAccountID, })) // ECR auth bridge signing key: reuses the IAM master key to encrypt the // signing key at rest in the same embedded JetStream KV, matching // production's awsgw-keys wiring (runtime/roles/awsgw/awsgw.go). signingKey, verifyKeys, err := ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey) require.NoError(t, err) bedrockAccess := gateway_bedrock.NewModelAccessStore(js) // ECR's control plane is composed as the awsgw role composes it, sharing // the OCI registry, meta store and token issuer the /v2 data plane uses. ecrMeta := ecrdomain.NewNATSMetaStore(nc) ecrRegistry := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), ecrMeta, awsidentifiers.GlobalAccountID) ecrIssuer := ecrauth.NewIssuer(signingKey, testECRAudience) ecrEndpoint := awsapi.RepositoryEndpoint{Region: testRegion, ServicesDomain: testECRServicesDomain} services := awsdispatch.NewBuilder() require.NoError(t, services.Register(awsapi.NewRegistration(awsapi.Deps{ Registry: awsapi.NewRegistryActionService(ecrRegistry, ecrRegistry, ecrRegistry, ecrRegistry), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(ecrMeta, ecrRegistry), Repository: awsapi.NewRepositoryActionService(ecrMeta, ecrEndpoint), AuthorizationToken: awsapi.NewAuthorizationTokenActionService(ecrIssuer, ecrEndpoint), NATS: nc, }))) require.NoError(t, services.Register(acmawsapi.NewRegistration(acmawsapi.Deps{NATS: nc}))) cfg := &gateway.GatewayConfig{ DisableLogging: true, NATSConn: nc, Services: services.Build(), ExpectedNodes: 1, Region: testRegion, AZ: testAZ, IAMService: iamSvc, STSService: stsSvc, // ECRRegistry serves the OCI Distribution v2 (/v2/*) data plane; its // Meta store is a NATS client, so a test that pushes/pulls or manages // repositories must additionally call StartECRDaemonLite to subscribe a // real MetaServiceImpl or every ECR request will time out with no // responder. Blob/manifest bytes are memory-backed: no predastore. ECRRegistry: ecrRegistry, ECRTokenIssuer: ecrIssuer, ECRTokenVerifier: ecrauth.NewVerifier(verifyKeys, testECRAudience), // Ochre model access is deny-by-default, so without a grant store every // bedrock route refuses. Tests sign as the system account, which the // store exempts from grants exactly as it does in production, so the // catalog is visible here without provisioning grants per test; a test // signing as a tenant account would have to grant first. BedrockAccess: bedrockAccess, BedrockAccessAdmin: bedrockAccess, } for _, opt := range opts { opt(cfg) } handler := cfg.SetupRoutes() if collector != nil { handler = conformanceMiddleware(handler, collector) } srv := httptest.NewServer(handler) t.Cleanup(srv.Close) gw := &Gateway{ Server: srv, NATSConn: nc, AccountID: awsidentifiers.GlobalAccountID, Config: cfg, } nodeReply, err := json.Marshal(clusterv1.NodeDiscoverResponse{Node: "integration-test-node"}) require.NoError(t, err) gw.StubSubject(t, nodeDiscoverSubject, nodeReply) return gw } // EC2Client returns an aws-sdk-go v1 EC2 client pointed at the gateway's // httptest server, signed with the seeded root credentials over plain HTTP // (the httptest server is not TLS). func (gw *Gateway) EC2Client(t *testing.T) *ec2.EC2 { t.Helper() return ec2.New(gw.session(t)) } // IAMClient returns an aws-sdk-go v1 IAM client pointed at the gateway's // httptest server, signed with the seeded root credentials. func (gw *Gateway) IAMClient(t *testing.T) *iam.IAM { t.Helper() return iam.New(gw.session(t)) } // STSClient returns an aws-sdk-go v1 STS client pointed at the gateway's // httptest server, signed with the seeded root credentials. func (gw *Gateway) STSClient(t *testing.T) *sts.STS { t.Helper() return sts.New(gw.session(t)) } // ECRClient returns an aws-sdk-go v1 ECR client pointed at the gateway's // httptest server, signed with the seeded root credentials. Repository and // image operations dispatch to ECRRegistry.Meta over NATS, so a test needs // StartECRDaemonLite running before calling anything beyond // GetAuthorizationToken. func (gw *Gateway) ECRClient(t *testing.T) *ecr.ECR { t.Helper() return ecr.New(gw.session(t)) } // BedrockClient returns an aws-sdk-go v1 Bedrock (control-plane) client // pointed at the gateway's httptest server, signed with the seeded root // credentials. ListFoundationModels/GetFoundationModel are pure gateway-side // catalog logic (gateway/bedrock/catalog.go) with no NATS/daemon hop, so no // DaemonLite wiring is needed to exercise them. func (gw *Gateway) BedrockClient(t *testing.T) *bedrock.Bedrock { t.Helper() return bedrock.New(gw.session(t)) } // PrincipalClients bundles the SDK service handles an authz test needs to act // as a principal other than the harness's seeded root user: an IAM user // created within the test, or a role assumed via STS. Populated by // ClientsWithCreds / ClientsWithSessionCreds. type PrincipalClients struct { EC2 *ec2.EC2 IAM *iam.IAM STS *sts.STS ECR *ecr.ECR } // ClientsWithCreds returns PrincipalClients signed with a static access // key/secret and no session token — the long-lived-access-key path (an AKIA // key from IAM CreateAccessKey), as opposed to the seeded-root session // EC2Client/IAMClient/STSClient use. func (gw *Gateway) ClientsWithCreds(t *testing.T, accessKeyID, secretAccessKey string) *PrincipalClients { t.Helper() return gw.principalClients(t, accessKeyID, secretAccessKey, "") } // ClientsWithSessionCreds returns PrincipalClients signed with STS temporary // credentials — the assumed-role path (an ASIA key, secret, and session // token from STS AssumeRole). func (gw *Gateway) ClientsWithSessionCreds(t *testing.T, accessKeyID, secretAccessKey, sessionToken string) *PrincipalClients { t.Helper() return gw.principalClients(t, accessKeyID, secretAccessKey, sessionToken) } // principalClients builds the session backing both ClientsWithCreds and // ClientsWithSessionCreds. Passing sessionToken == "" here — the // ClientsWithCreds path — must not add an X-Amz-Security-Token header at // all: aws-sdk-go's v4 signer only sets the header when SessionToken is // non-empty, so a single code path correctly serves both the long-lived-key // and assumed-role principals without a conditional. func (gw *Gateway) principalClients(t *testing.T, accessKeyID, secretAccessKey, sessionToken string) *PrincipalClients { t.Helper() sess, err := awssession.NewSession(&aws.Config{ Region: aws.String(testRegion), Endpoint: aws.String(gw.Server.URL), Credentials: awscreds.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), DisableSSL: aws.Bool(true), // Error-path tests assert on the first response; the default retry // loop would mask a deterministic 4xx behind minutes of backoff. MaxRetries: aws.Int(0), }) require.NoError(t, err) return &PrincipalClients{ EC2: ec2.New(sess), IAM: iam.New(sess), STS: sts.New(sess), ECR: ecr.New(sess), } } // session builds the shared aws-sdk-go v1 Session every per-service client // accessor (EC2Client, ...) derives from, pointed at this Gateway's httptest // server and signed with the seeded root credentials. func (gw *Gateway) session(t *testing.T) *awssession.Session { t.Helper() sess, err := awssession.NewSession(&aws.Config{ Region: aws.String(testRegion), Endpoint: aws.String(gw.Server.URL), Credentials: awscreds.NewStaticCredentials(testAccessKeyID, testSecretAccessKey, ""), DisableSSL: aws.Bool(true), // Error-path tests assert on the first response; the default retry // loop would mask a deterministic 4xx behind minutes of backoff. MaxRetries: aws.Int(0), }) require.NoError(t, err) return sess }