From 4c3da1e346e1b9489f83c9e672528de67d97f34d Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Mon, 20 Jul 2026 10:41:51 +1000 Subject: [PATCH] test: add in-process integration test tier Roughly half of the behavioural tests under tests/e2e/ assert control-plane API shapes, authz and error codes that need no live environment, yet they can only run against a tofu-provisioned environment on a scarce self-hosted runner. This adds the tier they can move down to. The real gateway router runs in-process: SetupRoutes() is a pure function of GatewayConfig, so it serves from an httptest server over embedded NATS JetStream with real IAM/STS services and genuine SigV4 verification. Only the daemon side of the wire is faked -- the ec2.*/ebs.* NATS subjects a live node would answer are stubbed per test. Nothing is provisioned and no fixed ports are bound, so the tier runs concurrently on a shared build host and is safe as a CI front gate ahead of the live suites. It is build-tagged and kept out of preflight, which stays unit-only. --- Makefile | 12 +- tests/integration/describe_instances_test.go | 105 ++++++++++++ tests/integration/harness.go | 164 +++++++++++++++++++ tests/integration/nats_stub.go | 75 +++++++++ 4 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 tests/integration/describe_instances_test.go create mode 100644 tests/integration/harness.go create mode 100644 tests/integration/nats_stub.go diff --git a/Makefile b/Makefile index a7529090d..800c4abf8 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,16 @@ test-harness: @echo -e "\n....Running e2e harness unit tests...." $(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ) +# In-process integration tier: the real gateway router against embedded NATS +# JetStream, with only the daemon-side NATS subjects stubbed. Build-tagged +# `integration` so it's skipped by the default `go test ./spinifex/...` and by +# `test-cover`/`test-race`. Nothing provisioned — no tofu, no docker, no +# Spinifex daemons — so it's safe to run as a fast CI front gate. Deliberately +# NOT part of `preflight`, which stays unit-only. +test-integration: + @echo -e "\n....Running in-process integration tests...." + $(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ) + # Validate docs/service-interfaces.yaml. Schema check + cross-reference # of services/suites/fixtures + on-disk path existence. Subject content # vs source is enforced separately in Bead 5 drift lint. @@ -297,7 +307,7 @@ distro-arm64: distro-clean: rm -rf dist/ -.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \ +.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness test-integration manifest-check manifest-lint manifest-lint-update \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ diff --git a/tests/integration/describe_instances_test.go b/tests/integration/describe_instances_test.go new file mode 100644 index 000000000..241b6d2e5 --- /dev/null +++ b/tests/integration/describe_instances_test.go @@ -0,0 +1,105 @@ +//go:build integration + +package integration + +import ( + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + awscreds "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stubEmptyInstanceBuckets stands in for the DescribeStoppedInstances and +// DescribeTerminatedInstances daemon responders that DescribeInstances also +// queries. Stubbing them keeps the fan-out deterministic and fast regardless +// of the embedded NATS server's no-responders behavior, instead of relying on +// a 3s RequestMsg timeout when nothing answers. +func stubEmptyInstanceBuckets(t *testing.T, gw *Gateway) { + t.Helper() + empty := mustMarshal(t, &ec2.DescribeInstancesOutput{}) + gw.StubSubject(t, "ec2.DescribeStoppedInstances", empty) + gw.StubSubject(t, "ec2.DescribeTerminatedInstances", empty) +} + +func mustMarshal(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return b +} + +// TestDescribeInstances_HappyPath proves the harness end to end: the real +// gateway router, real SigV4 auth against a seeded root IAM user, and a +// stubbed ec2.DescribeInstances daemon responder wired together in one +// process with nothing provisioned. A real spinifex daemon would answer +// "ec2.DescribeInstances" with exactly this JSON-encoded ec2.DescribeInstancesOutput. +func TestDescribeInstances_HappyPath(t *testing.T) { + gw := StartGateway(t) + stubEmptyInstanceBuckets(t, gw) + + want := &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{ + ReservationId: aws.String("r-abc123"), + Instances: []*ec2.Instance{{ + InstanceId: aws.String("i-abc123"), + InstanceType: aws.String("t2.micro"), + State: &ec2.InstanceState{Name: aws.String("running")}, + }}, + }}, + } + gw.StubSubject(t, "ec2.DescribeInstances", mustMarshal(t, want)) + + out, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{}) + require.NoError(t, err) + require.Len(t, out.Reservations, 1) + require.Len(t, out.Reservations[0].Instances, 1) + assert.Equal(t, "i-abc123", aws.StringValue(out.Reservations[0].Instances[0].InstanceId)) + assert.Equal(t, "running", aws.StringValue(out.Reservations[0].Instances[0].State.Name)) +} + +// TestDescribeInstances_ErrorPath drives a stubbed daemon error envelope +// through the same real gateway and asserts the AWS error code the SDK +// surfaces matches what a live environment would return for the same +// daemon-side failure — proving the tier can also cover error-code paths, not +// just happy-path shape assertions. +func TestDescribeInstances_ErrorPath(t *testing.T) { + gw := StartGateway(t) + stubEmptyInstanceBuckets(t, gw) + + gw.StubSubject(t, "ec2.DescribeInstances", utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound)) + + _, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{}) + require.Error(t, err) + + var awsErr awserr.Error + require.ErrorAs(t, err, &awsErr, "expected awserr.Error, got %T: %v", err, err) + assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, awsErr.Code()) +} + +// TestDescribeInstances_UnauthenticatedRejected proves the harness exercises +// real SigV4 authentication, not just routing: a request signed with a +// nonexistent access key never reaches the stubbed NATS responder and is +// rejected before dispatch. +func TestDescribeInstances_UnauthenticatedRejected(t *testing.T) { + gw := StartGateway(t) + stubEmptyInstanceBuckets(t, gw) + // Deliberately not stubbed: if auth failed to reject the request first, + // DescribeInstances would hang waiting on ExpectedNodes=1 that never answers. + + sess := gw.session(t) + sess.Config.Credentials = awscreds.NewStaticCredentials("AKIADOESNOTEXIST0000", "wrong-secret", "") + + _, err := ec2.New(sess).DescribeInstances(&ec2.DescribeInstancesInput{}) + require.Error(t, err) + + var awsErr awserr.Error + require.ErrorAs(t, err, &awsErr, "expected awserr.Error, got %T: %v", err, err) + assert.Equal(t, awserrors.ErrorInvalidClientTokenId, awsErr.Code()) +} diff --git a/tests/integration/harness.go b/tests/integration/harness.go new file mode 100644 index 000000000..776810d54 --- /dev/null +++ b/tests/integration/harness.go @@ -0,0 +1,164 @@ +//go:build integration + +// Package integration is the in-process integration test tier for spinifex's +// AWS-compatible control plane. Tests here start the REAL gateway router +// (gateway.SetupRoutes) against an embedded, ephemeral NATS JetStream +// instance, with real IAM/STS services wired in for authentic SigV4 authz — +// the only thing faked is the daemon side of the wire: the NATS subjects a +// live spinifex daemon would answer (ec2.*, ebs.*, ...) are stubbed per-test +// via StubSubject. +// +// Nothing is provisioned: no tofu, no docker, no Spinifex daemons, no fixed +// ports. Every dependency starts ephemeral and is torn down via t.Cleanup, so +// this tier is safe to run as a fast CI front gate ahead of the live E2E +// suites, and safe to run concurrently on a shared build host. +// +// This package is gated behind the "integration" build tag (see the +// `test-integration` Makefile target) so it is never swept by the default +// `go test ./spinifex/...`, `make test-cover`, or `make test-race`. +package integration + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go/aws" + awscreds "github.com/aws/aws-sdk-go/aws/credentials" + awssession "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/gateway" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/require" +) + +const ( + // testRegion/testAZ pin the gateway's SigV4 credential scope and AZ. + // Every SDK client built via Gateway.EC2Client signs against testRegion, + // so requests verify against the same scope the gateway expects. + testRegion = "us-east-1" + testAZ = "us-east-1a" + + // testAccessKeyID/testSecretAccessKey are the root credentials seeded + // into the real IAM service for every harness-started gateway. AKIA + // prefix routes SigV4 auth through the long-lived-key path (see + // spinifex/gateway/auth.go resolveLongLivedAKID). + testAccessKeyID = "AKIAINTEGRATIONTESTR" + testSecretAccessKey = "integration-test-root-secret" +) + +// Gateway is a running in-process instance of the real AWS gateway router, +// backed by embedded NATS JetStream and real IAM/STS services. Obtained via +// StartGateway; every resource it owns is torn down automatically via +// t.Cleanup, including the NATS server, the httptest server, and any +// StubSubject responders registered on it. +type Gateway struct { + // Server hosts gateway.SetupRoutes() on an ephemeral httptest port. + Server *httptest.Server + // NATSConn is the embedded JetStream connection the gateway, IAMService, + // and STSService all share. Tests use it directly with StubSubject to + // fake daemon-side responders. + NATSConn *nats.Conn + // AccountID is the account the seeded root credentials belong to + // (utils.GlobalAccountID). + AccountID string + // Config is the GatewayConfig SetupRoutes was built from, exposed so + // tests can inspect or further wire fields StartGateway did not set. + Config *gateway.GatewayConfig +} + +// Option customises a Gateway's GatewayConfig before SetupRoutes builds the +// router. Applied after StartGateway's own defaults, so an Option can +// override any of them (e.g. a different Region/AZ, or a nil IAMService to +// exercise the pre-IAM-compatibility bypass paths). +type Option func(*gateway.GatewayConfig) + +// StartGateway boots the real gateway router over an embedded, ephemeral NATS +// JetStream instance and serves it from an httptest.Server on an ephemeral +// port. It wires real IAMService/STSService implementations (not a mock) and +// seeds a root access key via SeedBootstrap, so SigV4 requests authenticate +// and authorize exactly as they would against a live environment — root +// bypasses IAM policy evaluation, matching gateway.evaluatePrincipalPolicy. +// +// ExpectedNodes is pinned to 1: utils.Gather waits the FULL timeout on every +// call when ExpectedNodes is 0, which would make every stubbed NATS +// round-trip pathologically slow. +// +// Callers stub whichever NATS subjects their test needs via StubSubject; +// every other control-plane path (auth, routing, XML/JSON marshaling, IAM +// policy evaluation) runs for real. +func StartGateway(t *testing.T, opts ...Option) *Gateway { + t.Helper() + + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + iamSvc, err := handlers_iam.NewIAMServiceWithRetry(nc, masterKey, 1) + require.NoError(t, err) + + stsSvc, err := handlers_sts.NewSTSServiceImpl(nc, iamSvc, masterKey, 1) + require.NoError(t, err) + + encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey) + require.NoError(t, err) + + require.NoError(t, iamSvc.SeedBootstrap(&handlers_iam.BootstrapData{ + AccessKeyID: testAccessKeyID, + EncryptedSecret: encryptedSecret, + AccountID: utils.GlobalAccountID, + })) + + cfg := &gateway.GatewayConfig{ + DisableLogging: true, + NATSConn: nc, + ExpectedNodes: 1, + Region: testRegion, + AZ: testAZ, + IAMService: iamSvc, + STSService: stsSvc, + } + for _, opt := range opts { + opt(cfg) + } + + srv := httptest.NewServer(cfg.SetupRoutes()) + t.Cleanup(srv.Close) + + return &Gateway{ + Server: srv, + NATSConn: nc, + AccountID: utils.GlobalAccountID, + Config: cfg, + } +} + +// EC2Client returns an aws-sdk-go v1 EC2 client pointed at the gateway's +// httptest server, signed with the seeded root credentials over plain HTTP +// (the httptest server is not TLS). +func (gw *Gateway) EC2Client(t *testing.T) *ec2.EC2 { + t.Helper() + return ec2.New(gw.session(t)) +} + +// session builds the shared aws-sdk-go v1 Session every per-service client +// accessor (EC2Client, ...) derives from, pointed at this Gateway's httptest +// server and signed with the seeded root credentials. +func (gw *Gateway) session(t *testing.T) *awssession.Session { + t.Helper() + sess, err := awssession.NewSession(&aws.Config{ + Region: aws.String(testRegion), + Endpoint: aws.String(gw.Server.URL), + Credentials: awscreds.NewStaticCredentials(testAccessKeyID, testSecretAccessKey, ""), + DisableSSL: aws.Bool(true), + // Error-path tests assert on the first response; the default retry + // loop would mask a deterministic 4xx behind minutes of backoff. + MaxRetries: aws.Int(0), + }) + require.NoError(t, err) + return sess +} diff --git a/tests/integration/nats_stub.go b/tests/integration/nats_stub.go new file mode 100644 index 000000000..b6abbea5b --- /dev/null +++ b/tests/integration/nats_stub.go @@ -0,0 +1,75 @@ +//go:build integration + +package integration + +import ( + "sync" + "testing" + + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/require" +) + +// subjectStubMu guards subjectStubRegistry. subjectStubRegistry is keyed by +// *nats.Conn (each Gateway owns its own embedded connection, so unrelated +// tests — even running concurrently — never collide) then by NATS subject, +// holding the payload the stub responder for that subject currently returns. +var ( + subjectStubMu sync.RWMutex + subjectStubRegistry = map[*nats.Conn]map[string][]byte{} +) + +// StubSubject registers a responder for subject on the gateway's NATS +// connection that replies to every request with payload, standing in for the +// daemon-side subscriber a live spinifex node would run. It satisfies both +// wire patterns the control plane uses: utils.Gather's scatter-gather (a +// fresh inbox per call, subject.Reply set) and a plain nc.RequestMsg — both +// publish with Reply set to an inbox, so a direct subscription on subject +// that calls msg.Respond answers either. +// +// To make a future test exercise a different control-plane path, call +// StubSubject once per NATS subject the handler under test publishes to (grep +// spinifex/gateway/**/*.go for the literal subject string passed to +// utils.Gather or natsConn.RequestMsg), then use SetSubjectReply to vary the +// response per test case — e.g. an error envelope from utils.GenerateErrorPayload +// for a validation-error test, or a populated output struct for a happy path. +func (gw *Gateway) StubSubject(t *testing.T, subject string, payload []byte) { + t.Helper() + gw.SetSubjectReply(subject, payload) + + sub, err := gw.NATSConn.Subscribe(subject, func(m *nats.Msg) { + if m.Reply == "" { + return + } + subjectStubMu.RLock() + resp := subjectStubRegistry[gw.NATSConn][subject] + subjectStubMu.RUnlock() + _ = m.Respond(resp) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + // Drop only this subject's entry: StubSubject is per-subject, so removing + // the whole connection's map here would wipe sibling stubs registered on + // the same Gateway. + t.Cleanup(func() { + subjectStubMu.Lock() + delete(subjectStubRegistry[gw.NATSConn], subject) + if len(subjectStubRegistry[gw.NATSConn]) == 0 { + delete(subjectStubRegistry, gw.NATSConn) + } + subjectStubMu.Unlock() + }) +} + +// SetSubjectReply overrides the reply payload for a subject already stubbed +// via StubSubject on gw, updating the existing subscriber's response in +// place. Safe to call mid-test — it never adds a second subscriber, which +// would otherwise race the first for who answers the next request. +func (gw *Gateway) SetSubjectReply(subject string, payload []byte) { + subjectStubMu.Lock() + defer subjectStubMu.Unlock() + if subjectStubRegistry[gw.NATSConn] == nil { + subjectStubRegistry[gw.NATSConn] = make(map[string][]byte) + } + subjectStubRegistry[gw.NATSConn][subject] = payload +}