test: add in-process integration test tier

Roughly half of the behavioural tests under tests/e2e/ assert control-plane API shapes, authz and error codes that need no live environment, yet they can only run against a tofu-provisioned environment on a scarce self-hosted runner. This adds the tier they can move down to. The real gateway router runs in-process: SetupRoutes() is a pure function of GatewayConfig, so it serves from an httptest server over embedded NATS JetStream with real IAM/STS services and genuine SigV4 verification. Only the daemon side of the wire is faked -- the ec2.*/ebs.* NATS subjects a live node would answer are stubbed per test. Nothing is provisioned and no fixed ports are bound, so the tier runs concurrently on a shared build host and is safe as a CI front gate ahead of the live suites. It is build-tagged and kept out of preflight, which stays unit-only.
2026-10-20 19:25:52 +00:00 · 2026-07-20 10:41:51 +10:00
parent ffc34b735b
commit 4c3da1e346
4 changed files with 355 additions and 1 deletions
@@ -133,6 +133,16 @@ test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
 	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)

+# In-process integration tier: the real gateway router against embedded NATS
+# JetStream, with only the daemon-side NATS subjects stubbed. Build-tagged
+# `integration` so it's skipped by the default `go test ./spinifex/...` and by
+# `test-cover`/`test-race`. Nothing provisioned — no tofu, no docker, no
+# Spinifex daemons — so it's safe to run as a fast CI front gate. Deliberately
+# NOT part of `preflight`, which stays unit-only.
+test-integration:
+	@echo -e "\n....Running in-process integration tests...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
 # of services/suites/fixtures + on-disk path existence. Subject content
 # vs source is enforced separately in Bead 5 drift lint.
@@ -297,7 +307,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/

-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness test-integration manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
@@ -0,0 +1,105 @@
+//go:build integration
+
+package integration
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubEmptyInstanceBuckets stands in for the DescribeStoppedInstances and
+// DescribeTerminatedInstances daemon responders that DescribeInstances also
+// queries. Stubbing them keeps the fan-out deterministic and fast regardless
+// of the embedded NATS server's no-responders behavior, instead of relying on
+// a 3s RequestMsg timeout when nothing answers.
+func stubEmptyInstanceBuckets(t *testing.T, gw *Gateway) {
+	t.Helper()
+	empty := mustMarshal(t, &ec2.DescribeInstancesOutput{})
+	gw.StubSubject(t, "ec2.DescribeStoppedInstances", empty)
+	gw.StubSubject(t, "ec2.DescribeTerminatedInstances", empty)
+}
+
+func mustMarshal(t *testing.T, v any) []byte {
+	t.Helper()
+	b, err := json.Marshal(v)
+	require.NoError(t, err)
+	return b
+}
+
+// TestDescribeInstances_HappyPath proves the harness end to end: the real
+// gateway router, real SigV4 auth against a seeded root IAM user, and a
+// stubbed ec2.DescribeInstances daemon responder wired together in one
+// process with nothing provisioned. A real spinifex daemon would answer
+// "ec2.DescribeInstances" with exactly this JSON-encoded ec2.DescribeInstancesOutput.
+func TestDescribeInstances_HappyPath(t *testing.T) {
+	gw := StartGateway(t)
+	stubEmptyInstanceBuckets(t, gw)
+
+	want := &ec2.DescribeInstancesOutput{
+		Reservations: []*ec2.Reservation{{
+			ReservationId: aws.String("r-abc123"),
+			Instances: []*ec2.Instance{{
+				InstanceId:   aws.String("i-abc123"),
+				InstanceType: aws.String("t2.micro"),
+				State:        &ec2.InstanceState{Name: aws.String("running")},
+			}},
+		}},
+	}
+	gw.StubSubject(t, "ec2.DescribeInstances", mustMarshal(t, want))
+
+	out, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{})
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+	assert.Equal(t, "i-abc123", aws.StringValue(out.Reservations[0].Instances[0].InstanceId))
+	assert.Equal(t, "running", aws.StringValue(out.Reservations[0].Instances[0].State.Name))
+}
+
+// TestDescribeInstances_ErrorPath drives a stubbed daemon error envelope
+// through the same real gateway and asserts the AWS error code the SDK
+// surfaces matches what a live environment would return for the same
+// daemon-side failure — proving the tier can also cover error-code paths, not
+// just happy-path shape assertions.
+func TestDescribeInstances_ErrorPath(t *testing.T) {
+	gw := StartGateway(t)
+	stubEmptyInstanceBuckets(t, gw)
+
+	gw.StubSubject(t, "ec2.DescribeInstances", utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound))
+
+	_, err := gw.EC2Client(t).DescribeInstances(&ec2.DescribeInstancesInput{})
+	require.Error(t, err)
+
+	var awsErr awserr.Error
+	require.ErrorAs(t, err, &awsErr, "expected awserr.Error, got %T: %v", err, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, awsErr.Code())
+}
+
+// TestDescribeInstances_UnauthenticatedRejected proves the harness exercises
+// real SigV4 authentication, not just routing: a request signed with a
+// nonexistent access key never reaches the stubbed NATS responder and is
+// rejected before dispatch.
+func TestDescribeInstances_UnauthenticatedRejected(t *testing.T) {
+	gw := StartGateway(t)
+	stubEmptyInstanceBuckets(t, gw)
+	// Deliberately not stubbed: if auth failed to reject the request first,
+	// DescribeInstances would hang waiting on ExpectedNodes=1 that never answers.
+
+	sess := gw.session(t)
+	sess.Config.Credentials = awscreds.NewStaticCredentials("AKIADOESNOTEXIST0000", "wrong-secret", "")
+
+	_, err := ec2.New(sess).DescribeInstances(&ec2.DescribeInstancesInput{})
+	require.Error(t, err)
+
+	var awsErr awserr.Error
+	require.ErrorAs(t, err, &awsErr, "expected awserr.Error, got %T: %v", err, err)
+	assert.Equal(t, awserrors.ErrorInvalidClientTokenId, awsErr.Code())
+}
@@ -0,0 +1,164 @@
+//go:build integration
+
+// Package integration is the in-process integration test tier for spinifex's
+// AWS-compatible control plane. Tests here start the REAL gateway router
+// (gateway.SetupRoutes) against an embedded, ephemeral NATS JetStream
+// instance, with real IAM/STS services wired in for authentic SigV4 authz —
+// the only thing faked is the daemon side of the wire: the NATS subjects a
+// live spinifex daemon would answer (ec2.*, ebs.*, ...) are stubbed per-test
+// via StubSubject.
+//
+// Nothing is provisioned: no tofu, no docker, no Spinifex daemons, no fixed
+// ports. Every dependency starts ephemeral and is torn down via t.Cleanup, so
+// this tier is safe to run as a fast CI front gate ahead of the live E2E
+// suites, and safe to run concurrently on a shared build host.
+//
+// This package is gated behind the "integration" build tag (see the
+// `test-integration` Makefile target) so it is never swept by the default
+// `go test ./spinifex/...`, `make test-cover`, or `make test-race`.
+package integration
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
+	awssession "github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// testRegion/testAZ pin the gateway's SigV4 credential scope and AZ.
+	// Every SDK client built via Gateway.EC2Client signs against testRegion,
+	// so requests verify against the same scope the gateway expects.
+	testRegion = "us-east-1"
+	testAZ     = "us-east-1a"
+
+	// testAccessKeyID/testSecretAccessKey are the root credentials seeded
+	// into the real IAM service for every harness-started gateway. AKIA
+	// prefix routes SigV4 auth through the long-lived-key path (see
+	// spinifex/gateway/auth.go resolveLongLivedAKID).
+	testAccessKeyID     = "AKIAINTEGRATIONTESTR"
+	testSecretAccessKey = "integration-test-root-secret"
+)
+
+// Gateway is a running in-process instance of the real AWS gateway router,
+// backed by embedded NATS JetStream and real IAM/STS services. Obtained via
+// StartGateway; every resource it owns is torn down automatically via
+// t.Cleanup, including the NATS server, the httptest server, and any
+// StubSubject responders registered on it.
+type Gateway struct {
+	// Server hosts gateway.SetupRoutes() on an ephemeral httptest port.
+	Server *httptest.Server
+	// NATSConn is the embedded JetStream connection the gateway, IAMService,
+	// and STSService all share. Tests use it directly with StubSubject to
+	// fake daemon-side responders.
+	NATSConn *nats.Conn
+	// AccountID is the account the seeded root credentials belong to
+	// (utils.GlobalAccountID).
+	AccountID string
+	// Config is the GatewayConfig SetupRoutes was built from, exposed so
+	// tests can inspect or further wire fields StartGateway did not set.
+	Config *gateway.GatewayConfig
+}
+
+// Option customises a Gateway's GatewayConfig before SetupRoutes builds the
+// router. Applied after StartGateway's own defaults, so an Option can
+// override any of them (e.g. a different Region/AZ, or a nil IAMService to
+// exercise the pre-IAM-compatibility bypass paths).
+type Option func(*gateway.GatewayConfig)
+
+// StartGateway boots the real gateway router over an embedded, ephemeral NATS
+// JetStream instance and serves it from an httptest.Server on an ephemeral
+// port. It wires real IAMService/STSService implementations (not a mock) and
+// seeds a root access key via SeedBootstrap, so SigV4 requests authenticate
+// and authorize exactly as they would against a live environment — root
+// bypasses IAM policy evaluation, matching gateway.evaluatePrincipalPolicy.
+//
+// ExpectedNodes is pinned to 1: utils.Gather waits the FULL timeout on every
+// call when ExpectedNodes is 0, which would make every stubbed NATS
+// round-trip pathologically slow.
+//
+// Callers stub whichever NATS subjects their test needs via StubSubject;
+// every other control-plane path (auth, routing, XML/JSON marshaling, IAM
+// policy evaluation) runs for real.
+func StartGateway(t *testing.T, opts ...Option) *Gateway {
+	t.Helper()
+
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+
+	iamSvc, err := handlers_iam.NewIAMServiceWithRetry(nc, masterKey, 1)
+	require.NoError(t, err)
+
+	stsSvc, err := handlers_sts.NewSTSServiceImpl(nc, iamSvc, masterKey, 1)
+	require.NoError(t, err)
+
+	encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey)
+	require.NoError(t, err)
+
+	require.NoError(t, iamSvc.SeedBootstrap(&handlers_iam.BootstrapData{
+		AccessKeyID:     testAccessKeyID,
+		EncryptedSecret: encryptedSecret,
+		AccountID:       utils.GlobalAccountID,
+	}))
+
+	cfg := &gateway.GatewayConfig{
+		DisableLogging: true,
+		NATSConn:       nc,
+		ExpectedNodes:  1,
+		Region:         testRegion,
+		AZ:             testAZ,
+		IAMService:     iamSvc,
+		STSService:     stsSvc,
+	}
+	for _, opt := range opts {
+		opt(cfg)
+	}
+
+	srv := httptest.NewServer(cfg.SetupRoutes())
+	t.Cleanup(srv.Close)
+
+	return &Gateway{
+		Server:    srv,
+		NATSConn:  nc,
+		AccountID: utils.GlobalAccountID,
+		Config:    cfg,
+	}
+}
+
+// EC2Client returns an aws-sdk-go v1 EC2 client pointed at the gateway's
+// httptest server, signed with the seeded root credentials over plain HTTP
+// (the httptest server is not TLS).
+func (gw *Gateway) EC2Client(t *testing.T) *ec2.EC2 {
+	t.Helper()
+	return ec2.New(gw.session(t))
+}
+
+// session builds the shared aws-sdk-go v1 Session every per-service client
+// accessor (EC2Client, ...) derives from, pointed at this Gateway's httptest
+// server and signed with the seeded root credentials.
+func (gw *Gateway) session(t *testing.T) *awssession.Session {
+	t.Helper()
+	sess, err := awssession.NewSession(&aws.Config{
+		Region:      aws.String(testRegion),
+		Endpoint:    aws.String(gw.Server.URL),
+		Credentials: awscreds.NewStaticCredentials(testAccessKeyID, testSecretAccessKey, ""),
+		DisableSSL:  aws.Bool(true),
+		// Error-path tests assert on the first response; the default retry
+		// loop would mask a deterministic 4xx behind minutes of backoff.
+		MaxRetries: aws.Int(0),
+	})
+	require.NoError(t, err)
+	return sess
+}
@@ -0,0 +1,75 @@
+//go:build integration
+
+package integration
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+// subjectStubMu guards subjectStubRegistry. subjectStubRegistry is keyed by
+// *nats.Conn (each Gateway owns its own embedded connection, so unrelated
+// tests — even running concurrently — never collide) then by NATS subject,
+// holding the payload the stub responder for that subject currently returns.
+var (
+	subjectStubMu       sync.RWMutex
+	subjectStubRegistry = map[*nats.Conn]map[string][]byte{}
+)
+
+// StubSubject registers a responder for subject on the gateway's NATS
+// connection that replies to every request with payload, standing in for the
+// daemon-side subscriber a live spinifex node would run. It satisfies both
+// wire patterns the control plane uses: utils.Gather's scatter-gather (a
+// fresh inbox per call, subject.Reply set) and a plain nc.RequestMsg — both
+// publish with Reply set to an inbox, so a direct subscription on subject
+// that calls msg.Respond answers either.
+//
+// To make a future test exercise a different control-plane path, call
+// StubSubject once per NATS subject the handler under test publishes to (grep
+// spinifex/gateway/**/*.go for the literal subject string passed to
+// utils.Gather or natsConn.RequestMsg), then use SetSubjectReply to vary the
+// response per test case — e.g. an error envelope from utils.GenerateErrorPayload
+// for a validation-error test, or a populated output struct for a happy path.
+func (gw *Gateway) StubSubject(t *testing.T, subject string, payload []byte) {
+	t.Helper()
+	gw.SetSubjectReply(subject, payload)
+
+	sub, err := gw.NATSConn.Subscribe(subject, func(m *nats.Msg) {
+		if m.Reply == "" {
+			return
+		}
+		subjectStubMu.RLock()
+		resp := subjectStubRegistry[gw.NATSConn][subject]
+		subjectStubMu.RUnlock()
+		_ = m.Respond(resp)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	// Drop only this subject's entry: StubSubject is per-subject, so removing
+	// the whole connection's map here would wipe sibling stubs registered on
+	// the same Gateway.
+	t.Cleanup(func() {
+		subjectStubMu.Lock()
+		delete(subjectStubRegistry[gw.NATSConn], subject)
+		if len(subjectStubRegistry[gw.NATSConn]) == 0 {
+			delete(subjectStubRegistry, gw.NATSConn)
+		}
+		subjectStubMu.Unlock()
+	})
+}
+
+// SetSubjectReply overrides the reply payload for a subject already stubbed
+// via StubSubject on gw, updating the existing subscriber's response in
+// place. Safe to call mid-test — it never adds a second subscriber, which
+// would otherwise race the first for who answers the next request.
+func (gw *Gateway) SetSubjectReply(subject string, payload []byte) {
+	subjectStubMu.Lock()
+	defer subjectStubMu.Unlock()
+	if subjectStubRegistry[gw.NATSConn] == nil {
+		subjectStubRegistry[gw.NATSConn] = make(map[string][]byte)
+	}
+	subjectStubRegistry[gw.NATSConn][subject] = payload
+}