test: add an e2e suite proving quotas gate every capped resource

.github/scripts/suite-sets.sh

@@ -24,10 +24,14 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

cmd/spinifex/cmd/admin_account_quota_test.go

@@ -1,3 +1,4 @@
+//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from.
 package cmd
 
 import (

spinifex/gateway/admin_accountquota_test.go

@@ -1,3 +1,4 @@
+//test:in-package — drives the unexported admin handlers and the request identity they read from the context.
 package gateway
 
 import (

tests/e2e/Makefile

@@ -18,9 +18,9 @@ SUITE_TAG   ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
 # BIN_DIR/<suite>.test so CI can ship binaries (not source + go toolchain) to
 # the remote VM.
 BIN_DIR      ?= $(CURDIR)/_bin
-SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds
+SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota
 
-.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-rds e2e-all clean e2e-build
+.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-rds e2e-all clean e2e-build
 
 RUN_FLAG = $(if $(RUN),-run '$(RUN)',)
 
@@ -61,6 +61,7 @@ e2e-partialblock:  ; $(MAKE) e2e PKG=./partialblock/...
 # minutes, so a single wedged create eats most of a 30m package timeout before
 # the tests still running get to report.
 e2e-rds:        ; $(MAKE) e2e PKG=./rds/... TIMEOUT=45m
+e2e-quota:      ; $(MAKE) e2e PKG=./quota/...
 e2e-all:        ; $(MAKE) e2e
 
 e2e-build:

tests/e2e/harness/quota.go

@@ -0,0 +1,318 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+)
+
+// QuotaLimits is the [quota] baseline a suite installs on the cluster. Every
+// field is written, so a zero value denies that dimension outright rather than
+// inheriting anything — pass a complete set.
+type QuotaLimits struct {
+	VCPUs         int
+	VPCs          int
+	Subnets       int
+	EIPs          int
+	Volumes       int
+	VolumesGiB    int
+	RDSInstances  int
+	LoadBalancers int
+}
+
+// SandboxQuotaLimits mirrors the baseline shipped in the awsgw.toml template,
+// which is what a self-service account gets in production. A suite asserting
+// against these numbers is asserting against what a real tenant sees.
+func SandboxQuotaLimits() QuotaLimits {
+	return QuotaLimits{
+		VCPUs: 16, VPCs: 4, Subnets: 16, EIPs: 4,
+		Volumes: 16, VolumesGiB: 200, RDSInstances: 2, LoadBalancers: 2,
+	}
+}
+
+// QuotaConfigPath returns the awsgw config file holding the [quota] block.
+func QuotaConfigPath(env *Env) string {
+	return filepath.Join(env.ConfigDir, "awsgw", "awsgw.toml")
+}
+
+// awsgwUnit is the systemd unit read the [quota] block at startup.
+const awsgwUnit = "spinifex-awsgw"
+
+// EnableQuota turns per-account quotas on across the cluster and restarts every
+// gateway so the new block takes effect. The returned function puts the
+// original config back and restarts again — a suite that leaves quotas on caps
+// every suite that follows it on the same VM, so it must always run.
+//
+// Restore reports through an error rather than the *testing.T: it belongs in
+// TestMain, after every test has finished, where logging to a t would panic.
+func EnableQuota(t *testing.T, env *Env, limits QuotaLimits) func() error {
+	t.Helper()
+	if env.ConfigDir == "" {
+		t.Fatalf("EnableQuota: no config dir located; set SPINIFEX_CONFIG_DIR")
+	}
+	path := QuotaConfigPath(env)
+	hosts := quotaHosts(env)
+
+	saved := make(map[string]string, len(hosts))
+	for _, h := range hosts {
+		body, err := readNodeFile(h, path)
+		if err != nil {
+			t.Fatalf("EnableQuota: read %s on %s: %v", path, hostLabel(h), err)
+		}
+		saved[h] = body
+	}
+
+	for _, h := range hosts {
+		if err := writeNodeFile(h, path, withQuotaBlock(saved[h], limits)); err != nil {
+			t.Fatalf("EnableQuota: write %s on %s: %v", path, hostLabel(h), err)
+		}
+		if err := restartGateway(env, h); err != nil {
+			t.Fatalf("EnableQuota: restart gateway on %s: %v", hostLabel(h), err)
+		}
+	}
+
+	return func() error {
+		// Every host is attempted even after one fails: a cluster left with the
+		// suite's quota block on some nodes and not others is worse than either.
+		var failures []string
+		for _, h := range hosts {
+			if err := writeNodeFile(h, path, saved[h]); err != nil {
+				failures = append(failures, fmt.Sprintf("write %s on %s: %v", path, hostLabel(h), err))
+				continue
+			}
+			if err := restartGateway(env, h); err != nil {
+				failures = append(failures, fmt.Sprintf("restart gateway on %s: %v", hostLabel(h), err))
+			}
+		}
+		if len(failures) > 0 {
+			return fmt.Errorf("restore quota config: %s", strings.Join(failures, "; "))
+		}
+		return nil
+	}
+}
+
+// quotaHosts returns the hosts whose gateway config must be patched. The empty
+// string means this machine, which is where the single-node suites run.
+func quotaHosts(env *Env) []string {
+	if env.Mode != ModeMultinode {
+		return []string{""}
+	}
+	cluster, err := ClusterFromEnv()
+	if err != nil {
+		return []string{""}
+	}
+	hosts := make([]string, 0, len(cluster.Nodes))
+	for _, n := range cluster.Nodes {
+		hosts = append(hosts, n.Addr)
+	}
+	return hosts
+}
+
+func hostLabel(host string) string {
+	if host == "" {
+		return "localhost"
+	}
+	return host
+}
+
+// runNodeShell runs script on host, or on this machine when host is empty.
+func runNodeShell(host, script string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	if host != "" {
+		return NewPeerSSH().Run(ctx, host, script)
+	}
+	out, err := exec.CommandContext(ctx, "bash", "-c", script).CombinedOutput()
+	if err != nil {
+		return out, fmt.Errorf("local shell: %w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return out, nil
+}
+
+func readNodeFile(host, path string) (string, error) {
+	out, err := runNodeShell(host, "sudo -n cat -- "+ShellQuote(path))
+	return string(out), err
+}
+
+// writeNodeFile replaces path's contents. The body travels base64-encoded so a
+// single command string carries it intact over SSH, quoting and newlines alike.
+func writeNodeFile(host, path, body string) error {
+	script := fmt.Sprintf("printf %%s %s | base64 -d | sudo -n tee -- %s >/dev/null",
+		ShellQuote(base64.StdEncoding.EncodeToString([]byte(body))), ShellQuote(path))
+	_, err := runNodeShell(host, script)
+	return err
+}
+
+func restartGateway(env *Env, host string) error {
+	if _, err := runNodeShell(host, "sudo -n systemctl restart "+awsgwUnit); err != nil {
+		return err
+	}
+	addr := host
+	if addr == "" {
+		addr = gatewayDialHost(env)
+	}
+	return waitGatewayListening(addr, env.AWSGWPort, 90*time.Second)
+}
+
+// gatewayDialHost picks an address the local gateway answers on. ServiceIPs[0]
+// is the awsgw bind IP, so it is correct even where the listener is not on
+// loopback.
+func gatewayDialHost(env *Env) string {
+	if len(env.ServiceIPs) > 0 && env.ServiceIPs[0] != "" {
+		return env.ServiceIPs[0]
+	}
+	return "127.0.0.1"
+}
+
+// waitGatewayListening polls until the gateway completes a TLS handshake. The
+// certificate is not checked: this is a readiness probe, and the cert suite is
+// what asserts the chain.
+func waitGatewayListening(host string, port int, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	dialer := &net.Dialer{Timeout: 5 * time.Second}
+	target := net.JoinHostPort(host, strconv.Itoa(port))
+	var last error
+	for time.Now().Before(deadline) {
+		conn, err := tls.DialWithDialer(dialer, "tcp", target,
+			&tls.Config{InsecureSkipVerify: true}) //nolint:gosec // readiness probe, not a trust decision
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		last = err
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("gateway %s did not accept connections within %s: %w", target, timeout, last)
+}
+
+// withQuotaBlock returns cfg with its [quota] table replaced by limits. The old
+// table is dropped whole — editing keys in place would leave a stale key behind
+// whenever the block being replaced carries one the new block does not.
+func withQuotaBlock(cfg string, limits QuotaLimits) string {
+	quotaTable := regexp.MustCompile(`^\s*\[quota(\.[^\]]+)?\]\s*$`)
+
+	var kept []string
+	skipping := false
+	for _, line := range strings.Split(cfg, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(line), "[") {
+			skipping = quotaTable.MatchString(line)
+		}
+		if !skipping {
+			kept = append(kept, line)
+		}
+	}
+
+	return strings.TrimRight(strings.Join(kept, "\n"), "\n") + "\n\n" + fmt.Sprintf(
+		`[quota]
+enabled = true
+vcpus = %d
+vpcs = %d
+subnets = %d
+eips = %d
+volumes = %d
+volumes_gib = %d
+rds_instances = %d
+load_balancers = %d
+`,
+		limits.VCPUs, limits.VPCs, limits.Subnets, limits.EIPs,
+		limits.Volumes, limits.VolumesGiB, limits.RDSInstances, limits.LoadBalancers)
+}
+
+// spxAdminEnv adds the AWS profile the /admin surface signs with. The quota
+// subcommands reach the gateway over SigV4 rather than NATS, so they need a
+// credential where the account commands need none.
+func spxAdminEnv() []string {
+	profile := os.Getenv("AWS_PROFILE")
+	if profile == "" {
+		profile = "spinifex"
+	}
+	return append(spxChildEnv(), "AWS_PROFILE="+profile)
+}
+
+// SpxAdminQuotaSet runs `spx admin account quota set` for one account. args are
+// the dimension flags, e.g. "--vcpus", "8".
+func SpxAdminQuotaSet(t *testing.T, accountID string, args ...string) string {
+	t.Helper()
+	return spxAdminQuota(t, append([]string{"set", accountID}, args...)...)
+}
+
+// SpxAdminQuotaClear drops every override, returning the account to the
+// configured baseline.
+func SpxAdminQuotaClear(t *testing.T, accountID string) string {
+	t.Helper()
+	return spxAdminQuota(t, "set", accountID, "--clear")
+}
+
+// ClearAccountQuota is SpxAdminQuotaClear for teardown that outlives the test
+// that registered it, where reporting through a *testing.T would panic.
+func ClearAccountQuota(accountID string) error {
+	_, err := runSpxAdminQuota("set", accountID, "--clear")
+	return err
+}
+
+// quotaRow matches one line of the table `spx admin account quota get` prints:
+// dimension, limit, and the layer the limit came from.
+var quotaRow = regexp.MustCompile(`^([a-z_]+)\s+(\S+)\s+(config|override)\s*$`)
+
+// SpxAdminQuotaGet returns the limits in force for an account and, per
+// dimension, whether each was inherited from the config or set as an override.
+func SpxAdminQuotaGet(t *testing.T, accountID string) (limits map[string]int, source map[string]string) {
+	t.Helper()
+	out := spxAdminQuota(t, "get", accountID)
+
+	limits, source = map[string]int{}, map[string]string{}
+	for _, line := range strings.Split(out, "\n") {
+		m := quotaRow.FindStringSubmatch(strings.TrimSpace(line))
+		if m == nil {
+			continue
+		}
+		value := handlers_quota.Unlimited
+		if m[2] != "unlimited" {
+			parsed, err := strconv.Atoi(m[2])
+			if err != nil {
+				t.Fatalf("quota get %s: unparseable limit %q for %s", accountID, m[2], m[1])
+			}
+			value = parsed
+		}
+		limits[m[1]], source[m[1]] = value, m[3]
+	}
+	if len(limits) == 0 {
+		t.Fatalf("quota get %s: no dimensions parsed from:\n%s", accountID, out)
+	}
+	return limits, source
+}
+
+func spxAdminQuota(t *testing.T, args ...string) string {
+	t.Helper()
+	out, err := runSpxAdminQuota(args...)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	return out
+}
+
+func runSpxAdminQuota(args ...string) (string, error) {
+	full := append([]string{"admin", "account", "quota"}, args...)
+	cmd := exec.Command(SpxBin(), full...)
+	cmd.Env = spxAdminEnv()
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("spx %s failed: %w\noutput:\n%s",
+			strings.Join(full, " "), err, out)
+	}
+	return string(out), nil
+}

tests/e2e/quota/helpers_test.go

@@ -0,0 +1,245 @@
+//go:build e2e
+
+package quota
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+	"github.com/stretchr/testify/require"
+)
+
+// quotaExceeded is the code every enforced dimension refuses with. Asserting on
+// it rather than on "an error" is what separates a quota denial from a create
+// that failed for an unrelated reason and would otherwise read as a pass.
+const quotaExceeded = "ResourceLimitExceeded"
+
+// setLimit overrides one dimension on an account for the duration of the test
+// and clears every override afterwards, so each test starts from the baseline
+// regardless of the order they run in.
+func setLimit(t *testing.T, accountID, dimension string, value int) {
+	t.Helper()
+	harness.SpxAdminQuotaSet(t, accountID, "--"+dimension, strconv.Itoa(value))
+	t.Cleanup(func() { harness.SpxAdminQuotaClear(t, accountID) })
+}
+
+// headroom resolves what an account already holds and caps the dimension one
+// above it. Every limit test is then about the boundary rather than about the
+// state the suite happened to inherit.
+func headroom(t *testing.T, accountID, dimension string, used int) {
+	t.Helper()
+	harness.Detail(t, "dimension", dimension, "used", used, "limit", used+1)
+	setLimit(t, accountID, dimension, used+1)
+}
+
+func itoa(v int) string { return strconv.Itoa(v) }
+
+func countVPCs(t *testing.T, c *harness.AWSClient) int {
+	t.Helper()
+	out, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{})
+	require.NoError(t, err, "describe-vpcs")
+	return len(out.Vpcs)
+}
+
+func countSubnets(t *testing.T, c *harness.AWSClient) int {
+	t.Helper()
+	out, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{})
+	require.NoError(t, err, "describe-subnets")
+	return len(out.Subnets)
+}
+
+func countAddresses(t *testing.T, c *harness.AWSClient) int {
+	t.Helper()
+	out, err := c.EC2.DescribeAddresses(&ec2.DescribeAddressesInput{})
+	require.NoError(t, err, "describe-addresses")
+	return len(out.Addresses)
+}
+
+// volumeUsage returns the account's volume count and the GiB they sum to, the
+// two independently capped facts about EBS.
+func volumeUsage(t *testing.T, c *harness.AWSClient) (count, gib int) {
+	t.Helper()
+	out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{})
+	require.NoError(t, err, "describe-volumes")
+	for _, v := range out.Volumes {
+		gib += int(aws.Int64Value(v.Size))
+	}
+	return len(out.Volumes), gib
+}
+
+// createVPC creates a VPC and registers its removal. The caller gets the ID so
+// it can delete early, which is how the release assertions are written.
+func createVPC(t *testing.T, c *harness.AWSClient, cidr string) string {
+	t.Helper()
+	// The create path is the subject under test, so it cannot route through a
+	// fixture that would hide the call being counted. e2e:allow-create
+	out, err := c.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(cidr)})
+	require.NoError(t, err, "create-vpc %s", cidr)
+	id := aws.StringValue(out.Vpc.VpcId)
+	require.NotEmpty(t, id, "create-vpc returned no VpcId")
+	t.Cleanup(func() { deleteVPC(c, id) })
+	return id
+}
+
+func deleteVPC(c *harness.AWSClient, vpcID string) {
+	_, _ = c.EC2.DeleteVpc(&ec2.DeleteVpcInput{VpcId: aws.String(vpcID)})
+}
+
+// createVolume creates one EBS volume and registers its removal.
+func createVolume(t *testing.T, c *harness.AWSClient, az string, sizeGiB int64) string {
+	t.Helper()
+	// e2e:allow-create — the create path is the subject under test.
+	out, err := c.EC2.CreateVolume(&ec2.CreateVolumeInput{
+		AvailabilityZone: aws.String(az),
+		Size:             aws.Int64(sizeGiB),
+	})
+	require.NoError(t, err, "create-volume %d GiB", sizeGiB)
+	id := aws.StringValue(out.VolumeId)
+	require.NotEmpty(t, id, "create-volume returned no VolumeId")
+	t.Cleanup(func() {
+		_, _ = c.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(id)})
+	})
+	return id
+}
+
+// tenantSubnet returns a subnet the tenant can launch into, preferring one of
+// the default VPC's so no test pays for a topology it does not assert on.
+func tenantSubnet(t *testing.T, c *harness.AWSClient) string {
+	t.Helper()
+	out, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{})
+	require.NoError(t, err, "describe-subnets")
+	require.NotEmpty(t, out.Subnets, "account has no subnet to launch into")
+	return aws.StringValue(out.Subnets[0].SubnetId)
+}
+
+// tenantKeyPair creates a key pair for the account and registers its removal.
+func tenantKeyPair(t *testing.T, c *harness.AWSClient) string {
+	t.Helper()
+	name := fmt.Sprintf("e2e-quota-%d", time.Now().UnixNano())
+	// The fixture key pair belongs to the super-admin account; a launch by this
+	// tenant needs one of its own. e2e:allow-create
+	_, err := c.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String(name)})
+	require.NoError(t, err, "create-key-pair")
+	t.Cleanup(func() {
+		_, _ = c.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(name)})
+	})
+	return name
+}
+
+// launchSpec is everything a tenant launch needs that is fixed for the run.
+type launchSpec struct {
+	amiID   string
+	keyName string
+	subnet  string
+}
+
+// runInstances issues one launch of count instances. It returns the error
+// unwrapped so the caller can assert on the code rather than on a wrapper.
+func (s launchSpec) runInstances(c *harness.AWSClient, instanceType string, count int64) (*ec2.Reservation, error) {
+	// The launch is the checked operation; a fixture would memoize it and the
+	// second, refused launch would never be issued. e2e:allow-create
+	return c.EC2.RunInstances(&ec2.RunInstancesInput{
+		ImageId:      aws.String(s.amiID),
+		InstanceType: aws.String(instanceType),
+		KeyName:      aws.String(s.keyName),
+		SubnetId:     aws.String(s.subnet),
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(count),
+	})
+}
+
+// launchOne runs a single instance and registers its termination.
+func (s launchSpec) launchOne(t *testing.T, c *harness.AWSClient, instanceType string) (string, error) {
+	t.Helper()
+	out, err := s.runInstances(c, instanceType, 1)
+	if err != nil {
+		return "", err
+	}
+	require.NotEmpty(t, out.Instances, "RunInstances returned no instance")
+	id := aws.StringValue(out.Instances[0].InstanceId)
+	t.Cleanup(func() { terminateInstance(c, id) })
+	return id, nil
+}
+
+func terminateInstance(c *harness.AWSClient, instanceID string) {
+	_, _ = c.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	})
+}
+
+// waitTerminated blocks until the instance reaches terminated. The vCPU counter
+// is released by the gateway's reconcile sweep, which reads instance state, so
+// polling the state first keeps the release assertion honest about what it is
+// waiting for.
+func waitTerminated(t *testing.T, c *harness.AWSClient, instanceID string) {
+	t.Helper()
+	harness.EventuallyErr(t, func() error {
+		out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
+			InstanceIds: []*string{aws.String(instanceID)},
+		})
+		if err != nil {
+			return err
+		}
+		for _, r := range out.Reservations {
+			for _, inst := range r.Instances {
+				if state := aws.StringValue(inst.State.Name); state != "terminated" {
+					return fmt.Errorf("instance %s state=%s", instanceID, state)
+				}
+			}
+		}
+		return nil
+	}, 5*time.Minute, 5*time.Second)
+}
+
+func countInstances(t *testing.T, c *harness.AWSClient) int {
+	t.Helper()
+	out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
+	require.NoError(t, err, "describe-instances")
+	live := 0
+	for _, r := range out.Reservations {
+		for _, inst := range r.Instances {
+			if aws.StringValue(inst.State.Name) != "terminated" {
+				live++
+			}
+		}
+	}
+	return live
+}
+
+// instanceTypeVCPUs returns the vCPUs one instance of instanceType reserves.
+func instanceTypeVCPUs(t *testing.T, c *harness.AWSClient, instanceType string) int {
+	t.Helper()
+	out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{
+		InstanceTypes: []*string{aws.String(instanceType)},
+	})
+	require.NoError(t, err, "describe-instance-types %s", instanceType)
+	require.NotEmpty(t, out.InstanceTypes, "instance type %s not offered", instanceType)
+	require.NotNil(t, out.InstanceTypes[0].VCpuInfo, "instance type %s reports no VCpuInfo", instanceType)
+	return int(aws.Int64Value(out.InstanceTypes[0].VCpuInfo.DefaultVCpus))
+}
+
+// widestInstanceType returns the offered type with the most vCPUs, which is how
+// a tenant would try to consume its whole allowance in one launch.
+func widestInstanceType(t *testing.T, c *harness.AWSClient) (string, int) {
+	t.Helper()
+	out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
+	require.NoError(t, err, "describe-instance-types")
+
+	var name string
+	var vcpus int
+	for _, it := range out.InstanceTypes {
+		if it.VCpuInfo == nil {
+			continue
+		}
+		if v := int(aws.Int64Value(it.VCpuInfo.DefaultVCpus)); v > vcpus {
+			name, vcpus = aws.StringValue(it.InstanceType), v
+		}
+	}
+	require.NotEmpty(t, name, "no instance type reports a vCPU count")
+	return name, vcpus
+}

tests/e2e/quota/main_test.go

@@ -0,0 +1,172 @@
+//go:build e2e
+
+// Package quota is the per-account service-quota E2E suite. It turns quotas on
+// for the duration of the run, gives the super-admin account an unlimited
+// override so nothing else on the box is capped, then drives each enforced
+// dimension from a freshly created tenant account: the account a self-service
+// signup produces, holding the baseline a production sandbox user gets.
+//
+// Every test resolves the account's current usage before it sets a limit, so an
+// assertion is about the boundary rather than about how much the account
+// happened to hold when the suite started.
+package quota
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+)
+
+// Package-scoped singleton fixture, built lazily by requireQuotaFixture so a
+// run with SPINIFEX_E2E unset neither dials AWS nor rewrites any config.
+var (
+	pkgFixOnce sync.Once
+	pkgFix     *Fixture
+	pkgFixErr  error
+)
+
+// TestMain owns process-level lifecycle. Teardown unwinds LIFO, and the order
+// is load-bearing: turning quotas back off closes the override bucket, so an
+// override still to be removed must go first or its removal cannot be stored.
+func TestMain(m *testing.M) {
+	code := m.Run()
+	if pkgFix != nil {
+		if pkgFix.Harness != nil {
+			if err := pkgFix.Harness.Close(); err != nil {
+				fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err)
+				code = 1
+			}
+		}
+		for i := len(pkgFix.restore) - 1; i >= 0; i-- {
+			if err := pkgFix.restore[i](); err != nil {
+				fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err)
+				code = 1
+			}
+		}
+	}
+	os.Exit(code)
+}
+
+// Fixture carries the per-process state shared by every Test* in the package.
+type Fixture struct {
+	Env *harness.Env
+
+	// Admin is the super-admin account: the credential the /admin surface
+	// accepts, and the account whose quota is lifted for the run.
+	Admin          *harness.AWSClient
+	AdminAccountID string
+
+	// Harness memoizes the AMI, instance type and AZ discovery. Bound to the
+	// admin client because gold images live in the super-admin account.
+	Harness *harness.Fixture
+
+	// Baseline is the [quota] block installed on the cluster, which is what an
+	// account with no override of its own resolves to.
+	Baseline harness.QuotaLimits
+
+	// Tenant is the account under test; Peer proves an override applies to one
+	// account and not to its neighbour.
+	Tenant *harness.Profile
+	Peer   *harness.Profile
+
+	// restore undoes the cluster-level changes the fixture made, in the order
+	// registered. Each reports an error rather than failing a test: they run
+	// from TestMain, after the last test has finished.
+	restore []func() error
+}
+
+// requireQuotaFixture returns the package-scoped Fixture, building it on first
+// call. Skips when SPINIFEX_E2E is unset; runs in both single and multinode
+// topologies, because the enforcement path is the same gateway code either way.
+func requireQuotaFixture(t *testing.T) *Fixture {
+	t.Helper()
+	pkgFixOnce.Do(func() {
+		if os.Getenv("SPINIFEX_E2E") == "" {
+			return
+		}
+		env := harness.LoadEnv(t)
+		admin := harness.NewAWSClient(t, env)
+
+		ident, err := admin.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{})
+		if err != nil {
+			pkgFixErr = fmt.Errorf("resolve super-admin account: %w", err)
+			return
+		}
+		fx, err := harness.NewProcessFixture(admin)
+		if err != nil {
+			pkgFixErr = err
+			return
+		}
+
+		baseline := harness.SandboxQuotaLimits()
+		fix := &Fixture{
+			Env:            env,
+			Admin:          admin,
+			AdminAccountID: aws.StringValue(ident.Account),
+			Harness:        fx,
+			Baseline:       baseline,
+		}
+
+		// Registered before anything can fail below, so a half-built fixture
+		// still puts the cluster back the way it found it.
+		fix.restore = append(fix.restore, harness.EnableQuota(t, env, baseline))
+		waitAdminSurface(t, fix)
+
+		// Suites share this VM and most of them run as the super-admin account,
+		// which the baseline would cap at 16 vCPUs. Lifting it is also how a
+		// production operator exempts an account, so the path is worth using.
+		liftAdminQuota(t, fix)
+
+		fix.Tenant = newTenant(t, env, "quota-tenant")
+		fix.Peer = newTenant(t, env, "quota-peer")
+		pkgFix = fix
+	})
+	if pkgFixErr != nil {
+		t.Fatalf("quota fixture init failed: %v", pkgFixErr)
+	}
+	if pkgFix == nil {
+		t.Skip("quota fixture unavailable (SPINIFEX_E2E unset)")
+	}
+	return pkgFix
+}
+
+// waitAdminSurface polls until the restarted gateway answers a signed request.
+// A TLS handshake only proves the listener is back; the quota buckets are
+// opened during startup and the first call must not race them.
+func waitAdminSurface(t *testing.T, fix *Fixture) {
+	t.Helper()
+	harness.EventuallyErr(t, func() error {
+		_, err := fix.Admin.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{})
+		return err
+	}, 90*time.Second, 2*time.Second)
+}
+
+// liftAdminQuota gives the super-admin account an unlimited override on every
+// dimension and registers its removal.
+func liftAdminQuota(t *testing.T, fix *Fixture) {
+	t.Helper()
+	harness.SpxAdminQuotaSet(t, fix.AdminAccountID,
+		"--vcpus", "-1", "--vpcs", "-1", "--subnets", "-1", "--eips", "-1",
+		"--volumes", "-1", "--volumes-gib", "-1", "--rds-instances", "-1",
+		"--load-balancers", "-1")
+	fix.restore = append(fix.restore, func() error {
+		return harness.ClearAccountQuota(fix.AdminAccountID)
+	})
+}
+
+// newTenant creates an account the way self-service signup does and returns a
+// client scoped to it. Its quota is whatever the account inherits, which is the
+// point: a sandbox user is never configured, only defaulted.
+func newTenant(t *testing.T, env *harness.Env, label string) *harness.Profile {
+	t.Helper()
+	name := fmt.Sprintf("E2E %s %d", label, time.Now().UnixNano())
+	info := harness.SpxAdminAccountCreate(t, name, "")
+	return harness.NewAccountCarousel().Add(t, env, label, info)
+}

tests/e2e/quota/tests_test.go

@@ -0,0 +1,308 @@
+//go:build e2e
+
+package quota
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/rds"
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+	"github.com/stretchr/testify/require"
+)
+
+// A sandbox account is never configured, only defaulted, so the numbers it
+// inherits are the ones production actually enforces. Every dimension must
+// report as inherited: a value that silently arrived as an override would mean
+// account creation is writing limits nobody asked it to write.
+func TestQuotaBaselineIsInherited(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — a new account inherits the configured baseline")
+
+	limits, source := harness.SpxAdminQuotaGet(t, fix.Tenant.AccountID)
+
+	want := map[string]int{
+		"vcpus":          fix.Baseline.VCPUs,
+		"vpcs":           fix.Baseline.VPCs,
+		"subnets":        fix.Baseline.Subnets,
+		"eips":           fix.Baseline.EIPs,
+		"volumes":        fix.Baseline.Volumes,
+		"volumes_gib":    fix.Baseline.VolumesGiB,
+		"rds_instances":  fix.Baseline.RDSInstances,
+		"load_balancers": fix.Baseline.LoadBalancers,
+	}
+	require.Len(t, limits, len(want), "every dimension must be reported")
+	for dimension, value := range want {
+		require.Equal(t, value, limits[dimension], "%s limit", dimension)
+		require.Equal(t, "config", source[dimension], "%s must be inherited, not set", dimension)
+	}
+}
+
+// The VPC gate, and with it the property the whole live-counted tier rests on:
+// deleting a resource releases its allowance with no uncharge path to go wrong.
+func TestQuotaVPCLimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — VPCs")
+	tenant := fix.Tenant.Client
+
+	headroom(t, fix.Tenant.AccountID, "vpcs", countVPCs(t, tenant))
+
+	harness.Step(t, "the one VPC the account has room for is created")
+	vpcID := createVPC(t, tenant, "10.180.0.0/16")
+
+	harness.Step(t, "the next is refused")
+	// e2e:allow-create — the refusal is the assertion.
+	_, err := tenant.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.181.0.0/16")})
+	harness.AssertAWSError(t, err, quotaExceeded)
+
+	harness.Step(t, "deleting releases the allowance")
+	deleteVPC(tenant, vpcID)
+	released := createVPC(t, tenant, "10.182.0.0/16")
+	require.NotEmpty(t, released, "a VPC deleted under the cap must free room for another")
+}
+
+// Subnets are capped per account rather than per VPC, so the count that matters
+// spans every VPC the account holds.
+func TestQuotaSubnetLimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — subnets")
+	tenant := fix.Tenant.Client
+
+	vpcID := createVPC(t, tenant, "10.183.0.0/16")
+	headroom(t, fix.Tenant.AccountID, "subnets", countSubnets(t, tenant))
+
+	// e2e:allow-create — the create path is the subject under test.
+	out, err := tenant.EC2.CreateSubnet(&ec2.CreateSubnetInput{
+		VpcId:     aws.String(vpcID),
+		CidrBlock: aws.String("10.183.1.0/24"),
+	})
+	require.NoError(t, err, "the subnet the account has room for")
+	subnetID := aws.StringValue(out.Subnet.SubnetId)
+	t.Cleanup(func() {
+		_, _ = tenant.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(subnetID)})
+	})
+
+	// e2e:allow-create — the refusal is the assertion.
+	_, err = tenant.EC2.CreateSubnet(&ec2.CreateSubnetInput{
+		VpcId:     aws.String(vpcID),
+		CidrBlock: aws.String("10.183.2.0/24"),
+	})
+	harness.AssertAWSError(t, err, quotaExceeded)
+}
+
+// Elastic IPs are the scarcest thing a cluster hands out — the pool is finite
+// and shared — so this gate is the one that decides how many tenants fit.
+func TestQuotaEIPLimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — Elastic IPs")
+	tenant := fix.Tenant.Client
+
+	headroom(t, fix.Tenant.AccountID, "eips", countAddresses(t, tenant))
+
+	// e2e:allow-create — the allocation is the checked operation.
+	alloc, err := tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")})
+	require.NoError(t, err, "the address the account has room for")
+	allocID := aws.StringValue(alloc.AllocationId)
+	released := false
+	t.Cleanup(func() {
+		if !released {
+			_, _ = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)})
+		}
+	})
+
+	// e2e:allow-create — the refusal is the assertion.
+	_, err = tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")})
+	harness.AssertAWSError(t, err, quotaExceeded)
+
+	harness.Step(t, "releasing the address frees the allowance")
+	_, err = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)})
+	require.NoError(t, err, "release-address")
+	released = true
+
+	// e2e:allow-create — proves the release, not the allocation.
+	regained, err := tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")})
+	require.NoError(t, err, "a released address must free room for another")
+	t.Cleanup(func() {
+		_, _ = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: regained.AllocationId})
+	})
+}
+
+// EBS is capped twice over, and the two caps catch different abuse: a tenant can
+// exhaust per-volume overhead long before it exhausts capacity. Each must be
+// able to refuse a create on its own.
+func TestQuotaVolumeLimits(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — EBS volumes")
+	tenant := fix.Tenant.Client
+	az := harness.DiscoverDefaultAZ(t, fix.Harness)
+
+	t.Run("count", func(t *testing.T) {
+		count, _ := volumeUsage(t, tenant)
+		harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID,
+			"--volumes", itoa(count+1), "--volumes-gib", "-1")
+		t.Cleanup(func() { harness.SpxAdminQuotaClear(t, fix.Tenant.AccountID) })
+
+		volID := createVolume(t, tenant, az, 1)
+		require.NotEmpty(t, volID)
+
+		// e2e:allow-create — the refusal is the assertion.
+		_, err := tenant.EC2.CreateVolume(&ec2.CreateVolumeInput{
+			AvailabilityZone: aws.String(az), Size: aws.Int64(1),
+		})
+		harness.AssertAWSError(t, err, quotaExceeded)
+	})
+
+	t.Run("capacity", func(t *testing.T) {
+		_, gib := volumeUsage(t, tenant)
+		harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID,
+			"--volumes", "-1", "--volumes-gib", itoa(gib+1))
+		t.Cleanup(func() { harness.SpxAdminQuotaClear(t, fix.Tenant.AccountID) })
+
+		require.NotEmpty(t, createVolume(t, tenant, az, 1), "the GiB the account has room for")
+
+		// e2e:allow-create — the refusal is the assertion.
+		_, err := tenant.EC2.CreateVolume(&ec2.CreateVolumeInput{
+			AvailabilityZone: aws.String(az), Size: aws.Int64(1),
+		})
+		harness.AssertAWSError(t, err, quotaExceeded)
+	})
+}
+
+// The headline gate. A sandbox account is bounded by total vCPUs rather than by
+// instance count, so it may spend its 16 however it likes — but not in one
+// oversized launch, and not past the cap one nano at a time.
+func TestQuotaVCPULimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — EC2 vCPUs")
+	tenant := fix.Tenant.Client
+
+	nanoType, arch := harness.DiscoverNanoInstanceType(t, fix.Harness)
+	spec := launchSpec{
+		amiID:   harness.DiscoverUbuntuAMI(t, fix.Harness, arch),
+		keyName: tenantKeyPair(t, tenant),
+		subnet:  tenantSubnet(t, tenant),
+	}
+
+	t.Run("one launch cannot exceed the cap", func(t *testing.T) {
+		wideType, wideVCPUs := widestInstanceType(t, tenant)
+		count := int64(fix.Baseline.VCPUs/wideVCPUs) + 1
+		harness.Detail(t, "type", wideType, "vcpus_each", wideVCPUs,
+			"count", count, "cap", fix.Baseline.VCPUs)
+
+		setLimit(t, fix.Tenant.AccountID, "vcpus", fix.Baseline.VCPUs)
+		before := countInstances(t, tenant)
+
+		_, err := spec.runInstances(tenant, wideType, count)
+		harness.AssertAWSError(t, err, quotaExceeded)
+
+		require.Equal(t, before, countInstances(t, tenant),
+			"a refused launch must not leave an instance behind")
+	})
+
+	t.Run("the cap is reached one instance at a time", func(t *testing.T) {
+		nanoVCPUs := instanceTypeVCPUs(t, tenant, nanoType)
+		setLimit(t, fix.Tenant.AccountID, "vcpus", nanoVCPUs)
+
+		instID, err := spec.launchOne(t, tenant, nanoType)
+		require.NoError(t, err, "the instance the account has room for")
+
+		_, err = spec.runInstances(tenant, nanoType, 1)
+		harness.AssertAWSError(t, err, quotaExceeded)
+
+		harness.Step(t, "terminating releases the reserved vCPUs")
+		terminateInstance(tenant, instID)
+		waitTerminated(t, tenant, instID)
+
+		// The counter is released by the gateway's periodic reconcile rather
+		// than by the terminate itself, so the retry window spans a sweep. A
+		// refused attempt launches nothing, so polling costs only the denials.
+		harness.EventuallyErr(t, func() error {
+			_, rerr := spec.launchOne(t, tenant, nanoType)
+			return rerr
+		}, 3*time.Minute, 10*time.Second)
+	})
+}
+
+// Load balancers are refused at the gateway before any subnet is resolved, so
+// the two halves of this test differ only in the limit: at zero the quota is
+// what refuses, and uncapped the same call gets far enough to fail on its own
+// bad input. That contrast is what proves the quota was the cause.
+func TestQuotaLoadBalancerLimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — load balancers")
+	tenant := fix.Tenant.Client
+
+	create := func() error {
+		// The gate runs before the input is validated, which is what lets this
+		// assert the limit without provisioning a balancer. e2e:allow-create
+		_, err := tenant.ELBv2.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
+			Name:    aws.String("e2e-quota-lb"),
+			Type:    aws.String("network"),
+			Subnets: []*string{aws.String("subnet-00000000000000000")},
+		})
+		return err
+	}
+
+	setLimit(t, fix.Tenant.AccountID, "load-balancers", 0)
+	harness.AssertAWSError(t, create(), quotaExceeded)
+
+	harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, "--load-balancers", "-1")
+	err := create()
+	require.Error(t, err, "the bogus subnet must still be refused")
+	require.False(t, harness.ErrorCodeIs(err, quotaExceeded),
+		"an uncapped account must get past the quota gate, got: %v", err)
+}
+
+// RDS is capped by instance count rather than by vCPUs: the engine VM runs in
+// the system account, so a tenant's database consumes none of the tenant's own
+// vCPU allowance and the count is the only thing bounding it.
+func TestQuotaRDSInstanceLimit(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — RDS instances")
+	tenant := fix.Tenant.Client
+
+	create := func() error {
+		// The gate runs before the input is validated, so the limit is asserted
+		// without booting a database.
+		_, err := tenant.RDS.CreateDBInstance(&rds.CreateDBInstanceInput{ //nolint:staticcheck // e2e:allow-create
+			DBInstanceIdentifier: aws.String("e2e-quota-db"),
+			DBInstanceClass:      aws.String("db.t3.micro"),
+			Engine:               aws.String("mariadb"),
+			MasterUsername:       aws.String("admin"),
+			MasterUserPassword:   aws.String("e2e-quota-password"),
+			AllocatedStorage:     aws.Int64(1),
+			DBSubnetGroupName:    aws.String("e2e-quota-missing-subnet-group"),
+		})
+		return err
+	}
+
+	setLimit(t, fix.Tenant.AccountID, "rds-instances", 0)
+	harness.AssertAWSError(t, create(), quotaExceeded)
+
+	harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, "--rds-instances", "-1")
+	err := create()
+	require.Error(t, err, "the missing subnet group must still be refused")
+	require.False(t, harness.ErrorCodeIs(err, quotaExceeded),
+		"an uncapped account must get past the quota gate, got: %v", err)
+}
+
+// A limit is a property of one account, not of the gateway. Two tenants served
+// by the same gateway, one capped and one not, is the whole reason overrides
+// exist: a special customer is raised without loosening anybody else.
+func TestQuotaOverrideIsPerAccount(t *testing.T) {
+	fix := requireQuotaFixture(t)
+	harness.Phase(t, "Quota — an override applies to one account only")
+
+	setLimit(t, fix.Tenant.AccountID, "vpcs", 0)
+
+	// e2e:allow-create — the refusal is the assertion.
+	_, err := fix.Tenant.Client.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.184.0.0/16")})
+	harness.AssertAWSError(t, err, quotaExceeded)
+
+	harness.Step(t, "the neighbouring account is untouched")
+	require.NotEmpty(t, createVPC(t, fix.Peer.Client, "10.185.0.0/16"),
+		"one account's override must not cap another")
+}