diff --git a/.github/scripts/suite-sets.sh b/.github/scripts/suite-sets.sh index c39207631..2acbfadca 100644 --- a/.github/scripts/suite-sets.sh +++ b/.github/scripts/suite-sets.sh @@ -24,10 +24,14 @@ # behaviour, so a second node adds no assertion. Its DB VMs are capped at four # concurrent by the suite's own semaphore, which is what keeps it inside the # job's 50-minute budget alongside the suites above. -E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds" +# quota belongs to both topologies because the gate it asserts is gateway code +# that runs identically on either, and because a quota left switched on would +# cap every suite sharing the environment — so it must be proved to restore +# itself wherever it runs, not only on one node. +E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota" # Suites runnable against a multi-node environment. -E2E_SUITES_MULTI="multinode lb cert" +E2E_SUITES_MULTI="multinode lb cert quota" # grep -xE alternation form, for narrowing a requested set to the eligible one. E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')" diff --git a/cmd/spinifex/cmd/admin_account_quota_test.go b/cmd/spinifex/cmd/admin_account_quota_test.go index 86713d63d..76f188368 100644 --- a/cmd/spinifex/cmd/admin_account_quota_test.go +++ b/cmd/spinifex/cmd/admin_account_quota_test.go @@ -1,3 +1,4 @@ +//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from. package cmd import ( diff --git a/spinifex/gateway/admin_accountquota_test.go b/spinifex/gateway/admin_accountquota_test.go index 00302384c..5e50a4ec6 100644 --- a/spinifex/gateway/admin_accountquota_test.go +++ b/spinifex/gateway/admin_accountquota_test.go @@ -1,3 +1,4 @@ +//test:in-package — drives the unexported admin handlers and the request identity they read from the context. package gateway import ( diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index e14ea83bd..8f6610830 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -18,9 +18,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG)))) # BIN_DIR/.test so CI can ship binaries (not source + go toolchain) to # the remote VM. BIN_DIR ?= $(CURDIR)/_bin -SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds +SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota -.PHONY: e2e e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-rds e2e-all clean e2e-build +.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-rds e2e-all clean e2e-build RUN_FLAG = $(if $(RUN),-run '$(RUN)',) @@ -61,6 +61,7 @@ e2e-partialblock: ; $(MAKE) e2e PKG=./partialblock/... # minutes, so a single wedged create eats most of a 30m package timeout before # the tests still running get to report. e2e-rds: ; $(MAKE) e2e PKG=./rds/... TIMEOUT=45m +e2e-quota: ; $(MAKE) e2e PKG=./quota/... e2e-all: ; $(MAKE) e2e e2e-build: diff --git a/tests/e2e/harness/quota.go b/tests/e2e/harness/quota.go new file mode 100644 index 000000000..23212b80e --- /dev/null +++ b/tests/e2e/harness/quota.go @@ -0,0 +1,318 @@ +//go:build e2e + +package harness + +import ( + "context" + "crypto/tls" + "encoding/base64" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota" +) + +// QuotaLimits is the [quota] baseline a suite installs on the cluster. Every +// field is written, so a zero value denies that dimension outright rather than +// inheriting anything — pass a complete set. +type QuotaLimits struct { + VCPUs int + VPCs int + Subnets int + EIPs int + Volumes int + VolumesGiB int + RDSInstances int + LoadBalancers int +} + +// SandboxQuotaLimits mirrors the baseline shipped in the awsgw.toml template, +// which is what a self-service account gets in production. A suite asserting +// against these numbers is asserting against what a real tenant sees. +func SandboxQuotaLimits() QuotaLimits { + return QuotaLimits{ + VCPUs: 16, VPCs: 4, Subnets: 16, EIPs: 4, + Volumes: 16, VolumesGiB: 200, RDSInstances: 2, LoadBalancers: 2, + } +} + +// QuotaConfigPath returns the awsgw config file holding the [quota] block. +func QuotaConfigPath(env *Env) string { + return filepath.Join(env.ConfigDir, "awsgw", "awsgw.toml") +} + +// awsgwUnit is the systemd unit read the [quota] block at startup. +const awsgwUnit = "spinifex-awsgw" + +// EnableQuota turns per-account quotas on across the cluster and restarts every +// gateway so the new block takes effect. The returned function puts the +// original config back and restarts again — a suite that leaves quotas on caps +// every suite that follows it on the same VM, so it must always run. +// +// Restore reports through an error rather than the *testing.T: it belongs in +// TestMain, after every test has finished, where logging to a t would panic. +func EnableQuota(t *testing.T, env *Env, limits QuotaLimits) func() error { + t.Helper() + if env.ConfigDir == "" { + t.Fatalf("EnableQuota: no config dir located; set SPINIFEX_CONFIG_DIR") + } + path := QuotaConfigPath(env) + hosts := quotaHosts(env) + + saved := make(map[string]string, len(hosts)) + for _, h := range hosts { + body, err := readNodeFile(h, path) + if err != nil { + t.Fatalf("EnableQuota: read %s on %s: %v", path, hostLabel(h), err) + } + saved[h] = body + } + + for _, h := range hosts { + if err := writeNodeFile(h, path, withQuotaBlock(saved[h], limits)); err != nil { + t.Fatalf("EnableQuota: write %s on %s: %v", path, hostLabel(h), err) + } + if err := restartGateway(env, h); err != nil { + t.Fatalf("EnableQuota: restart gateway on %s: %v", hostLabel(h), err) + } + } + + return func() error { + // Every host is attempted even after one fails: a cluster left with the + // suite's quota block on some nodes and not others is worse than either. + var failures []string + for _, h := range hosts { + if err := writeNodeFile(h, path, saved[h]); err != nil { + failures = append(failures, fmt.Sprintf("write %s on %s: %v", path, hostLabel(h), err)) + continue + } + if err := restartGateway(env, h); err != nil { + failures = append(failures, fmt.Sprintf("restart gateway on %s: %v", hostLabel(h), err)) + } + } + if len(failures) > 0 { + return fmt.Errorf("restore quota config: %s", strings.Join(failures, "; ")) + } + return nil + } +} + +// quotaHosts returns the hosts whose gateway config must be patched. The empty +// string means this machine, which is where the single-node suites run. +func quotaHosts(env *Env) []string { + if env.Mode != ModeMultinode { + return []string{""} + } + cluster, err := ClusterFromEnv() + if err != nil { + return []string{""} + } + hosts := make([]string, 0, len(cluster.Nodes)) + for _, n := range cluster.Nodes { + hosts = append(hosts, n.Addr) + } + return hosts +} + +func hostLabel(host string) string { + if host == "" { + return "localhost" + } + return host +} + +// runNodeShell runs script on host, or on this machine when host is empty. +func runNodeShell(host, script string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if host != "" { + return NewPeerSSH().Run(ctx, host, script) + } + out, err := exec.CommandContext(ctx, "bash", "-c", script).CombinedOutput() + if err != nil { + return out, fmt.Errorf("local shell: %w: %s", err, strings.TrimSpace(string(out))) + } + return out, nil +} + +func readNodeFile(host, path string) (string, error) { + out, err := runNodeShell(host, "sudo -n cat -- "+ShellQuote(path)) + return string(out), err +} + +// writeNodeFile replaces path's contents. The body travels base64-encoded so a +// single command string carries it intact over SSH, quoting and newlines alike. +func writeNodeFile(host, path, body string) error { + script := fmt.Sprintf("printf %%s %s | base64 -d | sudo -n tee -- %s >/dev/null", + ShellQuote(base64.StdEncoding.EncodeToString([]byte(body))), ShellQuote(path)) + _, err := runNodeShell(host, script) + return err +} + +func restartGateway(env *Env, host string) error { + if _, err := runNodeShell(host, "sudo -n systemctl restart "+awsgwUnit); err != nil { + return err + } + addr := host + if addr == "" { + addr = gatewayDialHost(env) + } + return waitGatewayListening(addr, env.AWSGWPort, 90*time.Second) +} + +// gatewayDialHost picks an address the local gateway answers on. ServiceIPs[0] +// is the awsgw bind IP, so it is correct even where the listener is not on +// loopback. +func gatewayDialHost(env *Env) string { + if len(env.ServiceIPs) > 0 && env.ServiceIPs[0] != "" { + return env.ServiceIPs[0] + } + return "127.0.0.1" +} + +// waitGatewayListening polls until the gateway completes a TLS handshake. The +// certificate is not checked: this is a readiness probe, and the cert suite is +// what asserts the chain. +func waitGatewayListening(host string, port int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + dialer := &net.Dialer{Timeout: 5 * time.Second} + target := net.JoinHostPort(host, strconv.Itoa(port)) + var last error + for time.Now().Before(deadline) { + conn, err := tls.DialWithDialer(dialer, "tcp", target, + &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // readiness probe, not a trust decision + if err == nil { + _ = conn.Close() + return nil + } + last = err + time.Sleep(time.Second) + } + return fmt.Errorf("gateway %s did not accept connections within %s: %w", target, timeout, last) +} + +// withQuotaBlock returns cfg with its [quota] table replaced by limits. The old +// table is dropped whole — editing keys in place would leave a stale key behind +// whenever the block being replaced carries one the new block does not. +func withQuotaBlock(cfg string, limits QuotaLimits) string { + quotaTable := regexp.MustCompile(`^\s*\[quota(\.[^\]]+)?\]\s*$`) + + var kept []string + skipping := false + for _, line := range strings.Split(cfg, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "[") { + skipping = quotaTable.MatchString(line) + } + if !skipping { + kept = append(kept, line) + } + } + + return strings.TrimRight(strings.Join(kept, "\n"), "\n") + "\n\n" + fmt.Sprintf( + `[quota] +enabled = true +vcpus = %d +vpcs = %d +subnets = %d +eips = %d +volumes = %d +volumes_gib = %d +rds_instances = %d +load_balancers = %d +`, + limits.VCPUs, limits.VPCs, limits.Subnets, limits.EIPs, + limits.Volumes, limits.VolumesGiB, limits.RDSInstances, limits.LoadBalancers) +} + +// spxAdminEnv adds the AWS profile the /admin surface signs with. The quota +// subcommands reach the gateway over SigV4 rather than NATS, so they need a +// credential where the account commands need none. +func spxAdminEnv() []string { + profile := os.Getenv("AWS_PROFILE") + if profile == "" { + profile = "spinifex" + } + return append(spxChildEnv(), "AWS_PROFILE="+profile) +} + +// SpxAdminQuotaSet runs `spx admin account quota set` for one account. args are +// the dimension flags, e.g. "--vcpus", "8". +func SpxAdminQuotaSet(t *testing.T, accountID string, args ...string) string { + t.Helper() + return spxAdminQuota(t, append([]string{"set", accountID}, args...)...) +} + +// SpxAdminQuotaClear drops every override, returning the account to the +// configured baseline. +func SpxAdminQuotaClear(t *testing.T, accountID string) string { + t.Helper() + return spxAdminQuota(t, "set", accountID, "--clear") +} + +// ClearAccountQuota is SpxAdminQuotaClear for teardown that outlives the test +// that registered it, where reporting through a *testing.T would panic. +func ClearAccountQuota(accountID string) error { + _, err := runSpxAdminQuota("set", accountID, "--clear") + return err +} + +// quotaRow matches one line of the table `spx admin account quota get` prints: +// dimension, limit, and the layer the limit came from. +var quotaRow = regexp.MustCompile(`^([a-z_]+)\s+(\S+)\s+(config|override)\s*$`) + +// SpxAdminQuotaGet returns the limits in force for an account and, per +// dimension, whether each was inherited from the config or set as an override. +func SpxAdminQuotaGet(t *testing.T, accountID string) (limits map[string]int, source map[string]string) { + t.Helper() + out := spxAdminQuota(t, "get", accountID) + + limits, source = map[string]int{}, map[string]string{} + for _, line := range strings.Split(out, "\n") { + m := quotaRow.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + value := handlers_quota.Unlimited + if m[2] != "unlimited" { + parsed, err := strconv.Atoi(m[2]) + if err != nil { + t.Fatalf("quota get %s: unparseable limit %q for %s", accountID, m[2], m[1]) + } + value = parsed + } + limits[m[1]], source[m[1]] = value, m[3] + } + if len(limits) == 0 { + t.Fatalf("quota get %s: no dimensions parsed from:\n%s", accountID, out) + } + return limits, source +} + +func spxAdminQuota(t *testing.T, args ...string) string { + t.Helper() + out, err := runSpxAdminQuota(args...) + if err != nil { + t.Fatalf("%v", err) + } + return out +} + +func runSpxAdminQuota(args ...string) (string, error) { + full := append([]string{"admin", "account", "quota"}, args...) + cmd := exec.Command(SpxBin(), full...) + cmd.Env = spxAdminEnv() + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("spx %s failed: %w\noutput:\n%s", + strings.Join(full, " "), err, out) + } + return string(out), nil +} diff --git a/tests/e2e/quota/helpers_test.go b/tests/e2e/quota/helpers_test.go new file mode 100644 index 000000000..4ce2e3032 --- /dev/null +++ b/tests/e2e/quota/helpers_test.go @@ -0,0 +1,245 @@ +//go:build e2e + +package quota + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// quotaExceeded is the code every enforced dimension refuses with. Asserting on +// it rather than on "an error" is what separates a quota denial from a create +// that failed for an unrelated reason and would otherwise read as a pass. +const quotaExceeded = "ResourceLimitExceeded" + +// setLimit overrides one dimension on an account for the duration of the test +// and clears every override afterwards, so each test starts from the baseline +// regardless of the order they run in. +func setLimit(t *testing.T, accountID, dimension string, value int) { + t.Helper() + harness.SpxAdminQuotaSet(t, accountID, "--"+dimension, strconv.Itoa(value)) + t.Cleanup(func() { harness.SpxAdminQuotaClear(t, accountID) }) +} + +// headroom resolves what an account already holds and caps the dimension one +// above it. Every limit test is then about the boundary rather than about the +// state the suite happened to inherit. +func headroom(t *testing.T, accountID, dimension string, used int) { + t.Helper() + harness.Detail(t, "dimension", dimension, "used", used, "limit", used+1) + setLimit(t, accountID, dimension, used+1) +} + +func itoa(v int) string { return strconv.Itoa(v) } + +func countVPCs(t *testing.T, c *harness.AWSClient) int { + t.Helper() + out, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{}) + require.NoError(t, err, "describe-vpcs") + return len(out.Vpcs) +} + +func countSubnets(t *testing.T, c *harness.AWSClient) int { + t.Helper() + out, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{}) + require.NoError(t, err, "describe-subnets") + return len(out.Subnets) +} + +func countAddresses(t *testing.T, c *harness.AWSClient) int { + t.Helper() + out, err := c.EC2.DescribeAddresses(&ec2.DescribeAddressesInput{}) + require.NoError(t, err, "describe-addresses") + return len(out.Addresses) +} + +// volumeUsage returns the account's volume count and the GiB they sum to, the +// two independently capped facts about EBS. +func volumeUsage(t *testing.T, c *harness.AWSClient) (count, gib int) { + t.Helper() + out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{}) + require.NoError(t, err, "describe-volumes") + for _, v := range out.Volumes { + gib += int(aws.Int64Value(v.Size)) + } + return len(out.Volumes), gib +} + +// createVPC creates a VPC and registers its removal. The caller gets the ID so +// it can delete early, which is how the release assertions are written. +func createVPC(t *testing.T, c *harness.AWSClient, cidr string) string { + t.Helper() + // The create path is the subject under test, so it cannot route through a + // fixture that would hide the call being counted. e2e:allow-create + out, err := c.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(cidr)}) + require.NoError(t, err, "create-vpc %s", cidr) + id := aws.StringValue(out.Vpc.VpcId) + require.NotEmpty(t, id, "create-vpc returned no VpcId") + t.Cleanup(func() { deleteVPC(c, id) }) + return id +} + +func deleteVPC(c *harness.AWSClient, vpcID string) { + _, _ = c.EC2.DeleteVpc(&ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}) +} + +// createVolume creates one EBS volume and registers its removal. +func createVolume(t *testing.T, c *harness.AWSClient, az string, sizeGiB int64) string { + t.Helper() + // e2e:allow-create — the create path is the subject under test. + out, err := c.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(sizeGiB), + }) + require.NoError(t, err, "create-volume %d GiB", sizeGiB) + id := aws.StringValue(out.VolumeId) + require.NotEmpty(t, id, "create-volume returned no VolumeId") + t.Cleanup(func() { + _, _ = c.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(id)}) + }) + return id +} + +// tenantSubnet returns a subnet the tenant can launch into, preferring one of +// the default VPC's so no test pays for a topology it does not assert on. +func tenantSubnet(t *testing.T, c *harness.AWSClient) string { + t.Helper() + out, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{}) + require.NoError(t, err, "describe-subnets") + require.NotEmpty(t, out.Subnets, "account has no subnet to launch into") + return aws.StringValue(out.Subnets[0].SubnetId) +} + +// tenantKeyPair creates a key pair for the account and registers its removal. +func tenantKeyPair(t *testing.T, c *harness.AWSClient) string { + t.Helper() + name := fmt.Sprintf("e2e-quota-%d", time.Now().UnixNano()) + // The fixture key pair belongs to the super-admin account; a launch by this + // tenant needs one of its own. e2e:allow-create + _, err := c.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String(name)}) + require.NoError(t, err, "create-key-pair") + t.Cleanup(func() { + _, _ = c.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(name)}) + }) + return name +} + +// launchSpec is everything a tenant launch needs that is fixed for the run. +type launchSpec struct { + amiID string + keyName string + subnet string +} + +// runInstances issues one launch of count instances. It returns the error +// unwrapped so the caller can assert on the code rather than on a wrapper. +func (s launchSpec) runInstances(c *harness.AWSClient, instanceType string, count int64) (*ec2.Reservation, error) { + // The launch is the checked operation; a fixture would memoize it and the + // second, refused launch would never be issued. e2e:allow-create + return c.EC2.RunInstances(&ec2.RunInstancesInput{ + ImageId: aws.String(s.amiID), + InstanceType: aws.String(instanceType), + KeyName: aws.String(s.keyName), + SubnetId: aws.String(s.subnet), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(count), + }) +} + +// launchOne runs a single instance and registers its termination. +func (s launchSpec) launchOne(t *testing.T, c *harness.AWSClient, instanceType string) (string, error) { + t.Helper() + out, err := s.runInstances(c, instanceType, 1) + if err != nil { + return "", err + } + require.NotEmpty(t, out.Instances, "RunInstances returned no instance") + id := aws.StringValue(out.Instances[0].InstanceId) + t.Cleanup(func() { terminateInstance(c, id) }) + return id, nil +} + +func terminateInstance(c *harness.AWSClient, instanceID string) { + _, _ = c.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) +} + +// waitTerminated blocks until the instance reaches terminated. The vCPU counter +// is released by the gateway's reconcile sweep, which reads instance state, so +// polling the state first keeps the release assertion honest about what it is +// waiting for. +func waitTerminated(t *testing.T, c *harness.AWSClient, instanceID string) { + t.Helper() + harness.EventuallyErr(t, func() error { + out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + if err != nil { + return err + } + for _, r := range out.Reservations { + for _, inst := range r.Instances { + if state := aws.StringValue(inst.State.Name); state != "terminated" { + return fmt.Errorf("instance %s state=%s", instanceID, state) + } + } + } + return nil + }, 5*time.Minute, 5*time.Second) +} + +func countInstances(t *testing.T, c *harness.AWSClient) int { + t.Helper() + out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{}) + require.NoError(t, err, "describe-instances") + live := 0 + for _, r := range out.Reservations { + for _, inst := range r.Instances { + if aws.StringValue(inst.State.Name) != "terminated" { + live++ + } + } + } + return live +} + +// instanceTypeVCPUs returns the vCPUs one instance of instanceType reserves. +func instanceTypeVCPUs(t *testing.T, c *harness.AWSClient, instanceType string) int { + t.Helper() + out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ + InstanceTypes: []*string{aws.String(instanceType)}, + }) + require.NoError(t, err, "describe-instance-types %s", instanceType) + require.NotEmpty(t, out.InstanceTypes, "instance type %s not offered", instanceType) + require.NotNil(t, out.InstanceTypes[0].VCpuInfo, "instance type %s reports no VCpuInfo", instanceType) + return int(aws.Int64Value(out.InstanceTypes[0].VCpuInfo.DefaultVCpus)) +} + +// widestInstanceType returns the offered type with the most vCPUs, which is how +// a tenant would try to consume its whole allowance in one launch. +func widestInstanceType(t *testing.T, c *harness.AWSClient) (string, int) { + t.Helper() + out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) + require.NoError(t, err, "describe-instance-types") + + var name string + var vcpus int + for _, it := range out.InstanceTypes { + if it.VCpuInfo == nil { + continue + } + if v := int(aws.Int64Value(it.VCpuInfo.DefaultVCpus)); v > vcpus { + name, vcpus = aws.StringValue(it.InstanceType), v + } + } + require.NotEmpty(t, name, "no instance type reports a vCPU count") + return name, vcpus +} diff --git a/tests/e2e/quota/main_test.go b/tests/e2e/quota/main_test.go new file mode 100644 index 000000000..c09ee21f8 --- /dev/null +++ b/tests/e2e/quota/main_test.go @@ -0,0 +1,172 @@ +//go:build e2e + +// Package quota is the per-account service-quota E2E suite. It turns quotas on +// for the duration of the run, gives the super-admin account an unlimited +// override so nothing else on the box is capped, then drives each enforced +// dimension from a freshly created tenant account: the account a self-service +// signup produces, holding the baseline a production sandbox user gets. +// +// Every test resolves the account's current usage before it sets a limit, so an +// assertion is about the boundary rather than about how much the account +// happened to hold when the suite started. +package quota + +import ( + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// Package-scoped singleton fixture, built lazily by requireQuotaFixture so a +// run with SPINIFEX_E2E unset neither dials AWS nor rewrites any config. +var ( + pkgFixOnce sync.Once + pkgFix *Fixture + pkgFixErr error +) + +// TestMain owns process-level lifecycle. Teardown unwinds LIFO, and the order +// is load-bearing: turning quotas back off closes the override bucket, so an +// override still to be removed must go first or its removal cannot be stored. +func TestMain(m *testing.M) { + code := m.Run() + if pkgFix != nil { + if pkgFix.Harness != nil { + if err := pkgFix.Harness.Close(); err != nil { + fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err) + code = 1 + } + } + for i := len(pkgFix.restore) - 1; i >= 0; i-- { + if err := pkgFix.restore[i](); err != nil { + fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err) + code = 1 + } + } + } + os.Exit(code) +} + +// Fixture carries the per-process state shared by every Test* in the package. +type Fixture struct { + Env *harness.Env + + // Admin is the super-admin account: the credential the /admin surface + // accepts, and the account whose quota is lifted for the run. + Admin *harness.AWSClient + AdminAccountID string + + // Harness memoizes the AMI, instance type and AZ discovery. Bound to the + // admin client because gold images live in the super-admin account. + Harness *harness.Fixture + + // Baseline is the [quota] block installed on the cluster, which is what an + // account with no override of its own resolves to. + Baseline harness.QuotaLimits + + // Tenant is the account under test; Peer proves an override applies to one + // account and not to its neighbour. + Tenant *harness.Profile + Peer *harness.Profile + + // restore undoes the cluster-level changes the fixture made, in the order + // registered. Each reports an error rather than failing a test: they run + // from TestMain, after the last test has finished. + restore []func() error +} + +// requireQuotaFixture returns the package-scoped Fixture, building it on first +// call. Skips when SPINIFEX_E2E is unset; runs in both single and multinode +// topologies, because the enforcement path is the same gateway code either way. +func requireQuotaFixture(t *testing.T) *Fixture { + t.Helper() + pkgFixOnce.Do(func() { + if os.Getenv("SPINIFEX_E2E") == "" { + return + } + env := harness.LoadEnv(t) + admin := harness.NewAWSClient(t, env) + + ident, err := admin.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{}) + if err != nil { + pkgFixErr = fmt.Errorf("resolve super-admin account: %w", err) + return + } + fx, err := harness.NewProcessFixture(admin) + if err != nil { + pkgFixErr = err + return + } + + baseline := harness.SandboxQuotaLimits() + fix := &Fixture{ + Env: env, + Admin: admin, + AdminAccountID: aws.StringValue(ident.Account), + Harness: fx, + Baseline: baseline, + } + + // Registered before anything can fail below, so a half-built fixture + // still puts the cluster back the way it found it. + fix.restore = append(fix.restore, harness.EnableQuota(t, env, baseline)) + waitAdminSurface(t, fix) + + // Suites share this VM and most of them run as the super-admin account, + // which the baseline would cap at 16 vCPUs. Lifting it is also how a + // production operator exempts an account, so the path is worth using. + liftAdminQuota(t, fix) + + fix.Tenant = newTenant(t, env, "quota-tenant") + fix.Peer = newTenant(t, env, "quota-peer") + pkgFix = fix + }) + if pkgFixErr != nil { + t.Fatalf("quota fixture init failed: %v", pkgFixErr) + } + if pkgFix == nil { + t.Skip("quota fixture unavailable (SPINIFEX_E2E unset)") + } + return pkgFix +} + +// waitAdminSurface polls until the restarted gateway answers a signed request. +// A TLS handshake only proves the listener is back; the quota buckets are +// opened during startup and the first call must not race them. +func waitAdminSurface(t *testing.T, fix *Fixture) { + t.Helper() + harness.EventuallyErr(t, func() error { + _, err := fix.Admin.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{}) + return err + }, 90*time.Second, 2*time.Second) +} + +// liftAdminQuota gives the super-admin account an unlimited override on every +// dimension and registers its removal. +func liftAdminQuota(t *testing.T, fix *Fixture) { + t.Helper() + harness.SpxAdminQuotaSet(t, fix.AdminAccountID, + "--vcpus", "-1", "--vpcs", "-1", "--subnets", "-1", "--eips", "-1", + "--volumes", "-1", "--volumes-gib", "-1", "--rds-instances", "-1", + "--load-balancers", "-1") + fix.restore = append(fix.restore, func() error { + return harness.ClearAccountQuota(fix.AdminAccountID) + }) +} + +// newTenant creates an account the way self-service signup does and returns a +// client scoped to it. Its quota is whatever the account inherits, which is the +// point: a sandbox user is never configured, only defaulted. +func newTenant(t *testing.T, env *harness.Env, label string) *harness.Profile { + t.Helper() + name := fmt.Sprintf("E2E %s %d", label, time.Now().UnixNano()) + info := harness.SpxAdminAccountCreate(t, name, "") + return harness.NewAccountCarousel().Add(t, env, label, info) +} diff --git a/tests/e2e/quota/tests_test.go b/tests/e2e/quota/tests_test.go new file mode 100644 index 000000000..d291fca69 --- /dev/null +++ b/tests/e2e/quota/tests_test.go @@ -0,0 +1,308 @@ +//go:build e2e + +package quota + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// A sandbox account is never configured, only defaulted, so the numbers it +// inherits are the ones production actually enforces. Every dimension must +// report as inherited: a value that silently arrived as an override would mean +// account creation is writing limits nobody asked it to write. +func TestQuotaBaselineIsInherited(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — a new account inherits the configured baseline") + + limits, source := harness.SpxAdminQuotaGet(t, fix.Tenant.AccountID) + + want := map[string]int{ + "vcpus": fix.Baseline.VCPUs, + "vpcs": fix.Baseline.VPCs, + "subnets": fix.Baseline.Subnets, + "eips": fix.Baseline.EIPs, + "volumes": fix.Baseline.Volumes, + "volumes_gib": fix.Baseline.VolumesGiB, + "rds_instances": fix.Baseline.RDSInstances, + "load_balancers": fix.Baseline.LoadBalancers, + } + require.Len(t, limits, len(want), "every dimension must be reported") + for dimension, value := range want { + require.Equal(t, value, limits[dimension], "%s limit", dimension) + require.Equal(t, "config", source[dimension], "%s must be inherited, not set", dimension) + } +} + +// The VPC gate, and with it the property the whole live-counted tier rests on: +// deleting a resource releases its allowance with no uncharge path to go wrong. +func TestQuotaVPCLimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — VPCs") + tenant := fix.Tenant.Client + + headroom(t, fix.Tenant.AccountID, "vpcs", countVPCs(t, tenant)) + + harness.Step(t, "the one VPC the account has room for is created") + vpcID := createVPC(t, tenant, "10.180.0.0/16") + + harness.Step(t, "the next is refused") + // e2e:allow-create — the refusal is the assertion. + _, err := tenant.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.181.0.0/16")}) + harness.AssertAWSError(t, err, quotaExceeded) + + harness.Step(t, "deleting releases the allowance") + deleteVPC(tenant, vpcID) + released := createVPC(t, tenant, "10.182.0.0/16") + require.NotEmpty(t, released, "a VPC deleted under the cap must free room for another") +} + +// Subnets are capped per account rather than per VPC, so the count that matters +// spans every VPC the account holds. +func TestQuotaSubnetLimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — subnets") + tenant := fix.Tenant.Client + + vpcID := createVPC(t, tenant, "10.183.0.0/16") + headroom(t, fix.Tenant.AccountID, "subnets", countSubnets(t, tenant)) + + // e2e:allow-create — the create path is the subject under test. + out, err := tenant.EC2.CreateSubnet(&ec2.CreateSubnetInput{ + VpcId: aws.String(vpcID), + CidrBlock: aws.String("10.183.1.0/24"), + }) + require.NoError(t, err, "the subnet the account has room for") + subnetID := aws.StringValue(out.Subnet.SubnetId) + t.Cleanup(func() { + _, _ = tenant.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(subnetID)}) + }) + + // e2e:allow-create — the refusal is the assertion. + _, err = tenant.EC2.CreateSubnet(&ec2.CreateSubnetInput{ + VpcId: aws.String(vpcID), + CidrBlock: aws.String("10.183.2.0/24"), + }) + harness.AssertAWSError(t, err, quotaExceeded) +} + +// Elastic IPs are the scarcest thing a cluster hands out — the pool is finite +// and shared — so this gate is the one that decides how many tenants fit. +func TestQuotaEIPLimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — Elastic IPs") + tenant := fix.Tenant.Client + + headroom(t, fix.Tenant.AccountID, "eips", countAddresses(t, tenant)) + + // e2e:allow-create — the allocation is the checked operation. + alloc, err := tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")}) + require.NoError(t, err, "the address the account has room for") + allocID := aws.StringValue(alloc.AllocationId) + released := false + t.Cleanup(func() { + if !released { + _, _ = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)}) + } + }) + + // e2e:allow-create — the refusal is the assertion. + _, err = tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")}) + harness.AssertAWSError(t, err, quotaExceeded) + + harness.Step(t, "releasing the address frees the allowance") + _, err = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)}) + require.NoError(t, err, "release-address") + released = true + + // e2e:allow-create — proves the release, not the allocation. + regained, err := tenant.EC2.AllocateAddress(&ec2.AllocateAddressInput{Domain: aws.String("vpc")}) + require.NoError(t, err, "a released address must free room for another") + t.Cleanup(func() { + _, _ = tenant.EC2.ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: regained.AllocationId}) + }) +} + +// EBS is capped twice over, and the two caps catch different abuse: a tenant can +// exhaust per-volume overhead long before it exhausts capacity. Each must be +// able to refuse a create on its own. +func TestQuotaVolumeLimits(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — EBS volumes") + tenant := fix.Tenant.Client + az := harness.DiscoverDefaultAZ(t, fix.Harness) + + t.Run("count", func(t *testing.T) { + count, _ := volumeUsage(t, tenant) + harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, + "--volumes", itoa(count+1), "--volumes-gib", "-1") + t.Cleanup(func() { harness.SpxAdminQuotaClear(t, fix.Tenant.AccountID) }) + + volID := createVolume(t, tenant, az, 1) + require.NotEmpty(t, volID) + + // e2e:allow-create — the refusal is the assertion. + _, err := tenant.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), Size: aws.Int64(1), + }) + harness.AssertAWSError(t, err, quotaExceeded) + }) + + t.Run("capacity", func(t *testing.T) { + _, gib := volumeUsage(t, tenant) + harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, + "--volumes", "-1", "--volumes-gib", itoa(gib+1)) + t.Cleanup(func() { harness.SpxAdminQuotaClear(t, fix.Tenant.AccountID) }) + + require.NotEmpty(t, createVolume(t, tenant, az, 1), "the GiB the account has room for") + + // e2e:allow-create — the refusal is the assertion. + _, err := tenant.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), Size: aws.Int64(1), + }) + harness.AssertAWSError(t, err, quotaExceeded) + }) +} + +// The headline gate. A sandbox account is bounded by total vCPUs rather than by +// instance count, so it may spend its 16 however it likes — but not in one +// oversized launch, and not past the cap one nano at a time. +func TestQuotaVCPULimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — EC2 vCPUs") + tenant := fix.Tenant.Client + + nanoType, arch := harness.DiscoverNanoInstanceType(t, fix.Harness) + spec := launchSpec{ + amiID: harness.DiscoverUbuntuAMI(t, fix.Harness, arch), + keyName: tenantKeyPair(t, tenant), + subnet: tenantSubnet(t, tenant), + } + + t.Run("one launch cannot exceed the cap", func(t *testing.T) { + wideType, wideVCPUs := widestInstanceType(t, tenant) + count := int64(fix.Baseline.VCPUs/wideVCPUs) + 1 + harness.Detail(t, "type", wideType, "vcpus_each", wideVCPUs, + "count", count, "cap", fix.Baseline.VCPUs) + + setLimit(t, fix.Tenant.AccountID, "vcpus", fix.Baseline.VCPUs) + before := countInstances(t, tenant) + + _, err := spec.runInstances(tenant, wideType, count) + harness.AssertAWSError(t, err, quotaExceeded) + + require.Equal(t, before, countInstances(t, tenant), + "a refused launch must not leave an instance behind") + }) + + t.Run("the cap is reached one instance at a time", func(t *testing.T) { + nanoVCPUs := instanceTypeVCPUs(t, tenant, nanoType) + setLimit(t, fix.Tenant.AccountID, "vcpus", nanoVCPUs) + + instID, err := spec.launchOne(t, tenant, nanoType) + require.NoError(t, err, "the instance the account has room for") + + _, err = spec.runInstances(tenant, nanoType, 1) + harness.AssertAWSError(t, err, quotaExceeded) + + harness.Step(t, "terminating releases the reserved vCPUs") + terminateInstance(tenant, instID) + waitTerminated(t, tenant, instID) + + // The counter is released by the gateway's periodic reconcile rather + // than by the terminate itself, so the retry window spans a sweep. A + // refused attempt launches nothing, so polling costs only the denials. + harness.EventuallyErr(t, func() error { + _, rerr := spec.launchOne(t, tenant, nanoType) + return rerr + }, 3*time.Minute, 10*time.Second) + }) +} + +// Load balancers are refused at the gateway before any subnet is resolved, so +// the two halves of this test differ only in the limit: at zero the quota is +// what refuses, and uncapped the same call gets far enough to fail on its own +// bad input. That contrast is what proves the quota was the cause. +func TestQuotaLoadBalancerLimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — load balancers") + tenant := fix.Tenant.Client + + create := func() error { + // The gate runs before the input is validated, which is what lets this + // assert the limit without provisioning a balancer. e2e:allow-create + _, err := tenant.ELBv2.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("e2e-quota-lb"), + Type: aws.String("network"), + Subnets: []*string{aws.String("subnet-00000000000000000")}, + }) + return err + } + + setLimit(t, fix.Tenant.AccountID, "load-balancers", 0) + harness.AssertAWSError(t, create(), quotaExceeded) + + harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, "--load-balancers", "-1") + err := create() + require.Error(t, err, "the bogus subnet must still be refused") + require.False(t, harness.ErrorCodeIs(err, quotaExceeded), + "an uncapped account must get past the quota gate, got: %v", err) +} + +// RDS is capped by instance count rather than by vCPUs: the engine VM runs in +// the system account, so a tenant's database consumes none of the tenant's own +// vCPU allowance and the count is the only thing bounding it. +func TestQuotaRDSInstanceLimit(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — RDS instances") + tenant := fix.Tenant.Client + + create := func() error { + // The gate runs before the input is validated, so the limit is asserted + // without booting a database. + _, err := tenant.RDS.CreateDBInstance(&rds.CreateDBInstanceInput{ //nolint:staticcheck // e2e:allow-create + DBInstanceIdentifier: aws.String("e2e-quota-db"), + DBInstanceClass: aws.String("db.t3.micro"), + Engine: aws.String("mariadb"), + MasterUsername: aws.String("admin"), + MasterUserPassword: aws.String("e2e-quota-password"), + AllocatedStorage: aws.Int64(1), + DBSubnetGroupName: aws.String("e2e-quota-missing-subnet-group"), + }) + return err + } + + setLimit(t, fix.Tenant.AccountID, "rds-instances", 0) + harness.AssertAWSError(t, create(), quotaExceeded) + + harness.SpxAdminQuotaSet(t, fix.Tenant.AccountID, "--rds-instances", "-1") + err := create() + require.Error(t, err, "the missing subnet group must still be refused") + require.False(t, harness.ErrorCodeIs(err, quotaExceeded), + "an uncapped account must get past the quota gate, got: %v", err) +} + +// A limit is a property of one account, not of the gateway. Two tenants served +// by the same gateway, one capped and one not, is the whole reason overrides +// exist: a special customer is raised without loosening anybody else. +func TestQuotaOverrideIsPerAccount(t *testing.T) { + fix := requireQuotaFixture(t) + harness.Phase(t, "Quota — an override applies to one account only") + + setLimit(t, fix.Tenant.AccountID, "vpcs", 0) + + // e2e:allow-create — the refusal is the assertion. + _, err := fix.Tenant.Client.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.184.0.0/16")}) + harness.AssertAWSError(t, err, quotaExceeded) + + harness.Step(t, "the neighbouring account is untouched") + require.NotEmpty(t, createVPC(t, fix.Peer.Client, "10.185.0.0/16"), + "one account's override must not cap another") +}