feat(e2e-harness): extend harness for single-node Go suite port

Stage A of mulga-siv-89 — add the primitives the single-node run-e2e.sh
port consumes:

- AWSClient gains IAM + STS service handles, plus NewAWSClientWithCreds
  for per-tenant account-scoping in Phase 8.
- awserror.go: ErrorCodeIs, AssertAWSError, ExpectError — Go equivalent
  of the bash expect_error helper used throughout the negative phases.
- poll.go: WaitForInstance/Volume/Snapshot/Image state to any target,
  with per-resource default timeouts tuned from the bash sleep loops.
- spxcli.go: thin shellouts for spx get nodes/vms, spx top nodes,
  spx admin account create with output parsing for tenant-id/access-key.
- rawhttp.go: signed raw POST against the AWS gateway for the Phase 8
  InvalidAction probe (gateway rejects on auth before action parsing).
- ovn.go: OvnNbctl/OvnSbctl wrappers + SkipIfNoOVN for Phase 8b-e
  datapath checks; WaitForPortGroupMember handles propagation polls.
- ec2helpers.go: InstancePublicSSHHost (public-IP-or-qemu-hostfwd),
  LsblkRootGiB, serial-console-access trio, DiscoverDefaultVPC,
  idempotent AuthorizeSSHIngress.
- account.go: AccountCarousel maps tenant name -> scoped AWSClient,
  built atop NewAWSClientWithCreds for the Phase 8 isolation tests.

Pure groundwork — no new tests, no production behaviour change. The
cert + lb scenarios still compile against the extended harness.

tests/e2e/harness/account.go

@@ -0,0 +1,75 @@
+//go:build e2e
+
+package harness
+
+import (
+	"sort"
+	"sync"
+	"testing"
+)
+
+// Profile bundles a tenant account's credentials with a pre-built AWSClient
+// scoped to that account. The Client field bypasses AWS_PROFILE shared-config
+// lookup — credentials are injected statically via NewAWSClientWithCreds so a
+// single test binary can drive many tenants concurrently without mutating
+// ~/.aws/credentials.
+type Profile struct {
+	Name      string
+	AccountID string
+	Client    *AWSClient
+	Info      AccountInfo
+}
+
+// AccountCarousel manages multiple Profile entries created via
+// `spx admin account create`. Used by Phase 8 account-scoping where the test
+// repeatedly switches between Alpha/Beta/Gamma to assert isolation.
+//
+// Concurrent reads via Get/Names are safe; Add must not race with itself.
+type AccountCarousel struct {
+	mu       sync.RWMutex
+	profiles map[string]*Profile
+}
+
+// NewAccountCarousel returns an empty carousel.
+func NewAccountCarousel() *AccountCarousel {
+	return &AccountCarousel{profiles: make(map[string]*Profile)}
+}
+
+// Add registers a new profile under the given name (e.g. "team-alpha") and
+// returns it. Duplicate names overwrite — caller is expected to use unique
+// labels per tenant. info.AccessKeyID and info.SecretAccessKey must be set;
+// NewAWSClientWithCreds t.Fatals otherwise.
+func (a *AccountCarousel) Add(t *testing.T, env *Env, name string, info AccountInfo) *Profile {
+	t.Helper()
+	client := NewAWSClientWithCreds(t, env, info.AccessKeyID, info.SecretAccessKey)
+	p := &Profile{
+		Name:      name,
+		AccountID: info.AccountID,
+		Client:    client,
+		Info:      info,
+	}
+	a.mu.Lock()
+	a.profiles[name] = p
+	a.mu.Unlock()
+	return p
+}
+
+// Get returns the named profile or nil if absent.
+func (a *AccountCarousel) Get(name string) *Profile {
+	a.mu.RLock()
+	defer a.mu.RUnlock()
+	return a.profiles[name]
+}
+
+// Names returns the registered profile names in sorted order — stable for
+// snapshot-style logging without depending on map iteration order.
+func (a *AccountCarousel) Names() []string {
+	a.mu.RLock()
+	defer a.mu.RUnlock()
+	out := make([]string, 0, len(a.profiles))
+	for k := range a.profiles {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}

tests/e2e/harness/aws.go

@@ -15,6 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/sts"
 )
 
 // AWSClient bundles the SDK service handles a scenario typically needs against
@@ -23,6 +25,8 @@ import (
 type AWSClient struct {
 	EC2   *ec2.EC2
 	ELBv2 *elbv2.ELBV2
+	IAM   *iam.IAM
+	STS   *sts.STS
 }
 
 // NewAWSClient builds clients pointed at https://<endpoint>:<port>/ using the
@@ -31,6 +35,23 @@ type AWSClient struct {
 // SPINIFEX_AWS_ACCESS_KEY_ID / SPINIFEX_AWS_SECRET_ACCESS_KEY for static creds.
 func NewAWSClient(t *testing.T, env *Env) *AWSClient {
 	t.Helper()
+	id, secret := os.Getenv("SPINIFEX_AWS_ACCESS_KEY_ID"), os.Getenv("SPINIFEX_AWS_SECRET_ACCESS_KEY")
+	return newAWSClient(t, env, id, secret)
+}
+
+// NewAWSClientWithCreds builds an AWSClient with explicit static credentials
+// — used by AccountCarousel to scope a client to a tenant account created
+// via `spx admin account create`. Bypasses AWS_PROFILE shared-config lookup.
+func NewAWSClientWithCreds(t *testing.T, env *Env, accessKey, secretKey string) *AWSClient {
+	t.Helper()
+	if accessKey == "" || secretKey == "" {
+		t.Fatalf("NewAWSClientWithCreds: empty credentials")
+	}
+	return newAWSClient(t, env, accessKey, secretKey)
+}
+
+func newAWSClient(t *testing.T, env *Env, accessKey, secretKey string) *AWSClient {
+	t.Helper()
 
 	endpoint := os.Getenv("SPINIFEX_AWS_ENDPOINT")
 	if endpoint == "" {
@@ -62,15 +83,18 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient {
 		},
 	}
 
-	if id, secret := os.Getenv("SPINIFEX_AWS_ACCESS_KEY_ID"), os.Getenv("SPINIFEX_AWS_SECRET_ACCESS_KEY"); id != "" && secret != "" {
-		cfg.Credentials = credentials.NewStaticCredentials(id, secret, "")
+	opts := session.Options{Config: *cfg}
+	if accessKey != "" && secretKey != "" {
+		// Static creds bypass shared-config lookup — required for the
+		// per-tenant carousel where each Profile holds its own access key.
+		cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
+		opts.Config = *cfg
+	} else {
+		opts.SharedConfigState = session.SharedConfigEnable
+		opts.Profile = getenv("AWS_PROFILE", "spinifex")
 	}
 
-	sess, err := session.NewSessionWithOptions(session.Options{
-		Config:            *cfg,
-		SharedConfigState: session.SharedConfigEnable,
-		Profile:           getenv("AWS_PROFILE", "spinifex"),
-	})
+	sess, err := session.NewSessionWithOptions(opts)
 	if err != nil {
 		t.Fatalf("AWS session: %v", err)
 	}
@@ -78,6 +102,8 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient {
 	return &AWSClient{
 		EC2:   ec2.New(sess),
 		ELBv2: elbv2.New(sess),
+		IAM:   iam.New(sess),
+		STS:   sts.New(sess),
 	}
 }
 
@@ -85,7 +111,10 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient {
 // for fault-injection scenarios — the cert scenario already covers trust path.
 func (c *AWSClient) IgnoreCertErrors() {
 	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
-	c.EC2.Config.HTTPClient = &http.Client{Transport: tr}
-	c.ELBv2.Config.HTTPClient = &http.Client{Transport: tr}
+	hc := &http.Client{Transport: tr}
+	c.EC2.Config.HTTPClient = hc
+	c.ELBv2.Config.HTTPClient = hc
+	c.IAM.Config.HTTPClient = hc
+	c.STS.Config.HTTPClient = hc
 	_ = (*x509.CertPool)(nil) // silence unused-import on hardened paths
 }

tests/e2e/harness/awserror.go

@@ -0,0 +1,49 @@
+//go:build e2e
+
+package harness
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+)
+
+// ErrorCodeIs reports whether err is an awserr.Error with the given Code.
+// Returns false for nil err or non-awserr wrappers.
+func ErrorCodeIs(err error, code string) bool {
+	if err == nil {
+		return false
+	}
+	var aerr awserr.Error
+	if errors.As(err, &aerr) {
+		return aerr.Code() == code
+	}
+	return false
+}
+
+// AssertAWSError fails the test if err is nil or its awserr.Code() != code.
+// Logs the full err on mismatch. Returns the matched awserr.Error so callers
+// can chain assertions on Message() etc.
+func AssertAWSError(t *testing.T, err error, code string) awserr.Error {
+	t.Helper()
+	if err == nil {
+		t.Fatalf("expected AWS error %q, got nil", code)
+	}
+	var aerr awserr.Error
+	if !errors.As(err, &aerr) {
+		t.Fatalf("expected awserr.Error with code %q, got %T: %v", code, err, err)
+	}
+	if aerr.Code() != code {
+		t.Fatalf("expected AWS error code %q, got %q (full: %v)", code, aerr.Code(), err)
+	}
+	return aerr
+}
+
+// ExpectError invokes fn and asserts the returned err has the given AWS code.
+// Convenience for one-shot negative tests — replaces the bash `expect_error`
+// helper used throughout run-e2e.sh.
+func ExpectError(t *testing.T, code string, fn func() error) {
+	t.Helper()
+	AssertAWSError(t, fn(), code)
+}

tests/e2e/harness/ec2helpers.go

@@ -0,0 +1,236 @@
+//go:build e2e
+
+package harness
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// SSHTarget is the minimal addressing tuple needed by LsblkRootGiB. The
+// existing harness.SSH interface requires a Cluster.Node, but the single-host
+// e2e suite addresses individual instances by (host, port, key) discovered at
+// runtime — those don't fit a Cluster. Defined here rather than in ssh.go to
+// keep the production SSH transport untouched.
+type SSHTarget struct {
+	User    string
+	Host    string
+	Port    int
+	KeyPath string
+}
+
+// InstancePublicSSHHost returns (host, port) for an SSH connection to inst.
+//
+// Preference order matches run-e2e.sh:
+//  1. Instance's public IP on port 22, when assigned (external networking).
+//  2. QEMU hostfwd discovered by scanning `ps -ef` for the qemu process whose
+//     command line contains the instance ID and a `hostfwd=tcp:<ip>:<port>-:22`
+//     argument (dev_networking mode).
+//
+// t.Fatal on neither resolving — both phases need a usable SSH endpoint.
+func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) {
+	t.Helper()
+	if inst == nil {
+		t.Fatalf("InstancePublicSSHHost: nil instance")
+	}
+	if pub := aws.StringValue(inst.PublicIpAddress); pub != "" && pub != "None" {
+		return pub, 22
+	}
+	id := aws.StringValue(inst.InstanceId)
+	host, port, err := discoverHostfwd(id)
+	if err != nil {
+		t.Fatalf("InstancePublicSSHHost: %v", err)
+	}
+	return host, port
+}
+
+// hostfwd=tcp:HOST:PORT-:22 — HOST may be empty (qemu defaults to 127.0.0.1).
+// Anchor on -:22 so we ignore other hostfwd rules (e.g. metadata, serial).
+var hostfwdRE = regexp.MustCompile(`hostfwd=tcp:([^:]*):([0-9]+)-:22`)
+
+func discoverHostfwd(instanceID string) (string, int, error) {
+	out, err := exec.Command("ps", "-eo", "args=").CombinedOutput()
+	if err != nil {
+		return "", 0, fmt.Errorf("ps: %w", err)
+	}
+	for _, line := range strings.Split(string(out), "\n") {
+		if !strings.Contains(line, instanceID) || !strings.Contains(line, "qemu") {
+			continue
+		}
+		m := hostfwdRE.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		host := m[1]
+		if host == "" {
+			host = "127.0.0.1"
+		}
+		port, err := strconv.Atoi(m[2])
+		if err != nil {
+			return "", 0, fmt.Errorf("parse hostfwd port %q: %w", m[2], err)
+		}
+		return host, port, nil
+	}
+	return "", 0, fmt.Errorf("no qemu hostfwd found for %s", instanceID)
+}
+
+// LsblkRootGiB SSHes into the VM and returns the root disk size in GiB,
+// cross-checking the value the API reports against the guest's view.
+//
+// Equivalent of run-e2e.sh's lsblk pipeline (findmnt → lsblk PKNAME → lsblk
+// -b -d). Returns the GiB rounded down (same math as bash: bytes / 1<<30).
+func LsblkRootGiB(t *testing.T, tgt SSHTarget) int {
+	t.Helper()
+	cmd := `SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"`
+	args := []string{
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "LogLevel=ERROR",
+		"-o", "ConnectTimeout=5",
+		"-o", "BatchMode=yes",
+		"-p", strconv.Itoa(tgt.Port),
+		"-i", tgt.KeyPath,
+		tgt.User + "@" + tgt.Host,
+		cmd,
+	}
+	var stdout, stderr bytes.Buffer
+	sshCmd := exec.Command("ssh", args...)
+	sshCmd.Stdout = &stdout
+	sshCmd.Stderr = &stderr
+	if err := sshCmd.Run(); err != nil {
+		t.Fatalf("ssh lsblk %s@%s:%d failed: %v\nstderr: %s", tgt.User, tgt.Host, tgt.Port, err, stderr.String())
+	}
+	raw := strings.TrimSpace(stdout.String())
+	bytesN, err := strconv.ParseInt(raw, 10, 64)
+	if err != nil {
+		t.Fatalf("ssh lsblk %s@%s:%d: parse %q: %v", tgt.User, tgt.Host, tgt.Port, raw, err)
+	}
+	return int(bytesN / (1 << 30))
+}
+
+// GetSerialConsoleAccessEnabled returns the current account attribute value.
+// Wraps ec2.GetSerialConsoleAccessStatus so callers don't have to deal with
+// the pointer-bool field.
+func GetSerialConsoleAccessEnabled(t *testing.T, c *AWSClient) bool {
+	t.Helper()
+	out, err := c.EC2.GetSerialConsoleAccessStatus(&ec2.GetSerialConsoleAccessStatusInput{})
+	if err != nil {
+		t.Fatalf("GetSerialConsoleAccessStatus: %v", err)
+	}
+	return aws.BoolValue(out.SerialConsoleAccessEnabled)
+}
+
+// EnableSerialConsoleAccess flips the account attribute on and returns the
+// new value (should be true).
+func EnableSerialConsoleAccess(t *testing.T, c *AWSClient) bool {
+	t.Helper()
+	out, err := c.EC2.EnableSerialConsoleAccess(&ec2.EnableSerialConsoleAccessInput{})
+	if err != nil {
+		t.Fatalf("EnableSerialConsoleAccess: %v", err)
+	}
+	return aws.BoolValue(out.SerialConsoleAccessEnabled)
+}
+
+// DisableSerialConsoleAccess flips the account attribute off and returns the
+// new value (should be false).
+func DisableSerialConsoleAccess(t *testing.T, c *AWSClient) bool {
+	t.Helper()
+	out, err := c.EC2.DisableSerialConsoleAccess(&ec2.DisableSerialConsoleAccessInput{})
+	if err != nil {
+		t.Fatalf("DisableSerialConsoleAccess: %v", err)
+	}
+	return aws.BoolValue(out.SerialConsoleAccessEnabled)
+}
+
+// DiscoverDefaultVPC returns (vpcID, defaultSGID, defaultSubnetID) for the
+// account's default VPC. t.Fatal if the default VPC is missing — the suite
+// can't run without one, and silently falling back masks the real failure
+// (daemon didn't create the default VPC on account bootstrap).
+func DiscoverDefaultVPC(t *testing.T, c *AWSClient) (vpcID, sgID, subnetID string) {
+	t.Helper()
+	vpcs, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{
+		Filters: []*ec2.Filter{{Name: aws.String("isDefault"), Values: []*string{aws.String("true")}}},
+	})
+	if err != nil {
+		t.Fatalf("describe-vpcs: %v", err)
+	}
+	if len(vpcs.Vpcs) == 0 {
+		t.Fatalf("no default VPC found")
+	}
+	vpcID = aws.StringValue(vpcs.Vpcs[0].VpcId)
+
+	sgs, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			{Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}},
+			{Name: aws.String("group-name"), Values: []*string{aws.String("default")}},
+		},
+	})
+	if err != nil {
+		t.Fatalf("describe-security-groups: %v", err)
+	}
+	if len(sgs.SecurityGroups) == 0 {
+		t.Fatalf("default SG missing for VPC %s", vpcID)
+	}
+	sgID = aws.StringValue(sgs.SecurityGroups[0].GroupId)
+
+	subnets, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{
+		Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}}},
+	})
+	if err != nil {
+		t.Fatalf("describe-subnets: %v", err)
+	}
+	for _, s := range subnets.Subnets {
+		if aws.BoolValue(s.DefaultForAz) {
+			subnetID = aws.StringValue(s.SubnetId)
+			break
+		}
+	}
+	// Fall back to the first subnet if none is marked DefaultForAz — pseudo
+	// multinode skips the marker, and callers just need a usable subnet ID.
+	if subnetID == "" && len(subnets.Subnets) > 0 {
+		subnetID = aws.StringValue(subnets.Subnets[0].SubnetId)
+	}
+	return vpcID, sgID, subnetID
+}
+
+// AuthorizeSSHIngress idempotently authorizes tcp/22 ingress from 0.0.0.0/0
+// on sgID. InvalidPermission.Duplicate is treated as success — the rule was
+// already in place on a re-run.
+func AuthorizeSSHIngress(t *testing.T, c *AWSClient, sgID string) {
+	t.Helper()
+	_, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
+		GroupId: aws.String(sgID),
+		IpPermissions: []*ec2.IpPermission{{
+			IpProtocol: aws.String("tcp"),
+			FromPort:   aws.Int64(22),
+			ToPort:     aws.Int64(22),
+			IpRanges:   []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
+		}},
+	})
+	if err == nil {
+		return
+	}
+	var aerr awserr.Error
+	if asErr(err, &aerr) && aerr.Code() == "InvalidPermission.Duplicate" {
+		return
+	}
+	t.Fatalf("authorize-security-group-ingress tcp/22 on %s: %v", sgID, err)
+}
+
+// asErr is a thin errors.As that lets the helper stay testify-free.
+func asErr(err error, target *awserr.Error) bool {
+	if a, ok := err.(awserr.Error); ok {
+		*target = a
+		return true
+	}
+	return false
+}

tests/e2e/harness/ovn.go

@@ -0,0 +1,78 @@
+//go:build e2e
+
+package harness
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+// SkipIfNoOVN calls t.Skip when ovn-nbctl isn't on PATH or passwordless sudo
+// isn't available. Developer laptops don't have OVN; CI VMs have both.
+func SkipIfNoOVN(t *testing.T) {
+	t.Helper()
+	if _, err := exec.LookPath("ovn-nbctl"); err != nil {
+		t.Skipf("ovn-nbctl not on PATH (%v); skipping OVN check", err)
+	}
+	// `sudo -n true` returns 0 only if a sudoers entry allows passwordless
+	// for the current user. Anything else (prompt, deny, missing sudo) skips.
+	if err := exec.Command("sudo", "-n", "true").Run(); err != nil {
+		t.Skipf("passwordless sudo unavailable (%v); skipping OVN check", err)
+	}
+}
+
+// OvnNbctl runs `sudo -n ovn-nbctl <args...>`. Returns trimmed stdout on
+// success; t.Fatal on non-zero exit. Wrap in Eventually for propagation polls.
+func OvnNbctl(t *testing.T, args ...string) string {
+	t.Helper()
+	return runOvn(t, "ovn-nbctl", args...)
+}
+
+// OvnSbctl is the SB-database equivalent of OvnNbctl.
+func OvnSbctl(t *testing.T, args ...string) string {
+	t.Helper()
+	return runOvn(t, "ovn-sbctl", args...)
+}
+
+func runOvn(t *testing.T, tool string, args ...string) string {
+	t.Helper()
+	full := append([]string{"-n", tool}, args...)
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("sudo", full...)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("sudo %s %s failed: %v\nstderr: %s", tool, strings.Join(args, " "), err, stderr.String())
+	}
+	return strings.TrimSpace(stdout.String())
+}
+
+// WaitForPortGroupMember polls until the OVN port_group pg contains the
+// logical-switch-port lsp. Default 10s/1s — northd propagation is normally
+// sub-second but tests run on busy nodes.
+//
+// Lookup uses ovn-nbctl --bare --columns=ports find port_group name=<pg>
+// (same shape as run-e2e.sh:3311) and checks for the lsp's UUID in the
+// returned set.
+func WaitForPortGroupMember(t *testing.T, pg, lsp string, opts ...PollOpt) {
+	t.Helper()
+	cfg := applyOpts(pollCfg{timeout: 10 * time.Second, interval: 1 * time.Second}, opts...)
+
+	EventuallyErr(t, func() error {
+		lspUUID := OvnNbctl(t, "--no-leader-only", "--bare", "--columns=_uuid",
+			"find", "logical_switch_port", "name="+lsp)
+		if lspUUID == "" {
+			return fmt.Errorf("LSP %s not found in NB", lsp)
+		}
+		ports := OvnNbctl(t, "--no-leader-only", "--bare", "--columns=ports",
+			"find", "port_group", "name="+pg)
+		if !strings.Contains(ports, lspUUID) {
+			return fmt.Errorf("LSP %s (%s) not in port_group %s (ports=%s)", lsp, lspUUID, pg, ports)
+		}
+		return nil
+	}, cfg.timeout, cfg.interval)
+}

tests/e2e/harness/poll.go

@@ -0,0 +1,156 @@
+//go:build e2e
+
+package harness
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// PollOpt tunes the default timeout / interval for a Wait* helper. Defaults
+// are tuned per-resource based on the bash `sleep N; check` loops in
+// run-e2e.sh — instance/volume = 5min, snapshot/image = 10min, all polled
+// every 2s.
+type PollOpt func(*pollCfg)
+
+type pollCfg struct {
+	timeout  time.Duration
+	interval time.Duration
+}
+
+// WithTimeout overrides the resource-default timeout.
+func WithTimeout(d time.Duration) PollOpt { return func(c *pollCfg) { c.timeout = d } }
+
+// WithPoll overrides the resource-default polling interval.
+func WithPoll(d time.Duration) PollOpt { return func(c *pollCfg) { c.interval = d } }
+
+func applyOpts(def pollCfg, opts ...PollOpt) pollCfg {
+	for _, o := range opts {
+		o(&def)
+	}
+	return def
+}
+
+// WaitForInstanceState polls DescribeInstances until State.Name == target.
+// Returns the latest *ec2.Instance on success; t.Fatal on timeout. Separate
+// from lb.go's WaitForInstanceRunning so callers can wait for stopped /
+// terminated / pending without one helper per state.
+func WaitForInstanceState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Instance {
+	t.Helper()
+	cfg := applyOpts(pollCfg{timeout: 5 * time.Minute, interval: 2 * time.Second}, opts...)
+	var last *ec2.Instance
+	var lastState string
+	EventuallyErr(t, func() error {
+		out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
+			InstanceIds: []*string{aws.String(id)},
+		})
+		if err != nil {
+			return fmt.Errorf("describe %s: %w", id, err)
+		}
+		if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
+			return fmt.Errorf("%s not found", id)
+		}
+		last = out.Reservations[0].Instances[0]
+		lastState = aws.StringValue(last.State.Name)
+		if lastState == target {
+			return nil
+		}
+		return fmt.Errorf("%s state=%s want=%s", id, lastState, target)
+	}, cfg.timeout, cfg.interval)
+	t.Logf("instance %s reached state %s", id, target)
+	return last
+}
+
+// WaitForVolumeState polls DescribeVolumes until State == target. Common
+// targets: "available", "in-use", "deleted".
+func WaitForVolumeState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Volume {
+	t.Helper()
+	cfg := applyOpts(pollCfg{timeout: 5 * time.Minute, interval: 2 * time.Second}, opts...)
+	var last *ec2.Volume
+	var lastState string
+	EventuallyErr(t, func() error {
+		out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{
+			VolumeIds: []*string{aws.String(id)},
+		})
+		if err != nil {
+			return fmt.Errorf("describe-volume %s: %w", id, err)
+		}
+		if len(out.Volumes) == 0 {
+			return fmt.Errorf("%s not found", id)
+		}
+		last = out.Volumes[0]
+		lastState = aws.StringValue(last.State)
+		if lastState == target {
+			return nil
+		}
+		return fmt.Errorf("%s state=%s want=%s", id, lastState, target)
+	}, cfg.timeout, cfg.interval)
+	t.Logf("volume %s reached state %s", id, target)
+	return last
+}
+
+// WaitForSnapshotState polls DescribeSnapshots until State == target. Snapshot
+// creation can be slow on busy nodes — default timeout 10min.
+func WaitForSnapshotState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Snapshot {
+	t.Helper()
+	cfg := applyOpts(pollCfg{timeout: 10 * time.Minute, interval: 2 * time.Second}, opts...)
+	var last *ec2.Snapshot
+	var lastState string
+	EventuallyErr(t, func() error {
+		out, err := c.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
+			SnapshotIds: []*string{aws.String(id)},
+		})
+		if err != nil {
+			return fmt.Errorf("describe-snapshot %s: %w", id, err)
+		}
+		if len(out.Snapshots) == 0 {
+			return fmt.Errorf("%s not found", id)
+		}
+		last = out.Snapshots[0]
+		lastState = aws.StringValue(last.State)
+		if lastState == target {
+			return nil
+		}
+		if lastState == "error" {
+			return fmt.Errorf("%s entered error state: %s", id, aws.StringValue(last.StateMessage))
+		}
+		return fmt.Errorf("%s state=%s want=%s", id, lastState, target)
+	}, cfg.timeout, cfg.interval)
+	t.Logf("snapshot %s reached state %s", id, target)
+	return last
+}
+
+// WaitForImageState polls DescribeImages until State == target. AMI creation
+// from a snapshot is the slowest path; default timeout 10min.
+func WaitForImageState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Image {
+	t.Helper()
+	cfg := applyOpts(pollCfg{timeout: 10 * time.Minute, interval: 2 * time.Second}, opts...)
+	var last *ec2.Image
+	var lastState string
+	EventuallyErr(t, func() error {
+		out, err := c.EC2.DescribeImages(&ec2.DescribeImagesInput{
+			ImageIds: []*string{aws.String(id)},
+		})
+		if err != nil {
+			return fmt.Errorf("describe-image %s: %w", id, err)
+		}
+		if len(out.Images) == 0 {
+			return fmt.Errorf("%s not found", id)
+		}
+		last = out.Images[0]
+		lastState = aws.StringValue(last.State)
+		if lastState == target {
+			return nil
+		}
+		if lastState == "failed" {
+			return fmt.Errorf("%s entered failed state: %s", id, aws.StringValue(last.StateReason.Message))
+		}
+		return fmt.Errorf("%s state=%s want=%s", id, lastState, target)
+	}, cfg.timeout, cfg.interval)
+	t.Logf("image %s reached state %s", id, target)
+	return last
+}

tests/e2e/harness/rawhttp.go

@@ -0,0 +1,123 @@
+//go:build e2e
+
+package harness
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+)
+
+// PostAWSAction sends a raw application/x-www-form-urlencoded POST with
+// Action=<action> + Version=2016-11-15 + caller params, signed with the same
+// credentials the AWS SDK is using. Gateway rejects on auth before parsing
+// the action, so signing is mandatory even for InvalidAction probes.
+//
+// Returns the HTTP status, raw body, and the parsed <Code> from the AWS XML
+// error envelope (empty for 2xx). t.Fatal on transport / signing failure.
+func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params map[string]string) (status int, body []byte, awsCode string) {
+	t.Helper()
+
+	endpoint := c.EC2.Endpoint
+	if endpoint == "" {
+		t.Fatalf("PostAWSAction: EC2 endpoint not configured")
+	}
+	region := aws.StringValue(c.EC2.Config.Region)
+
+	form := url.Values{}
+	form.Set("Action", action)
+	if _, ok := params["Version"]; !ok {
+		// EC2 query API version — same value the SDK injects on every call.
+		// Hard-coded here because we are bypassing the SDK entirely.
+		form.Set("Version", "2016-11-15")
+	}
+	for k, v := range params {
+		form.Set(k, v)
+	}
+	bodyBytes := []byte(form.Encode())
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(endpoint, "/")+"/", bytes.NewReader(bodyBytes))
+	if err != nil {
+		t.Fatalf("PostAWSAction: build request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
+
+	creds := c.EC2.Config.Credentials
+	if creds == nil {
+		t.Fatalf("PostAWSAction: no credentials on client")
+	}
+	signer := v4.NewSigner(creds)
+	// SignedReader keeps the body intact for replay; v4.Sign reads-then-resets.
+	if _, err := signer.Sign(req, bytes.NewReader(bodyBytes), "ec2", region, time.Now()); err != nil {
+		t.Fatalf("PostAWSAction: sign: %v", err)
+	}
+
+	// Reuse the EC2 client's HTTP transport so TLS trust matches the rest of
+	// the suite. Fall back to a CA-loaded transport if the embedded client
+	// is using the SDK default.
+	httpClient := c.EC2.Config.HTTPClient
+	if httpClient == nil {
+		httpClient = &http.Client{
+			Transport: &http.Transport{TLSClientConfig: &tls.Config{}},
+			Timeout:   env.DefaultTimeout,
+		}
+	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		t.Fatalf("PostAWSAction: %v", err)
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("PostAWSAction: read body: %v", err)
+	}
+	status = resp.StatusCode
+
+	if status >= 200 && status < 300 {
+		return status, body, ""
+	}
+	awsCode = parseAWSXMLErrorCode(body)
+	return status, body, awsCode
+}
+
+// EC2 returns either <Response><Errors><Error><Code>…</Code> or the SDK-style
+// <ErrorResponse><Error><Code>…</Code> envelope. Try both shapes; the
+// gateway has shipped one then the other across releases.
+type ec2ErrorResponse struct {
+	XMLName xml.Name `xml:"Response"`
+	Errors  struct {
+		Error struct {
+			Code    string `xml:"Code"`
+			Message string `xml:"Message"`
+		} `xml:"Error"`
+	} `xml:"Errors"`
+}
+
+type sdkErrorResponse struct {
+	XMLName xml.Name `xml:"ErrorResponse"`
+	Error   struct {
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	} `xml:"Error"`
+}
+
+func parseAWSXMLErrorCode(body []byte) string {
+	var e1 ec2ErrorResponse
+	if err := xml.Unmarshal(body, &e1); err == nil && e1.Errors.Error.Code != "" {
+		return e1.Errors.Error.Code
+	}
+	var e2 sdkErrorResponse
+	if err := xml.Unmarshal(body, &e2); err == nil && e2.Error.Code != "" {
+		return e2.Error.Code
+	}
+	return ""
+}

tests/e2e/harness/spxcli.go

@@ -0,0 +1,115 @@
+//go:build e2e
+
+package harness
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// AccountInfo is the parsed result of `spx admin account create`. Fields
+// match the lines emitted by cmd/spinifex/cmd/admin.go:runAccountCreate.
+type AccountInfo struct {
+	AccountID       string
+	AccountName     string
+	AccessKeyID     string
+	SecretAccessKey string
+	Profile         string
+}
+
+// SpxBin returns the spx binary path (SPX_BIN env var or "spx" on PATH).
+func SpxBin() string {
+	if v := os.Getenv("SPX_BIN"); v != "" {
+		return v
+	}
+	return "spx"
+}
+
+// SpxRun runs `spx <args...>` and returns combined stdout+stderr. If wantErr
+// is false the test fails on non-zero exit; if true the call returns the
+// output and the test continues (caller asserts on output content).
+func SpxRun(t *testing.T, wantErr bool, args ...string) string {
+	t.Helper()
+	var buf bytes.Buffer
+	cmd := exec.Command(SpxBin(), args...)
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	err := cmd.Run()
+	out := buf.String()
+	if err != nil && !wantErr {
+		t.Fatalf("spx %s failed: %v\noutput:\n%s", strings.Join(args, " "), err, out)
+	}
+	if err == nil && wantErr {
+		t.Fatalf("spx %s expected non-zero exit, got success\noutput:\n%s", strings.Join(args, " "), out)
+	}
+	return out
+}
+
+// SpxGetNodes runs `spx get nodes`.
+func SpxGetNodes(t *testing.T) string {
+	t.Helper()
+	return SpxRun(t, false, "get", "nodes")
+}
+
+// SpxGetVMs runs `spx get vms`.
+func SpxGetVMs(t *testing.T) string {
+	t.Helper()
+	return SpxRun(t, false, "get", "vms")
+}
+
+// SpxTopNodes runs `spx top nodes`.
+func SpxTopNodes(t *testing.T) string {
+	t.Helper()
+	return SpxRun(t, false, "top", "nodes")
+}
+
+// admin account create output is a fixed two-column key/value list; pin each
+// field to a line anchor so a layout change (added/reordered keys) surfaces
+// loudly instead of silently matching the wrong value.
+var (
+	spxAcctRE   = regexp.MustCompile(`(?m)^\s+Account ID:\s+(\S+)`)
+	spxNameRE   = regexp.MustCompile(`(?m)^\s+Account Name:\s+(.+?)\s*$`)
+	spxKeyIDRE  = regexp.MustCompile(`(?m)^\s+Access Key ID:\s+(\S+)`)
+	spxSecretRE = regexp.MustCompile(`(?m)^\s+Secret Access Key:\s+(\S+)`)
+	spxProfRE   = regexp.MustCompile(`(?m)^\s+AWS Profile:\s+(\S+)`)
+)
+
+// SpxAdminAccountCreate creates a tenant account via the spx CLI and parses
+// the printed credentials block. email is forwarded as --email when non-empty;
+// `spx admin account create` ignores unknown flags today, but the CLI may
+// adopt --email in future and any caller that wants to gate on it (e.g. IAM
+// e-mail uniqueness checks) should pass it through unchanged.
+func SpxAdminAccountCreate(t *testing.T, name, email string) AccountInfo {
+	t.Helper()
+	args := []string{"admin", "account", "create", "--name", name}
+	if email != "" {
+		args = append(args, "--email", email)
+	}
+	out := SpxRun(t, false, args...)
+
+	info := AccountInfo{
+		AccountID:       firstSubmatch(spxAcctRE, out),
+		AccountName:     firstSubmatch(spxNameRE, out),
+		AccessKeyID:     firstSubmatch(spxKeyIDRE, out),
+		SecretAccessKey: firstSubmatch(spxSecretRE, out),
+		Profile:         firstSubmatch(spxProfRE, out),
+	}
+	if info.AccountID == "" || info.AccessKeyID == "" || info.SecretAccessKey == "" {
+		// Dump raw output to the artifact dir if available — otherwise the
+		// formatted log block lives only in the JUnit XML.
+		t.Fatalf("spx admin account create: failed to parse credentials block\n--- raw output ---\n%s\n--- end ---", out)
+	}
+	return info
+}
+
+func firstSubmatch(re *regexp.Regexp, s string) string {
+	m := re.FindStringSubmatch(s)
+	if len(m) < 2 {
+		return ""
+	}
+	return strings.TrimSpace(m[1])
+}