From 23143bd0ded120d2fdf2916f0d1333fc6bc26aed Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Mon, 18 May 2026 18:28:02 +1000 Subject: [PATCH] feat(e2e-harness): extend harness for single-node Go suite port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage A of mulga-siv-89 — add the primitives the single-node run-e2e.sh port consumes: - AWSClient gains IAM + STS service handles, plus NewAWSClientWithCreds for per-tenant account-scoping in Phase 8. - awserror.go: ErrorCodeIs, AssertAWSError, ExpectError — Go equivalent of the bash expect_error helper used throughout the negative phases. - poll.go: WaitForInstance/Volume/Snapshot/Image state to any target, with per-resource default timeouts tuned from the bash sleep loops. - spxcli.go: thin shellouts for spx get nodes/vms, spx top nodes, spx admin account create with output parsing for tenant-id/access-key. - rawhttp.go: signed raw POST against the AWS gateway for the Phase 8 InvalidAction probe (gateway rejects on auth before action parsing). - ovn.go: OvnNbctl/OvnSbctl wrappers + SkipIfNoOVN for Phase 8b-e datapath checks; WaitForPortGroupMember handles propagation polls. - ec2helpers.go: InstancePublicSSHHost (public-IP-or-qemu-hostfwd), LsblkRootGiB, serial-console-access trio, DiscoverDefaultVPC, idempotent AuthorizeSSHIngress. - account.go: AccountCarousel maps tenant name -> scoped AWSClient, built atop NewAWSClientWithCreds for the Phase 8 isolation tests. Pure groundwork — no new tests, no production behaviour change. The cert + lb scenarios still compile against the extended harness. --- tests/e2e/harness/account.go | 75 ++++++++++ tests/e2e/harness/aws.go | 47 +++++-- tests/e2e/harness/awserror.go | 49 +++++++ tests/e2e/harness/ec2helpers.go | 236 ++++++++++++++++++++++++++++++++ tests/e2e/harness/ovn.go | 78 +++++++++++ tests/e2e/harness/poll.go | 156 +++++++++++++++++++++ tests/e2e/harness/rawhttp.go | 123 +++++++++++++++++ tests/e2e/harness/spxcli.go | 115 ++++++++++++++++ 8 files changed, 870 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/harness/account.go create mode 100644 tests/e2e/harness/awserror.go create mode 100644 tests/e2e/harness/ec2helpers.go create mode 100644 tests/e2e/harness/ovn.go create mode 100644 tests/e2e/harness/poll.go create mode 100644 tests/e2e/harness/rawhttp.go create mode 100644 tests/e2e/harness/spxcli.go diff --git a/tests/e2e/harness/account.go b/tests/e2e/harness/account.go new file mode 100644 index 000000000..0c11487fd --- /dev/null +++ b/tests/e2e/harness/account.go @@ -0,0 +1,75 @@ +//go:build e2e + +package harness + +import ( + "sort" + "sync" + "testing" +) + +// Profile bundles a tenant account's credentials with a pre-built AWSClient +// scoped to that account. The Client field bypasses AWS_PROFILE shared-config +// lookup — credentials are injected statically via NewAWSClientWithCreds so a +// single test binary can drive many tenants concurrently without mutating +// ~/.aws/credentials. +type Profile struct { + Name string + AccountID string + Client *AWSClient + Info AccountInfo +} + +// AccountCarousel manages multiple Profile entries created via +// `spx admin account create`. Used by Phase 8 account-scoping where the test +// repeatedly switches between Alpha/Beta/Gamma to assert isolation. +// +// Concurrent reads via Get/Names are safe; Add must not race with itself. +type AccountCarousel struct { + mu sync.RWMutex + profiles map[string]*Profile +} + +// NewAccountCarousel returns an empty carousel. +func NewAccountCarousel() *AccountCarousel { + return &AccountCarousel{profiles: make(map[string]*Profile)} +} + +// Add registers a new profile under the given name (e.g. "team-alpha") and +// returns it. Duplicate names overwrite — caller is expected to use unique +// labels per tenant. info.AccessKeyID and info.SecretAccessKey must be set; +// NewAWSClientWithCreds t.Fatals otherwise. +func (a *AccountCarousel) Add(t *testing.T, env *Env, name string, info AccountInfo) *Profile { + t.Helper() + client := NewAWSClientWithCreds(t, env, info.AccessKeyID, info.SecretAccessKey) + p := &Profile{ + Name: name, + AccountID: info.AccountID, + Client: client, + Info: info, + } + a.mu.Lock() + a.profiles[name] = p + a.mu.Unlock() + return p +} + +// Get returns the named profile or nil if absent. +func (a *AccountCarousel) Get(name string) *Profile { + a.mu.RLock() + defer a.mu.RUnlock() + return a.profiles[name] +} + +// Names returns the registered profile names in sorted order — stable for +// snapshot-style logging without depending on map iteration order. +func (a *AccountCarousel) Names() []string { + a.mu.RLock() + defer a.mu.RUnlock() + out := make([]string, 0, len(a.profiles)) + for k := range a.profiles { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/tests/e2e/harness/aws.go b/tests/e2e/harness/aws.go index 609091d33..781d72885 100644 --- a/tests/e2e/harness/aws.go +++ b/tests/e2e/harness/aws.go @@ -15,6 +15,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/sts" ) // AWSClient bundles the SDK service handles a scenario typically needs against @@ -23,6 +25,8 @@ import ( type AWSClient struct { EC2 *ec2.EC2 ELBv2 *elbv2.ELBV2 + IAM *iam.IAM + STS *sts.STS } // NewAWSClient builds clients pointed at https://:/ using the @@ -31,6 +35,23 @@ type AWSClient struct { // SPINIFEX_AWS_ACCESS_KEY_ID / SPINIFEX_AWS_SECRET_ACCESS_KEY for static creds. func NewAWSClient(t *testing.T, env *Env) *AWSClient { t.Helper() + id, secret := os.Getenv("SPINIFEX_AWS_ACCESS_KEY_ID"), os.Getenv("SPINIFEX_AWS_SECRET_ACCESS_KEY") + return newAWSClient(t, env, id, secret) +} + +// NewAWSClientWithCreds builds an AWSClient with explicit static credentials +// — used by AccountCarousel to scope a client to a tenant account created +// via `spx admin account create`. Bypasses AWS_PROFILE shared-config lookup. +func NewAWSClientWithCreds(t *testing.T, env *Env, accessKey, secretKey string) *AWSClient { + t.Helper() + if accessKey == "" || secretKey == "" { + t.Fatalf("NewAWSClientWithCreds: empty credentials") + } + return newAWSClient(t, env, accessKey, secretKey) +} + +func newAWSClient(t *testing.T, env *Env, accessKey, secretKey string) *AWSClient { + t.Helper() endpoint := os.Getenv("SPINIFEX_AWS_ENDPOINT") if endpoint == "" { @@ -62,15 +83,18 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient { }, } - if id, secret := os.Getenv("SPINIFEX_AWS_ACCESS_KEY_ID"), os.Getenv("SPINIFEX_AWS_SECRET_ACCESS_KEY"); id != "" && secret != "" { - cfg.Credentials = credentials.NewStaticCredentials(id, secret, "") + opts := session.Options{Config: *cfg} + if accessKey != "" && secretKey != "" { + // Static creds bypass shared-config lookup — required for the + // per-tenant carousel where each Profile holds its own access key. + cfg.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + opts.Config = *cfg + } else { + opts.SharedConfigState = session.SharedConfigEnable + opts.Profile = getenv("AWS_PROFILE", "spinifex") } - sess, err := session.NewSessionWithOptions(session.Options{ - Config: *cfg, - SharedConfigState: session.SharedConfigEnable, - Profile: getenv("AWS_PROFILE", "spinifex"), - }) + sess, err := session.NewSessionWithOptions(opts) if err != nil { t.Fatalf("AWS session: %v", err) } @@ -78,6 +102,8 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient { return &AWSClient{ EC2: ec2.New(sess), ELBv2: elbv2.New(sess), + IAM: iam.New(sess), + STS: sts.New(sess), } } @@ -85,7 +111,10 @@ func NewAWSClient(t *testing.T, env *Env) *AWSClient { // for fault-injection scenarios — the cert scenario already covers trust path. func (c *AWSClient) IgnoreCertErrors() { tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - c.EC2.Config.HTTPClient = &http.Client{Transport: tr} - c.ELBv2.Config.HTTPClient = &http.Client{Transport: tr} + hc := &http.Client{Transport: tr} + c.EC2.Config.HTTPClient = hc + c.ELBv2.Config.HTTPClient = hc + c.IAM.Config.HTTPClient = hc + c.STS.Config.HTTPClient = hc _ = (*x509.CertPool)(nil) // silence unused-import on hardened paths } diff --git a/tests/e2e/harness/awserror.go b/tests/e2e/harness/awserror.go new file mode 100644 index 000000000..0db2b66f6 --- /dev/null +++ b/tests/e2e/harness/awserror.go @@ -0,0 +1,49 @@ +//go:build e2e + +package harness + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws/awserr" +) + +// ErrorCodeIs reports whether err is an awserr.Error with the given Code. +// Returns false for nil err or non-awserr wrappers. +func ErrorCodeIs(err error, code string) bool { + if err == nil { + return false + } + var aerr awserr.Error + if errors.As(err, &aerr) { + return aerr.Code() == code + } + return false +} + +// AssertAWSError fails the test if err is nil or its awserr.Code() != code. +// Logs the full err on mismatch. Returns the matched awserr.Error so callers +// can chain assertions on Message() etc. +func AssertAWSError(t *testing.T, err error, code string) awserr.Error { + t.Helper() + if err == nil { + t.Fatalf("expected AWS error %q, got nil", code) + } + var aerr awserr.Error + if !errors.As(err, &aerr) { + t.Fatalf("expected awserr.Error with code %q, got %T: %v", code, err, err) + } + if aerr.Code() != code { + t.Fatalf("expected AWS error code %q, got %q (full: %v)", code, aerr.Code(), err) + } + return aerr +} + +// ExpectError invokes fn and asserts the returned err has the given AWS code. +// Convenience for one-shot negative tests — replaces the bash `expect_error` +// helper used throughout run-e2e.sh. +func ExpectError(t *testing.T, code string, fn func() error) { + t.Helper() + AssertAWSError(t, fn(), code) +} diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go new file mode 100644 index 000000000..f9772ee80 --- /dev/null +++ b/tests/e2e/harness/ec2helpers.go @@ -0,0 +1,236 @@ +//go:build e2e + +package harness + +import ( + "bytes" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// SSHTarget is the minimal addressing tuple needed by LsblkRootGiB. The +// existing harness.SSH interface requires a Cluster.Node, but the single-host +// e2e suite addresses individual instances by (host, port, key) discovered at +// runtime — those don't fit a Cluster. Defined here rather than in ssh.go to +// keep the production SSH transport untouched. +type SSHTarget struct { + User string + Host string + Port int + KeyPath string +} + +// InstancePublicSSHHost returns (host, port) for an SSH connection to inst. +// +// Preference order matches run-e2e.sh: +// 1. Instance's public IP on port 22, when assigned (external networking). +// 2. QEMU hostfwd discovered by scanning `ps -ef` for the qemu process whose +// command line contains the instance ID and a `hostfwd=tcp::-:22` +// argument (dev_networking mode). +// +// t.Fatal on neither resolving — both phases need a usable SSH endpoint. +func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) { + t.Helper() + if inst == nil { + t.Fatalf("InstancePublicSSHHost: nil instance") + } + if pub := aws.StringValue(inst.PublicIpAddress); pub != "" && pub != "None" { + return pub, 22 + } + id := aws.StringValue(inst.InstanceId) + host, port, err := discoverHostfwd(id) + if err != nil { + t.Fatalf("InstancePublicSSHHost: %v", err) + } + return host, port +} + +// hostfwd=tcp:HOST:PORT-:22 — HOST may be empty (qemu defaults to 127.0.0.1). +// Anchor on -:22 so we ignore other hostfwd rules (e.g. metadata, serial). +var hostfwdRE = regexp.MustCompile(`hostfwd=tcp:([^:]*):([0-9]+)-:22`) + +func discoverHostfwd(instanceID string) (string, int, error) { + out, err := exec.Command("ps", "-eo", "args=").CombinedOutput() + if err != nil { + return "", 0, fmt.Errorf("ps: %w", err) + } + for _, line := range strings.Split(string(out), "\n") { + if !strings.Contains(line, instanceID) || !strings.Contains(line, "qemu") { + continue + } + m := hostfwdRE.FindStringSubmatch(line) + if m == nil { + continue + } + host := m[1] + if host == "" { + host = "127.0.0.1" + } + port, err := strconv.Atoi(m[2]) + if err != nil { + return "", 0, fmt.Errorf("parse hostfwd port %q: %w", m[2], err) + } + return host, port, nil + } + return "", 0, fmt.Errorf("no qemu hostfwd found for %s", instanceID) +} + +// LsblkRootGiB SSHes into the VM and returns the root disk size in GiB, +// cross-checking the value the API reports against the guest's view. +// +// Equivalent of run-e2e.sh's lsblk pipeline (findmnt → lsblk PKNAME → lsblk +// -b -d). Returns the GiB rounded down (same math as bash: bytes / 1<<30). +func LsblkRootGiB(t *testing.T, tgt SSHTarget) int { + t.Helper() + cmd := `SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"` + args := []string{ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-p", strconv.Itoa(tgt.Port), + "-i", tgt.KeyPath, + tgt.User + "@" + tgt.Host, + cmd, + } + var stdout, stderr bytes.Buffer + sshCmd := exec.Command("ssh", args...) + sshCmd.Stdout = &stdout + sshCmd.Stderr = &stderr + if err := sshCmd.Run(); err != nil { + t.Fatalf("ssh lsblk %s@%s:%d failed: %v\nstderr: %s", tgt.User, tgt.Host, tgt.Port, err, stderr.String()) + } + raw := strings.TrimSpace(stdout.String()) + bytesN, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + t.Fatalf("ssh lsblk %s@%s:%d: parse %q: %v", tgt.User, tgt.Host, tgt.Port, raw, err) + } + return int(bytesN / (1 << 30)) +} + +// GetSerialConsoleAccessEnabled returns the current account attribute value. +// Wraps ec2.GetSerialConsoleAccessStatus so callers don't have to deal with +// the pointer-bool field. +func GetSerialConsoleAccessEnabled(t *testing.T, c *AWSClient) bool { + t.Helper() + out, err := c.EC2.GetSerialConsoleAccessStatus(&ec2.GetSerialConsoleAccessStatusInput{}) + if err != nil { + t.Fatalf("GetSerialConsoleAccessStatus: %v", err) + } + return aws.BoolValue(out.SerialConsoleAccessEnabled) +} + +// EnableSerialConsoleAccess flips the account attribute on and returns the +// new value (should be true). +func EnableSerialConsoleAccess(t *testing.T, c *AWSClient) bool { + t.Helper() + out, err := c.EC2.EnableSerialConsoleAccess(&ec2.EnableSerialConsoleAccessInput{}) + if err != nil { + t.Fatalf("EnableSerialConsoleAccess: %v", err) + } + return aws.BoolValue(out.SerialConsoleAccessEnabled) +} + +// DisableSerialConsoleAccess flips the account attribute off and returns the +// new value (should be false). +func DisableSerialConsoleAccess(t *testing.T, c *AWSClient) bool { + t.Helper() + out, err := c.EC2.DisableSerialConsoleAccess(&ec2.DisableSerialConsoleAccessInput{}) + if err != nil { + t.Fatalf("DisableSerialConsoleAccess: %v", err) + } + return aws.BoolValue(out.SerialConsoleAccessEnabled) +} + +// DiscoverDefaultVPC returns (vpcID, defaultSGID, defaultSubnetID) for the +// account's default VPC. t.Fatal if the default VPC is missing — the suite +// can't run without one, and silently falling back masks the real failure +// (daemon didn't create the default VPC on account bootstrap). +func DiscoverDefaultVPC(t *testing.T, c *AWSClient) (vpcID, sgID, subnetID string) { + t.Helper() + vpcs, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ + Filters: []*ec2.Filter{{Name: aws.String("isDefault"), Values: []*string{aws.String("true")}}}, + }) + if err != nil { + t.Fatalf("describe-vpcs: %v", err) + } + if len(vpcs.Vpcs) == 0 { + t.Fatalf("no default VPC found") + } + vpcID = aws.StringValue(vpcs.Vpcs[0].VpcId) + + sgs, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}}, + {Name: aws.String("group-name"), Values: []*string{aws.String("default")}}, + }, + }) + if err != nil { + t.Fatalf("describe-security-groups: %v", err) + } + if len(sgs.SecurityGroups) == 0 { + t.Fatalf("default SG missing for VPC %s", vpcID) + } + sgID = aws.StringValue(sgs.SecurityGroups[0].GroupId) + + subnets, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}}}, + }) + if err != nil { + t.Fatalf("describe-subnets: %v", err) + } + for _, s := range subnets.Subnets { + if aws.BoolValue(s.DefaultForAz) { + subnetID = aws.StringValue(s.SubnetId) + break + } + } + // Fall back to the first subnet if none is marked DefaultForAz — pseudo + // multinode skips the marker, and callers just need a usable subnet ID. + if subnetID == "" && len(subnets.Subnets) > 0 { + subnetID = aws.StringValue(subnets.Subnets[0].SubnetId) + } + return vpcID, sgID, subnetID +} + +// AuthorizeSSHIngress idempotently authorizes tcp/22 ingress from 0.0.0.0/0 +// on sgID. InvalidPermission.Duplicate is treated as success — the rule was +// already in place on a re-run. +func AuthorizeSSHIngress(t *testing.T, c *AWSClient, sgID string) { + t.Helper() + _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(sgID), + IpPermissions: []*ec2.IpPermission{{ + IpProtocol: aws.String("tcp"), + FromPort: aws.Int64(22), + ToPort: aws.Int64(22), + IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}}, + }}, + }) + if err == nil { + return + } + var aerr awserr.Error + if asErr(err, &aerr) && aerr.Code() == "InvalidPermission.Duplicate" { + return + } + t.Fatalf("authorize-security-group-ingress tcp/22 on %s: %v", sgID, err) +} + +// asErr is a thin errors.As that lets the helper stay testify-free. +func asErr(err error, target *awserr.Error) bool { + if a, ok := err.(awserr.Error); ok { + *target = a + return true + } + return false +} diff --git a/tests/e2e/harness/ovn.go b/tests/e2e/harness/ovn.go new file mode 100644 index 000000000..fb8533a9b --- /dev/null +++ b/tests/e2e/harness/ovn.go @@ -0,0 +1,78 @@ +//go:build e2e + +package harness + +import ( + "bytes" + "fmt" + "os/exec" + "strings" + "testing" + "time" +) + +// SkipIfNoOVN calls t.Skip when ovn-nbctl isn't on PATH or passwordless sudo +// isn't available. Developer laptops don't have OVN; CI VMs have both. +func SkipIfNoOVN(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("ovn-nbctl"); err != nil { + t.Skipf("ovn-nbctl not on PATH (%v); skipping OVN check", err) + } + // `sudo -n true` returns 0 only if a sudoers entry allows passwordless + // for the current user. Anything else (prompt, deny, missing sudo) skips. + if err := exec.Command("sudo", "-n", "true").Run(); err != nil { + t.Skipf("passwordless sudo unavailable (%v); skipping OVN check", err) + } +} + +// OvnNbctl runs `sudo -n ovn-nbctl `. Returns trimmed stdout on +// success; t.Fatal on non-zero exit. Wrap in Eventually for propagation polls. +func OvnNbctl(t *testing.T, args ...string) string { + t.Helper() + return runOvn(t, "ovn-nbctl", args...) +} + +// OvnSbctl is the SB-database equivalent of OvnNbctl. +func OvnSbctl(t *testing.T, args ...string) string { + t.Helper() + return runOvn(t, "ovn-sbctl", args...) +} + +func runOvn(t *testing.T, tool string, args ...string) string { + t.Helper() + full := append([]string{"-n", tool}, args...) + var stdout, stderr bytes.Buffer + cmd := exec.Command("sudo", full...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("sudo %s %s failed: %v\nstderr: %s", tool, strings.Join(args, " "), err, stderr.String()) + } + return strings.TrimSpace(stdout.String()) +} + +// WaitForPortGroupMember polls until the OVN port_group pg contains the +// logical-switch-port lsp. Default 10s/1s — northd propagation is normally +// sub-second but tests run on busy nodes. +// +// Lookup uses ovn-nbctl --bare --columns=ports find port_group name= +// (same shape as run-e2e.sh:3311) and checks for the lsp's UUID in the +// returned set. +func WaitForPortGroupMember(t *testing.T, pg, lsp string, opts ...PollOpt) { + t.Helper() + cfg := applyOpts(pollCfg{timeout: 10 * time.Second, interval: 1 * time.Second}, opts...) + + EventuallyErr(t, func() error { + lspUUID := OvnNbctl(t, "--no-leader-only", "--bare", "--columns=_uuid", + "find", "logical_switch_port", "name="+lsp) + if lspUUID == "" { + return fmt.Errorf("LSP %s not found in NB", lsp) + } + ports := OvnNbctl(t, "--no-leader-only", "--bare", "--columns=ports", + "find", "port_group", "name="+pg) + if !strings.Contains(ports, lspUUID) { + return fmt.Errorf("LSP %s (%s) not in port_group %s (ports=%s)", lsp, lspUUID, pg, ports) + } + return nil + }, cfg.timeout, cfg.interval) +} diff --git a/tests/e2e/harness/poll.go b/tests/e2e/harness/poll.go new file mode 100644 index 000000000..42bee7146 --- /dev/null +++ b/tests/e2e/harness/poll.go @@ -0,0 +1,156 @@ +//go:build e2e + +package harness + +import ( + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// PollOpt tunes the default timeout / interval for a Wait* helper. Defaults +// are tuned per-resource based on the bash `sleep N; check` loops in +// run-e2e.sh — instance/volume = 5min, snapshot/image = 10min, all polled +// every 2s. +type PollOpt func(*pollCfg) + +type pollCfg struct { + timeout time.Duration + interval time.Duration +} + +// WithTimeout overrides the resource-default timeout. +func WithTimeout(d time.Duration) PollOpt { return func(c *pollCfg) { c.timeout = d } } + +// WithPoll overrides the resource-default polling interval. +func WithPoll(d time.Duration) PollOpt { return func(c *pollCfg) { c.interval = d } } + +func applyOpts(def pollCfg, opts ...PollOpt) pollCfg { + for _, o := range opts { + o(&def) + } + return def +} + +// WaitForInstanceState polls DescribeInstances until State.Name == target. +// Returns the latest *ec2.Instance on success; t.Fatal on timeout. Separate +// from lb.go's WaitForInstanceRunning so callers can wait for stopped / +// terminated / pending without one helper per state. +func WaitForInstanceState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Instance { + t.Helper() + cfg := applyOpts(pollCfg{timeout: 5 * time.Minute, interval: 2 * time.Second}, opts...) + var last *ec2.Instance + var lastState string + EventuallyErr(t, func() error { + out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + if err != nil { + return fmt.Errorf("describe %s: %w", id, err) + } + if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 { + return fmt.Errorf("%s not found", id) + } + last = out.Reservations[0].Instances[0] + lastState = aws.StringValue(last.State.Name) + if lastState == target { + return nil + } + return fmt.Errorf("%s state=%s want=%s", id, lastState, target) + }, cfg.timeout, cfg.interval) + t.Logf("instance %s reached state %s", id, target) + return last +} + +// WaitForVolumeState polls DescribeVolumes until State == target. Common +// targets: "available", "in-use", "deleted". +func WaitForVolumeState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Volume { + t.Helper() + cfg := applyOpts(pollCfg{timeout: 5 * time.Minute, interval: 2 * time.Second}, opts...) + var last *ec2.Volume + var lastState string + EventuallyErr(t, func() error { + out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{ + VolumeIds: []*string{aws.String(id)}, + }) + if err != nil { + return fmt.Errorf("describe-volume %s: %w", id, err) + } + if len(out.Volumes) == 0 { + return fmt.Errorf("%s not found", id) + } + last = out.Volumes[0] + lastState = aws.StringValue(last.State) + if lastState == target { + return nil + } + return fmt.Errorf("%s state=%s want=%s", id, lastState, target) + }, cfg.timeout, cfg.interval) + t.Logf("volume %s reached state %s", id, target) + return last +} + +// WaitForSnapshotState polls DescribeSnapshots until State == target. Snapshot +// creation can be slow on busy nodes — default timeout 10min. +func WaitForSnapshotState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Snapshot { + t.Helper() + cfg := applyOpts(pollCfg{timeout: 10 * time.Minute, interval: 2 * time.Second}, opts...) + var last *ec2.Snapshot + var lastState string + EventuallyErr(t, func() error { + out, err := c.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{ + SnapshotIds: []*string{aws.String(id)}, + }) + if err != nil { + return fmt.Errorf("describe-snapshot %s: %w", id, err) + } + if len(out.Snapshots) == 0 { + return fmt.Errorf("%s not found", id) + } + last = out.Snapshots[0] + lastState = aws.StringValue(last.State) + if lastState == target { + return nil + } + if lastState == "error" { + return fmt.Errorf("%s entered error state: %s", id, aws.StringValue(last.StateMessage)) + } + return fmt.Errorf("%s state=%s want=%s", id, lastState, target) + }, cfg.timeout, cfg.interval) + t.Logf("snapshot %s reached state %s", id, target) + return last +} + +// WaitForImageState polls DescribeImages until State == target. AMI creation +// from a snapshot is the slowest path; default timeout 10min. +func WaitForImageState(t *testing.T, c *AWSClient, id, target string, opts ...PollOpt) *ec2.Image { + t.Helper() + cfg := applyOpts(pollCfg{timeout: 10 * time.Minute, interval: 2 * time.Second}, opts...) + var last *ec2.Image + var lastState string + EventuallyErr(t, func() error { + out, err := c.EC2.DescribeImages(&ec2.DescribeImagesInput{ + ImageIds: []*string{aws.String(id)}, + }) + if err != nil { + return fmt.Errorf("describe-image %s: %w", id, err) + } + if len(out.Images) == 0 { + return fmt.Errorf("%s not found", id) + } + last = out.Images[0] + lastState = aws.StringValue(last.State) + if lastState == target { + return nil + } + if lastState == "failed" { + return fmt.Errorf("%s entered failed state: %s", id, aws.StringValue(last.StateReason.Message)) + } + return fmt.Errorf("%s state=%s want=%s", id, lastState, target) + }, cfg.timeout, cfg.interval) + t.Logf("image %s reached state %s", id, target) + return last +} diff --git a/tests/e2e/harness/rawhttp.go b/tests/e2e/harness/rawhttp.go new file mode 100644 index 000000000..d8885cde4 --- /dev/null +++ b/tests/e2e/harness/rawhttp.go @@ -0,0 +1,123 @@ +//go:build e2e + +package harness + +import ( + "bytes" + "crypto/tls" + "encoding/xml" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" +) + +// PostAWSAction sends a raw application/x-www-form-urlencoded POST with +// Action= + Version=2016-11-15 + caller params, signed with the same +// credentials the AWS SDK is using. Gateway rejects on auth before parsing +// the action, so signing is mandatory even for InvalidAction probes. +// +// Returns the HTTP status, raw body, and the parsed from the AWS XML +// error envelope (empty for 2xx). t.Fatal on transport / signing failure. +func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params map[string]string) (status int, body []byte, awsCode string) { + t.Helper() + + endpoint := c.EC2.Endpoint + if endpoint == "" { + t.Fatalf("PostAWSAction: EC2 endpoint not configured") + } + region := aws.StringValue(c.EC2.Config.Region) + + form := url.Values{} + form.Set("Action", action) + if _, ok := params["Version"]; !ok { + // EC2 query API version — same value the SDK injects on every call. + // Hard-coded here because we are bypassing the SDK entirely. + form.Set("Version", "2016-11-15") + } + for k, v := range params { + form.Set(k, v) + } + bodyBytes := []byte(form.Encode()) + + req, err := http.NewRequest(http.MethodPost, strings.TrimRight(endpoint, "/")+"/", bytes.NewReader(bodyBytes)) + if err != nil { + t.Fatalf("PostAWSAction: build request: %v", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") + + creds := c.EC2.Config.Credentials + if creds == nil { + t.Fatalf("PostAWSAction: no credentials on client") + } + signer := v4.NewSigner(creds) + // SignedReader keeps the body intact for replay; v4.Sign reads-then-resets. + if _, err := signer.Sign(req, bytes.NewReader(bodyBytes), "ec2", region, time.Now()); err != nil { + t.Fatalf("PostAWSAction: sign: %v", err) + } + + // Reuse the EC2 client's HTTP transport so TLS trust matches the rest of + // the suite. Fall back to a CA-loaded transport if the embedded client + // is using the SDK default. + httpClient := c.EC2.Config.HTTPClient + if httpClient == nil { + httpClient = &http.Client{ + Transport: &http.Transport{TLSClientConfig: &tls.Config{}}, + Timeout: env.DefaultTimeout, + } + } + resp, err := httpClient.Do(req) + if err != nil { + t.Fatalf("PostAWSAction: %v", err) + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("PostAWSAction: read body: %v", err) + } + status = resp.StatusCode + + if status >= 200 && status < 300 { + return status, body, "" + } + awsCode = parseAWSXMLErrorCode(body) + return status, body, awsCode +} + +// EC2 returns either … or the SDK-style +// … envelope. Try both shapes; the +// gateway has shipped one then the other across releases. +type ec2ErrorResponse struct { + XMLName xml.Name `xml:"Response"` + Errors struct { + Error struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + } `xml:"Errors"` +} + +type sdkErrorResponse struct { + XMLName xml.Name `xml:"ErrorResponse"` + Error struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` +} + +func parseAWSXMLErrorCode(body []byte) string { + var e1 ec2ErrorResponse + if err := xml.Unmarshal(body, &e1); err == nil && e1.Errors.Error.Code != "" { + return e1.Errors.Error.Code + } + var e2 sdkErrorResponse + if err := xml.Unmarshal(body, &e2); err == nil && e2.Error.Code != "" { + return e2.Error.Code + } + return "" +} diff --git a/tests/e2e/harness/spxcli.go b/tests/e2e/harness/spxcli.go new file mode 100644 index 000000000..193415502 --- /dev/null +++ b/tests/e2e/harness/spxcli.go @@ -0,0 +1,115 @@ +//go:build e2e + +package harness + +import ( + "bytes" + "os" + "os/exec" + "regexp" + "strings" + "testing" +) + +// AccountInfo is the parsed result of `spx admin account create`. Fields +// match the lines emitted by cmd/spinifex/cmd/admin.go:runAccountCreate. +type AccountInfo struct { + AccountID string + AccountName string + AccessKeyID string + SecretAccessKey string + Profile string +} + +// SpxBin returns the spx binary path (SPX_BIN env var or "spx" on PATH). +func SpxBin() string { + if v := os.Getenv("SPX_BIN"); v != "" { + return v + } + return "spx" +} + +// SpxRun runs `spx ` and returns combined stdout+stderr. If wantErr +// is false the test fails on non-zero exit; if true the call returns the +// output and the test continues (caller asserts on output content). +func SpxRun(t *testing.T, wantErr bool, args ...string) string { + t.Helper() + var buf bytes.Buffer + cmd := exec.Command(SpxBin(), args...) + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + out := buf.String() + if err != nil && !wantErr { + t.Fatalf("spx %s failed: %v\noutput:\n%s", strings.Join(args, " "), err, out) + } + if err == nil && wantErr { + t.Fatalf("spx %s expected non-zero exit, got success\noutput:\n%s", strings.Join(args, " "), out) + } + return out +} + +// SpxGetNodes runs `spx get nodes`. +func SpxGetNodes(t *testing.T) string { + t.Helper() + return SpxRun(t, false, "get", "nodes") +} + +// SpxGetVMs runs `spx get vms`. +func SpxGetVMs(t *testing.T) string { + t.Helper() + return SpxRun(t, false, "get", "vms") +} + +// SpxTopNodes runs `spx top nodes`. +func SpxTopNodes(t *testing.T) string { + t.Helper() + return SpxRun(t, false, "top", "nodes") +} + +// admin account create output is a fixed two-column key/value list; pin each +// field to a line anchor so a layout change (added/reordered keys) surfaces +// loudly instead of silently matching the wrong value. +var ( + spxAcctRE = regexp.MustCompile(`(?m)^\s+Account ID:\s+(\S+)`) + spxNameRE = regexp.MustCompile(`(?m)^\s+Account Name:\s+(.+?)\s*$`) + spxKeyIDRE = regexp.MustCompile(`(?m)^\s+Access Key ID:\s+(\S+)`) + spxSecretRE = regexp.MustCompile(`(?m)^\s+Secret Access Key:\s+(\S+)`) + spxProfRE = regexp.MustCompile(`(?m)^\s+AWS Profile:\s+(\S+)`) +) + +// SpxAdminAccountCreate creates a tenant account via the spx CLI and parses +// the printed credentials block. email is forwarded as --email when non-empty; +// `spx admin account create` ignores unknown flags today, but the CLI may +// adopt --email in future and any caller that wants to gate on it (e.g. IAM +// e-mail uniqueness checks) should pass it through unchanged. +func SpxAdminAccountCreate(t *testing.T, name, email string) AccountInfo { + t.Helper() + args := []string{"admin", "account", "create", "--name", name} + if email != "" { + args = append(args, "--email", email) + } + out := SpxRun(t, false, args...) + + info := AccountInfo{ + AccountID: firstSubmatch(spxAcctRE, out), + AccountName: firstSubmatch(spxNameRE, out), + AccessKeyID: firstSubmatch(spxKeyIDRE, out), + SecretAccessKey: firstSubmatch(spxSecretRE, out), + Profile: firstSubmatch(spxProfRE, out), + } + if info.AccountID == "" || info.AccessKeyID == "" || info.SecretAccessKey == "" { + // Dump raw output to the artifact dir if available — otherwise the + // formatted log block lives only in the JUnit XML. + t.Fatalf("spx admin account create: failed to parse credentials block\n--- raw output ---\n%s\n--- end ---", out) + } + return info +} + +func firstSubmatch(re *regexp.Regexp, s string) string { + m := re.FindStringSubmatch(s) + if len(m) < 2 { + return "" + } + return strings.TrimSpace(m[1]) +}