diff --git a/spinifex/utils/utils_test.go b/spinifex/utils/utils_test.go index b2e38ba83..03ead0e34 100644 --- a/spinifex/utils/utils_test.go +++ b/spinifex/utils/utils_test.go @@ -1182,23 +1182,6 @@ func TestHashMAC_IdSeparates(t *testing.T) { assert.NotEqual(t, a, b) } -func TestHashMAC_Distribution(t *testing.T) { - // 100k random ids — 0 collisions expected. 46-bit space, birthday- - // paradox 1% at ~1.2M; 100k is well below. - if testing.Short() { - t.Skip("skipping distribution test in -short mode") - } - seen := make(map[string]string, 100_000) - for range 100_000 { - id := GenerateResourceID("res") - mac := HashMAC(id) - if prev, ok := seen[mac]; ok { - t.Fatalf("collision on %s: id=%s prev=%s", mac, id, prev) - } - seen[mac] = id - } -} - // writeTempImage writes content to a file named name under t.TempDir() and // returns the absolute path. func writeTempImage(t *testing.T, name string, content []byte) string { diff --git a/tests/e2e/eks/eks_test.go b/tests/e2e/eks/eks_test.go index 2471a2810..7fdde11ff 100644 --- a/tests/e2e/eks/eks_test.go +++ b/tests/e2e/eks/eks_test.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" "github.com/mulgadc/spinifex/tests/e2e/harness" "github.com/stretchr/testify/assert" @@ -208,6 +209,11 @@ func TestEKS(t *testing.T) { }, 3*time.Minute, 10*time.Second) }) + t.Run("IRSAWebIdentity", func(t *testing.T) { + requireClusterReady(t, fx) + runIRSAWebIdentity(t, c, env, artifacts, fx) + }) + t.Run("DeleteCluster", func(t *testing.T) { requireClusterReady(t, fx) _, err := c.EKS.DeleteCluster(&eks.DeleteClusterInput{Name: aws.String(fx.ClusterName)}) @@ -217,6 +223,103 @@ func TestEKS(t *testing.T) { }) } +// runIRSAWebIdentity exercises the IRSA token-exchange path at the API level +// (no pod, no nodegroup): register the cluster's OIDC provider, create a role +// trusting it, mint a real ServiceAccount token via `kubectl create token` +// (signed by the cluster's OIDC key, JWKS published at CreateCluster), exchange +// it through AssumeRoleWithWebIdentity, and prove the returned credentials are +// usable via GetCallerIdentity. +func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) { + require.NotNil(t, fx.Cluster.Identity, "ACTIVE cluster must expose Identity") + require.NotNil(t, fx.Cluster.Identity.Oidc, "ACTIVE cluster must expose Identity.Oidc") + issuer := aws.StringValue(fx.Cluster.Identity.Oidc.Issuer) + require.NotEmpty(t, issuer, "cluster OIDC issuer must be published") + t.Logf("OIDC issuer: %s", issuer) + + // 1) Register the cluster's OIDC provider in the caller account so the STS + // handler will accept tokens carrying this issuer. + oidcOut, err := c.IAM.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{ + Url: aws.String(issuer), + ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}), + ThumbprintList: aws.StringSlice([]string{"0000000000000000000000000000000000000000"}), + }) + require.NoError(t, err, "create-open-id-connect-provider") + providerArn := aws.StringValue(oidcOut.OpenIDConnectProviderArn) + require.NotEmpty(t, providerArn, "OIDC provider ARN empty") + t.Cleanup(func() { + _, _ = c.IAM.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{ + OpenIDConnectProviderArn: aws.String(providerArn), + }) + }) + t.Logf("OIDC provider: %s", providerArn) + + // 2) Create a role whose trust policy federates the OIDC provider. No + // Condition block — the Federated principal + AssumeRoleWithWebIdentity + // action are sufficient to grant, which keeps the test independent of the + // condition-key issuer-prefix format. + roleName := fmt.Sprintf("%s-irsa", fx.ClusterName) + trustPolicy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":%q},"Action":"sts:AssumeRoleWithWebIdentity"}]}`, providerArn) + roleOut, err := c.IAM.CreateRole(&iam.CreateRoleInput{ + RoleName: aws.String(roleName), + AssumeRolePolicyDocument: aws.String(trustPolicy), + Description: aws.String("E2E IRSA web-identity role"), + }) + require.NoError(t, err, "create-role") + roleArn := aws.StringValue(roleOut.Role.Arn) + require.NotEmpty(t, roleArn, "role ARN empty") + t.Cleanup(func() { + _, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)}) + }) + t.Logf("role: %s", roleArn) + + // 3) Mint a ServiceAccount token bound to sts.amazonaws.com. `kubectl create + // token` uses the TokenRequest API — no pod required. The token's iss is + // the cluster OIDC issuer and aud includes sts.amazonaws.com (k3s is wired + // with --service-account-issuer / --api-audiences at CreateCluster). + kcPath := writeKubeconfig(t, artifacts, fx.Cluster) + kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env)) + tokenOut, err := kc.Run(30*time.Second, "create", "token", "default", + "--namespace", "default", "--audience", "sts.amazonaws.com") + require.NoErrorf(t, err, "kubectl create token:\n%s", tokenOut) + token := strings.TrimSpace(tokenOut) + require.Equal(t, 2, strings.Count(token, "."), "web-identity token must be a JWT (3 dot-separated parts)") + + // 4) Exchange the token. AssumeRoleWithWebIdentity is anonymous (the SDK + // strips SigV4 for this op); the JWT is the identity. + const sessionName = "e2e-irsa" + var assumeOut *sts.AssumeRoleWithWebIdentityOutput + harness.EventuallyErr(t, func() error { + out, aerr := c.STS.AssumeRoleWithWebIdentity(&sts.AssumeRoleWithWebIdentityInput{ + RoleArn: aws.String(roleArn), + RoleSessionName: aws.String(sessionName), + WebIdentityToken: aws.String(token), + }) + if aerr != nil { + return fmt.Errorf("assume-role-with-web-identity: %w", aerr) + } + assumeOut = out + return nil + }, 60*time.Second, 5*time.Second) + require.NotNil(t, assumeOut.Credentials, "AssumeRoleWithWebIdentity returned no credentials") + assert.Equal(t, "system:serviceaccount:default:default", + aws.StringValue(assumeOut.SubjectFromWebIdentityToken), "subject must be the default SA") + require.True(t, strings.HasPrefix(aws.StringValue(assumeOut.Credentials.AccessKeyId), "ASIA"), + "web-identity credentials must be temporary (ASIA…)") + + // 5) The returned temporary credentials must be usable: GetCallerIdentity + // must resolve to the assumed-role principal. + sessClient := harness.NewAWSClientWithSessionCreds(t, env, + aws.StringValue(assumeOut.Credentials.AccessKeyId), + aws.StringValue(assumeOut.Credentials.SecretAccessKey), + aws.StringValue(assumeOut.Credentials.SessionToken)) + ident, err := sessClient.STS.GetCallerIdentity(&sts.GetCallerIdentityInput{}) + require.NoError(t, err, "get-caller-identity with web-identity creds") + assert.Equal(t, fx.AccountID, aws.StringValue(ident.Account), "caller account must match") + assert.Contains(t, aws.StringValue(ident.Arn), "assumed-role/"+roleName, + "caller ARN must reflect the assumed IRSA role") + t.Logf("GetCallerIdentity (web-identity creds): %s", aws.StringValue(ident.Arn)) +} + // --- Fixture -------------------------------------------------------------- type clusterFixture struct { diff --git a/tests/e2e/harness/aws.go b/tests/e2e/harness/aws.go index 25e3c29e4..0774cc5a4 100644 --- a/tests/e2e/harness/aws.go +++ b/tests/e2e/harness/aws.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/acm" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/eks" @@ -33,6 +34,7 @@ type AWSClient struct { IAM *iam.IAM STS *sts.STS EKS *eks.EKS + ACM *acm.ACM } // NewAWSClient builds clients pointed at the spinifex gateway using the @@ -128,6 +130,7 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str IAM: iam.New(sess), STS: sts.New(sess), EKS: eks.New(sess), + ACM: acm.New(sess), } } @@ -141,5 +144,6 @@ func (c *AWSClient) IgnoreCertErrors() { c.EKS.Config.HTTPClient = hc c.IAM.Config.HTTPClient = hc c.STS.Config.HTTPClient = hc + c.ACM.Config.HTTPClient = hc _ = (*x509.CertPool)(nil) // silence unused-import on hardened paths } diff --git a/tests/e2e/harness/cert.go b/tests/e2e/harness/cert.go index 78a64b073..292760a25 100644 --- a/tests/e2e/harness/cert.go +++ b/tests/e2e/harness/cert.go @@ -3,15 +3,21 @@ package harness import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/x509" + "crypto/x509/pkix" "encoding/pem" "fmt" + "math/big" "net" "os" "os/exec" "path/filepath" "strings" "testing" + "time" ) // ResolveCACert walks the standard config locations for the Spinifex CA cert @@ -121,6 +127,50 @@ func OpenSSLVerify(t *testing.T, caFile, host string, port int) error { return fmt.Errorf("openssl verify %s failed: %v\noutput:\n%s", target, err, out) } +// GenerateSelfSignedCertPEM mints an ECDSA P-256 self-signed leaf for host +// (an IP or DNS name) and returns the cert PEM, the PKCS#8 key PEM, and the +// parsed leaf. Used by the ACM ImportCertificate → HTTPS listener e2e to feed +// a real keypair through the import → HAProxy termination path. +func GenerateSelfSignedCertPEM(host string) (certPEM, keyPEM []byte, leaf *x509.Certificate, err error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, nil, fmt.Errorf("generate key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, nil, fmt.Errorf("serial: %w", err) + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, nil, nil, fmt.Errorf("create certificate: %w", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, nil, fmt.Errorf("marshal key: %w", err) + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + leaf, err = x509.ParseCertificate(der) + if err != nil { + return nil, nil, nil, fmt.Errorf("parse leaf: %w", err) + } + return certPEM, keyPEM, leaf, nil +} + // FingerprintMatches returns true if the two PEM certs share the SHA-256 // fingerprint that openssl x509 -fingerprint prints. func FingerprintMatches(a, b *x509.Certificate) bool { diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go index a1732cc49..d834eb054 100644 --- a/tests/e2e/harness/ec2helpers.go +++ b/tests/e2e/harness/ec2helpers.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -213,6 +214,59 @@ func EnsureDefaultSGOpen(t *testing.T, c *AWSClient) { AuthorizeICMPIngress(t, c, sgID) } +// AttachVolumeWait attaches volID to instanceID at device and blocks until the +// volume reaches "in-use". Used by the data-durability tests where the attach +// must complete before the guest can format the disk. +func AttachVolumeWait(t *testing.T, c *AWSClient, volID, instanceID, device string) { + t.Helper() + _, err := c.EC2.AttachVolume(&ec2.AttachVolumeInput{ + VolumeId: aws.String(volID), + InstanceId: aws.String(instanceID), + Device: aws.String(device), + }) + if err != nil { + t.Fatalf("attach-volume %s -> %s as %s: %v", volID, instanceID, device, err) + } + WaitForVolumeState(t, c, volID, ec2.VolumeStateInUse, WithPoll(500*time.Millisecond)) +} + +// DetachVolumeWait detaches volID and blocks until it reaches "available". +func DetachVolumeWait(t *testing.T, c *AWSClient, volID string) { + t.Helper() + _, err := c.EC2.DetachVolume(&ec2.DetachVolumeInput{VolumeId: aws.String(volID)}) + if err != nil { + t.Fatalf("detach-volume %s: %v", volID, err) + } + WaitForVolumeState(t, c, volID, "available", WithPoll(500*time.Millisecond)) +} + +// RegisterVolumeTeardown best-effort force-detaches then deletes volID at test +// end. Non-fatal — it polls for "available" within a bounded window and never +// calls t.Fatal so a cleanup hiccup doesn't mask the test's real result. +func RegisterVolumeTeardown(t *testing.T, c *AWSClient, volID string) { + t.Helper() + t.Cleanup(func() { + _, _ = c.EC2.DetachVolume(&ec2.DetachVolumeInput{ + VolumeId: aws.String(volID), + Force: aws.Bool(true), + }) + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{ + VolumeIds: []*string{aws.String(volID)}, + }) + if err != nil || len(out.Volumes) == 0 { + return + } + if aws.StringValue(out.Volumes[0].State) == "available" { + break + } + time.Sleep(2 * time.Second) + } + _, _ = c.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(volID)}) + }) +} + // asErr is a thin errors.As that lets the helper stay testify-free. func asErr(err error, target *awserr.Error) bool { if a, ok := err.(awserr.Error); ok { diff --git a/tests/e2e/harness/eks_mgmt.go b/tests/e2e/harness/eks_mgmt.go index 31d5207fb..076b911e1 100644 --- a/tests/e2e/harness/eks_mgmt.go +++ b/tests/e2e/harness/eks_mgmt.go @@ -11,6 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/spf13/viper" ) // ControlPlaneMgmtIP resolves the EKS control-plane VM's br-mgmt address — @@ -56,7 +57,7 @@ func natsConn(t *testing.T, env *Env) (host, token, ca string) { } cfgPath := filepath.Join(env.ConfigDir, "spinifex.toml") - cc, err := config.LoadConfig(cfgPath) + cc, err := loadClusterConfig(cfgPath) if err != nil { t.Fatalf("load %s for NATS settings: %v", cfgPath, err) } @@ -67,6 +68,23 @@ func natsConn(t *testing.T, env *Env) (host, token, ca string) { return dialableNATSHost(node.NATS.Host), node.NATS.ACL.Token, node.NATS.CACert } +// loadClusterConfig reads spinifex.toml via a private viper instance with no +// AutomaticEnv, so the harness's own SPINIFEX_* vars (the multinode workflow +// exports SPINIFEX_NODES) cannot shadow the [nodes.*] table and blank the parse. +func loadClusterConfig(path string) (*config.ClusterConfig, error) { + v := viper.New() + v.SetConfigFile(path) + v.SetConfigType("toml") + if err := v.ReadInConfig(); err != nil { + return nil, err + } + var cc config.ClusterConfig + if err := v.Unmarshal(&cc); err != nil { + return nil, err + } + return &cc, nil +} + // dialableNATSHost rewrites a wildcard bind address to loopback. NATS.Host is a // bind address (commonly 0.0.0.0:4222); the TLS serving cert SANs the loopback // and real node IPs but not 0.0.0.0, so dial 127.0.0.1 instead. diff --git a/tests/e2e/harness/guest_data.go b/tests/e2e/harness/guest_data.go new file mode 100644 index 000000000..4bebae1f5 --- /dev/null +++ b/tests/e2e/harness/guest_data.go @@ -0,0 +1,159 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "testing" + "time" +) + +// Guest-side data-durability helpers shared by the volume-durability, +// snapshot-restore, and CreateImage e2e tests. They drive a real guest over SSH +// to format, write, sync, and checksum bytes so the assembled +// QEMU↔viperblock↔predastore I/O path is exercised end-to-end — something the +// per-layer unit tests cannot cover. + +// guestSentinelFile is the file written into a freshly formatted data volume. +const guestSentinelFile = "e2e-sentinel.bin" + +// guestExecTimeout bounds a single guest command (mkfs/dd are sub-second on the +// small payloads used here; the ceiling only guards against a hung SSH). +const guestExecTimeout = 2 * time.Minute + +// sha256RE matches a bare sha256 digest so the checksum can be lifted out of +// command output that may also carry the file path (`␠␠`). +var sha256RE = regexp.MustCompile(`\b[0-9a-f]{64}\b`) + +// GuestExec runs cmd over SSH against tgt and returns combined stdout+stderr +// plus the run error. It never calls t.Fatal, so callers can branch on an +// expected non-zero exit. +func GuestExec(tgt SSHTarget, cmd string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), guestExecTimeout) + defer cancel() + args := []string{ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-p", strconv.Itoa(tgt.Port), + "-i", tgt.KeyPath, + tgt.User + "@" + tgt.Host, + cmd, + } + out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput() + return string(out), err +} + +// guestDiskSet returns the set of whole-disk device names (lsblk TYPE=disk) +// currently visible in the guest, keyed by bare name (e.g. "vda", "vdc"). +func guestDiskSet(tgt SSHTarget) (map[string]struct{}, error) { + out, err := GuestExec(tgt, "lsblk -dn -o NAME,TYPE") + if err != nil { + return nil, fmt.Errorf("lsblk: %w\n%s", err, out) + } + set := map[string]struct{}{} + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + f := strings.Fields(line) + if len(f) >= 2 && f[1] == "disk" { + set[f[0]] = struct{}{} + } + } + return set, nil +} + +// GuestDiskSet is the t.Fatal-on-error wrapper around guestDiskSet, used to +// snapshot the guest's disk inventory before an AttachVolume. +func GuestDiskSet(t *testing.T, tgt SSHTarget) map[string]struct{} { + t.Helper() + set, err := guestDiskSet(tgt) + if err != nil { + t.Fatalf("GuestDiskSet: %v", err) + } + return set +} + +// WaitForNewGuestDisk polls the guest until a whole-disk device absent from +// `before` appears, returning its bare name (e.g. "vdc"). Proves a hotplugged +// volume actually reached the guest kernel, not just the EC2 control plane. +func WaitForNewGuestDisk(t *testing.T, tgt SSHTarget, before map[string]struct{}, timeout time.Duration) string { + t.Helper() + var found string + EventuallyErr(t, func() error { + now, err := guestDiskSet(tgt) + if err != nil { + return err + } + for name := range now { + if _, ok := before[name]; !ok { + found = name + return nil + } + } + names := make([]string, 0, len(now)) + for name := range now { + names = append(names, name) + } + return fmt.Errorf("no new disk yet (visible: %s)", strings.Join(names, ",")) + }, timeout, 2*time.Second) + return found +} + +// GuestFormatWriteSentinel formats /dev/ as ext4 with the given label, +// mounts it, writes sizeMiB of random data to the sentinel file, fsyncs, and +// returns the file's sha256. Unmounts before returning so the volume can be +// detached cleanly. +func GuestFormatWriteSentinel(t *testing.T, tgt SSHTarget, dev, label string, sizeMiB int) string { + t.Helper() + mnt := "/mnt/" + label + script := strings.Join([]string{ + fmt.Sprintf("sudo mkfs.ext4 -F -L %s /dev/%s >/dev/null 2>&1", label, dev), + fmt.Sprintf("sudo mkdir -p %s", mnt), + fmt.Sprintf("sudo mount /dev/%s %s", dev, mnt), + fmt.Sprintf("sudo dd if=/dev/urandom of=%s/%s bs=1M count=%d conv=fsync status=none", mnt, guestSentinelFile, sizeMiB), + "sync", + fmt.Sprintf("sudo sha256sum %s/%s", mnt, guestSentinelFile), + fmt.Sprintf("sudo umount %s", mnt), + }, " && ") + out, err := GuestExec(tgt, script) + if err != nil { + t.Fatalf("GuestFormatWriteSentinel(%s,%s): %v\n%s", dev, label, err, out) + } + return mustSha(t, out) +} + +// GuestReadSentinelSha mounts source (a device path such as "/dev/vdc" or a +// "/dev/disk/by-label/