diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index 6246d6bf9..a2b0242f0 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -11,9 +11,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG)))) # named suite into BIN_DIR/.test so CI can ship binaries (not source + # go toolchain) to the remote VM. BIN_DIR ?= $(CURDIR)/_bin -SUITES_BUILD ?= cert lb single +SUITES_BUILD ?= cert lb single gpu -.PHONY: e2e e2e-cert e2e-lb e2e-single e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-all clean e2e-build +.PHONY: e2e e2e-cert e2e-lb e2e-single e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-all clean e2e-build ifeq ($(GOTESTSUM),1) E2E_CMD = gotestsum \ @@ -39,6 +39,7 @@ e2e-multinode: ; $(MAKE) e2e PKG=./multinode/... MODE=multinode e2e-bm: ; $(MAKE) e2e PKG=./baremetal/... MODE=baremetal e2e-reboot: ; $(MAKE) e2e PKG=./reboot/... e2e-tofu: ; $(MAKE) e2e PKG=./tofu/... +e2e-gpu: ; $(MAKE) e2e PKG=./gpu/... e2e-all: ; $(MAKE) e2e e2e-build: diff --git a/tests/e2e/gpu/gpu_test.go b/tests/e2e/gpu/gpu_test.go new file mode 100644 index 000000000..866875d20 --- /dev/null +++ b/tests/e2e/gpu/gpu_test.go @@ -0,0 +1,142 @@ +//go:build e2e + +package gpu + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// TestGPUPassthrough_Launch verifies that RunInstances with a GPU instance +// type succeeds and reaches running state, confirming the VFIO bind path. +func TestGPUPassthrough_Launch(t *testing.T) { + fix := requireGPUFixture(t) + harness.Phase(t, "GPU — Launch: RunInstances with %s", fix.GPUInstanceType) + + vpc := harness.EnsureDefaultVPC(t, fix.Harness) + keyName, _ := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t)) + + instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName) + harness.Detail(t, "instance", instanceID, "type", fix.GPUInstanceType, "ami", fix.AMIID) + + inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + assert.Equal(t, "running", aws.StringValue(inst.State.Name)) +} + +// TestGPUPassthrough_DeviceVisible SSHes into the GPU VM and confirms the +// NVIDIA PCI device is visible via sysfs, proving the vfio-pci passthrough +// wired the device into the guest. +func TestGPUPassthrough_DeviceVisible(t *testing.T) { + fix := requireGPUFixture(t) + harness.Phase(t, "GPU — DeviceVisible: NVIDIA PCI device present in VM sysfs") + + vpc := harness.EnsureDefaultVPC(t, fix.Harness) + keyName, keyPath := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t)) + harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID) + + instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName) + inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + harness.Detail(t, "instance", instanceID) + + host, port := harness.InstancePublicSSHHost(t, inst) + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + + harness.Step(t, "waiting for SSH on %s:%d", host, port) + waitForSSH(t, tgt, 5*time.Minute) + + harness.Step(t, "checking sysfs for NVIDIA vendor (0x10de)") + out := sshRun(t, tgt, "grep -l 0x10de /sys/bus/pci/devices/*/vendor 2>/dev/null | wc -l") + assert.NotEqual(t, "0", out, "no NVIDIA PCI device found in VM sysfs — GPU passthrough did not wire the device into the guest") +} + +// TestGPUPassthrough_DriverAccess confirms nvidia-smi succeeds inside the VM, +// proving the guest OS can communicate with the passed-through GPU. +func TestGPUPassthrough_DriverAccess(t *testing.T) { + fix := requireGPUFixture(t) + harness.Phase(t, "GPU — DriverAccess: nvidia-smi succeeds in VM") + + vpc := harness.EnsureDefaultVPC(t, fix.Harness) + keyName, keyPath := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t)) + harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID) + + instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName) + inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + harness.Detail(t, "instance", instanceID) + + host, port := harness.InstancePublicSSHHost(t, inst) + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + + harness.Step(t, "waiting for SSH on %s:%d", host, port) + waitForSSH(t, tgt, 5*time.Minute) + + harness.Step(t, "running nvidia-smi") + out := sshRun(t, tgt, "nvidia-smi --query-gpu=name --format=csv,noheader") + assert.NotEmpty(t, out, "nvidia-smi returned no GPU name — driver not loaded or GPU not accessible") + harness.Detail(t, "gpu_name", out) +} + +// TestGPUPassthrough_Release terminates a GPU instance and proves the GPU +// returns to the pool by launching a second instance with the same type. +// A hung VFIO release would cause the second RunInstances to fail with an +// insufficient-capacity error. +func TestGPUPassthrough_Release(t *testing.T) { + fix := requireGPUFixture(t) + harness.Phase(t, "GPU — Release: GPU returns to pool after termination") + + vpc := harness.EnsureDefaultVPC(t, fix.Harness) + keyName, _ := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t)) + sgIDs := []*string{aws.String(vpc.SGID)} + + harness.Step(t, "launching first GPU instance") + out1, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{ + ImageId: aws.String(fix.AMIID), + InstanceType: aws.String(fix.GPUInstanceType), + KeyName: aws.String(keyName), + SubnetId: aws.String(vpc.SubnetID), + SecurityGroupIds: sgIDs, + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + }) + require.NoError(t, err, "RunInstances first GPU instance") + require.NotEmpty(t, out1.Instances) + id1 := aws.StringValue(out1.Instances[0].InstanceId) + harness.Detail(t, "instance1", id1) + + harness.WaitForInstanceState(t, fix.AWS, id1, "running") + + harness.Step(t, "terminating first instance — exercises gpuManager.Release()") + _, err = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(id1)}, + }) + require.NoError(t, err, "TerminateInstances") + harness.WaitForInstanceState(t, fix.AWS, id1, "terminated") + + harness.Step(t, "launching second GPU instance to prove pool was released") + out2, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{ + ImageId: aws.String(fix.AMIID), + InstanceType: aws.String(fix.GPUInstanceType), + KeyName: aws.String(keyName), + SubnetId: aws.String(vpc.SubnetID), + SecurityGroupIds: sgIDs, + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + }) + require.NoError(t, err, "RunInstances second GPU instance — GPU was not returned to pool after termination") + require.NotEmpty(t, out2.Instances) + id2 := aws.StringValue(out2.Instances[0].InstanceId) + harness.Detail(t, "instance2", id2) + t.Cleanup(func() { + _, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(id2)}, + }) + harness.WaitForInstanceState(t, fix.AWS, id2, "terminated") + }) + harness.WaitForInstanceState(t, fix.AWS, id2, "running") +} diff --git a/tests/e2e/gpu/helpers_test.go b/tests/e2e/gpu/helpers_test.go new file mode 100644 index 000000000..cb574e8f6 --- /dev/null +++ b/tests/e2e/gpu/helpers_test.go @@ -0,0 +1,95 @@ +//go:build e2e + +package gpu + +import ( + "context" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// launchGPUInstance runs a single GPU instance using the fixture's instance +// type and AMI, registers a terminate-and-wait cleanup, and returns the +// instance ID. The caller receives the ID before the instance is running — +// call harness.WaitForInstanceState to block until ready. +func launchGPUInstance(t *testing.T, fix *Fixture, subnetID string, sgIDs []string, keyName string) string { + t.Helper() + sgs := make([]*string, len(sgIDs)) + for i, id := range sgIDs { + sgs[i] = aws.String(id) + } + out, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{ + ImageId: aws.String(fix.AMIID), + InstanceType: aws.String(fix.GPUInstanceType), + KeyName: aws.String(keyName), + SubnetId: aws.String(subnetID), + SecurityGroupIds: sgs, + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + }) + require.NoError(t, err, "RunInstances") + require.NotEmpty(t, out.Instances, "RunInstances returned no instances") + id := aws.StringValue(out.Instances[0].InstanceId) + t.Cleanup(func() { + _, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + harness.WaitForInstanceState(t, fix.AWS, id, "terminated") + }) + return id +} + +// waitForSSH polls until an SSH handshake to tgt succeeds or timeout elapses. +// GPU AMIs initialise the NVIDIA driver on first boot which can add latency. +func waitForSSH(t *testing.T, tgt harness.SSHTarget, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := exec.CommandContext(ctx, "ssh", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-p", strconv.Itoa(tgt.Port), + "-i", tgt.KeyPath, + tgt.User+"@"+tgt.Host, + "true", + ).Run() + cancel() + if err == nil { + return + } + time.Sleep(5 * time.Second) + } + t.Fatalf("SSH to %s:%d not ready after %s", tgt.Host, tgt.Port, timeout) +} + +// sshRun executes cmd on tgt and returns trimmed stdout+stderr. +func sshRun(t *testing.T, tgt harness.SSHTarget, cmd string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "ssh", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-p", strconv.Itoa(tgt.Port), + "-i", tgt.KeyPath, + tgt.User+"@"+tgt.Host, + cmd, + ).CombinedOutput() + require.NoError(t, err, "ssh %q: %s", cmd, out) + return strings.TrimSpace(string(out)) +} diff --git a/tests/e2e/gpu/main_test.go b/tests/e2e/gpu/main_test.go new file mode 100644 index 000000000..d05038fee --- /dev/null +++ b/tests/e2e/gpu/main_test.go @@ -0,0 +1,142 @@ +//go:build e2e + +package gpu + +import ( + "os" + "sync" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +var ( + pkgFixOnce sync.Once + pkgFix *Fixture + pkgFixErr error + pkgSkipReason string +) + +func TestMain(m *testing.M) { + code := m.Run() + if pkgFix != nil { + if pkgFix.Harness != nil { + pkgFix.Harness.Close() + } + if pkgFix.TmpDir != "" { + _ = os.RemoveAll(pkgFix.TmpDir) + } + } + os.Exit(code) +} + +// Fixture carries per-process state shared across GPU passthrough tests. +type Fixture struct { + Env *harness.Env + AWS *harness.AWSClient + Harness *harness.Fixture + TmpDir string + GPUInstanceType string // first GPU instance type advertised by the node, e.g. "g5.xlarge" + AMIID string // AMI ID of the imported ubuntu-26.04-nvidia-gpu-x86_64 image +} + +func (f *Fixture) ArtifactDir(t *testing.T) string { + t.Helper() + return harness.ArtifactDir(t, f.Env) +} + +// requireGPUFixture returns the package-scoped Fixture singleton, building it +// on first call. Skips the calling test when: +// - SPINIFEX_E2E is unset +// - no GPU instance types are advertised (node has no GPU or gpu_passthrough=false) +// - the NVIDIA GPU AMI has not been imported yet +func requireGPUFixture(t *testing.T) *Fixture { + t.Helper() + pkgFixOnce.Do(func() { + if os.Getenv("SPINIFEX_E2E") == "" { + return + } + env := harness.LoadEnv(t) + if env.Mode != harness.ModeSingle { + pkgSkipReason = "gpu suite requires SPINIFEX_MODE=single" + return + } + // Guard against harness.NewAWSClient calling t.Fatal (which exits via + // runtime.Goexit and corrupts the Once state for subsequent tests) when + // no Spinifex node is running. ResolveCACert uses the same candidate + // paths, so a failure here gives a clean skip with an actionable message. + if _, err := harness.ResolveCACert(env); err != nil { + pkgSkipReason = "no Spinifex node running — provision first: ansible-playbook ansible/playbooks/dev-reset.yml" + return + } + awsCli := harness.NewAWSClient(t, env) + + gpuType, amiID, reason := discoverGPUResources(awsCli) + if reason != "" { + pkgSkipReason = reason + return + } + + h, err := harness.NewProcessFixture(awsCli) + if err != nil { + pkgFixErr = err + return + } + tmpDir, err := os.MkdirTemp("", "gpu-pkgfix-*") + if err != nil { + pkgFixErr = err + return + } + harness.EnsureDefaultSGOpen(t, awsCli) + pkgFix = &Fixture{ + Env: env, + AWS: awsCli, + Harness: h, + TmpDir: tmpDir, + GPUInstanceType: gpuType, + AMIID: amiID, + } + }) + if pkgFixErr != nil { + t.Fatalf("gpu fixture init: %v", pkgFixErr) + } + if pkgFix == nil { + if pkgSkipReason != "" { + t.Skip(pkgSkipReason) + } + t.Skip("SPINIFEX_E2E unset") + } + return pkgFix +} + +// discoverGPUResources returns the first GPU instance type and the NVIDIA GPU +// AMI ID available on this node. A non-empty reason means one or both are +// absent and the suite should be skipped. +func discoverGPUResources(c *harness.AWSClient) (gpuType, amiID, reason string) { + typesOut, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) + if err != nil { + return "", "", "DescribeInstanceTypes: " + err.Error() + } + for _, it := range typesOut.InstanceTypes { + if it.GpuInfo != nil && len(it.GpuInfo.Gpus) > 0 { + gpuType = aws.StringValue(it.InstanceType) + break + } + } + if gpuType == "" { + return "", "", "no GPU instance types advertised — node has no GPU or gpu_passthrough is disabled" + } + + imgsOut, err := c.EC2.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("name"), Values: []*string{aws.String("ubuntu-26.04-nvidia-gpu-x86_64")}}, + {Name: aws.String("state"), Values: []*string{aws.String("available")}}, + }, + }) + if err != nil || len(imgsOut.Images) == 0 { + return "", "", "ubuntu-26.04-nvidia-gpu-x86_64 AMI not imported — run: spx admin images import --name ubuntu-26.04-nvidia-gpu-x86_64" + } + return gpuType, aws.StringValue(imgsOut.Images[0].ImageId), "" +}