mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
Adding GPU passthrough e2e tests - intended to be run on Wattle
This commit is contained in:
tomnewton-mulga
+3
-2
@@ -11,9 +11,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
|
||||
# named suite into BIN_DIR/<suite>.test so CI can ship binaries (not source +
|
||||
# go toolchain) to the remote VM.
|
||||
BIN_DIR ?= $(CURDIR)/_bin
|
||||
SUITES_BUILD ?= cert lb single
|
||||
SUITES_BUILD ?= cert lb single gpu
|
||||
|
||||
.PHONY: e2e e2e-cert e2e-lb e2e-single e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-all clean e2e-build
|
||||
.PHONY: e2e e2e-cert e2e-lb e2e-single e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-all clean e2e-build
|
||||
|
||||
ifeq ($(GOTESTSUM),1)
|
||||
E2E_CMD = gotestsum \
|
||||
@@ -39,6 +39,7 @@ e2e-multinode: ; $(MAKE) e2e PKG=./multinode/... MODE=multinode
|
||||
e2e-bm: ; $(MAKE) e2e PKG=./baremetal/... MODE=baremetal
|
||||
e2e-reboot: ; $(MAKE) e2e PKG=./reboot/...
|
||||
e2e-tofu: ; $(MAKE) e2e PKG=./tofu/...
|
||||
e2e-gpu: ; $(MAKE) e2e PKG=./gpu/...
|
||||
e2e-all: ; $(MAKE) e2e
|
||||
|
||||
e2e-build:
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
//go:build e2e
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
)
|
||||
|
||||
// TestGPUPassthrough_Launch verifies that RunInstances with a GPU instance
|
||||
// type succeeds and reaches running state, confirming the VFIO bind path.
|
||||
func TestGPUPassthrough_Launch(t *testing.T) {
|
||||
fix := requireGPUFixture(t)
|
||||
harness.Phase(t, "GPU — Launch: RunInstances with %s", fix.GPUInstanceType)
|
||||
|
||||
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
|
||||
keyName, _ := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t))
|
||||
|
||||
instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName)
|
||||
harness.Detail(t, "instance", instanceID, "type", fix.GPUInstanceType, "ami", fix.AMIID)
|
||||
|
||||
inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
|
||||
assert.Equal(t, "running", aws.StringValue(inst.State.Name))
|
||||
}
|
||||
|
||||
// TestGPUPassthrough_DeviceVisible SSHes into the GPU VM and confirms the
|
||||
// NVIDIA PCI device is visible via sysfs, proving the vfio-pci passthrough
|
||||
// wired the device into the guest.
|
||||
func TestGPUPassthrough_DeviceVisible(t *testing.T) {
|
||||
fix := requireGPUFixture(t)
|
||||
harness.Phase(t, "GPU — DeviceVisible: NVIDIA PCI device present in VM sysfs")
|
||||
|
||||
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
|
||||
keyName, keyPath := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t))
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID)
|
||||
|
||||
instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName)
|
||||
inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
|
||||
harness.Detail(t, "instance", instanceID)
|
||||
|
||||
host, port := harness.InstancePublicSSHHost(t, inst)
|
||||
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
|
||||
|
||||
harness.Step(t, "waiting for SSH on %s:%d", host, port)
|
||||
waitForSSH(t, tgt, 5*time.Minute)
|
||||
|
||||
harness.Step(t, "checking sysfs for NVIDIA vendor (0x10de)")
|
||||
out := sshRun(t, tgt, "grep -l 0x10de /sys/bus/pci/devices/*/vendor 2>/dev/null | wc -l")
|
||||
assert.NotEqual(t, "0", out, "no NVIDIA PCI device found in VM sysfs — GPU passthrough did not wire the device into the guest")
|
||||
}
|
||||
|
||||
// TestGPUPassthrough_DriverAccess confirms nvidia-smi succeeds inside the VM,
|
||||
// proving the guest OS can communicate with the passed-through GPU.
|
||||
func TestGPUPassthrough_DriverAccess(t *testing.T) {
|
||||
fix := requireGPUFixture(t)
|
||||
harness.Phase(t, "GPU — DriverAccess: nvidia-smi succeeds in VM")
|
||||
|
||||
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
|
||||
keyName, keyPath := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t))
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID)
|
||||
|
||||
instanceID := launchGPUInstance(t, fix, vpc.SubnetID, []string{vpc.SGID}, keyName)
|
||||
inst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
|
||||
harness.Detail(t, "instance", instanceID)
|
||||
|
||||
host, port := harness.InstancePublicSSHHost(t, inst)
|
||||
tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
|
||||
|
||||
harness.Step(t, "waiting for SSH on %s:%d", host, port)
|
||||
waitForSSH(t, tgt, 5*time.Minute)
|
||||
|
||||
harness.Step(t, "running nvidia-smi")
|
||||
out := sshRun(t, tgt, "nvidia-smi --query-gpu=name --format=csv,noheader")
|
||||
assert.NotEmpty(t, out, "nvidia-smi returned no GPU name — driver not loaded or GPU not accessible")
|
||||
harness.Detail(t, "gpu_name", out)
|
||||
}
|
||||
|
||||
// TestGPUPassthrough_Release terminates a GPU instance and proves the GPU
|
||||
// returns to the pool by launching a second instance with the same type.
|
||||
// A hung VFIO release would cause the second RunInstances to fail with an
|
||||
// insufficient-capacity error.
|
||||
func TestGPUPassthrough_Release(t *testing.T) {
|
||||
fix := requireGPUFixture(t)
|
||||
harness.Phase(t, "GPU — Release: GPU returns to pool after termination")
|
||||
|
||||
vpc := harness.EnsureDefaultVPC(t, fix.Harness)
|
||||
keyName, _ := harness.EnsureKeyPair(t, fix.Harness, fix.ArtifactDir(t))
|
||||
sgIDs := []*string{aws.String(vpc.SGID)}
|
||||
|
||||
harness.Step(t, "launching first GPU instance")
|
||||
out1, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{
|
||||
ImageId: aws.String(fix.AMIID),
|
||||
InstanceType: aws.String(fix.GPUInstanceType),
|
||||
KeyName: aws.String(keyName),
|
||||
SubnetId: aws.String(vpc.SubnetID),
|
||||
SecurityGroupIds: sgIDs,
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
})
|
||||
require.NoError(t, err, "RunInstances first GPU instance")
|
||||
require.NotEmpty(t, out1.Instances)
|
||||
id1 := aws.StringValue(out1.Instances[0].InstanceId)
|
||||
harness.Detail(t, "instance1", id1)
|
||||
|
||||
harness.WaitForInstanceState(t, fix.AWS, id1, "running")
|
||||
|
||||
harness.Step(t, "terminating first instance — exercises gpuManager.Release()")
|
||||
_, err = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id1)},
|
||||
})
|
||||
require.NoError(t, err, "TerminateInstances")
|
||||
harness.WaitForInstanceState(t, fix.AWS, id1, "terminated")
|
||||
|
||||
harness.Step(t, "launching second GPU instance to prove pool was released")
|
||||
out2, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{
|
||||
ImageId: aws.String(fix.AMIID),
|
||||
InstanceType: aws.String(fix.GPUInstanceType),
|
||||
KeyName: aws.String(keyName),
|
||||
SubnetId: aws.String(vpc.SubnetID),
|
||||
SecurityGroupIds: sgIDs,
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
})
|
||||
require.NoError(t, err, "RunInstances second GPU instance — GPU was not returned to pool after termination")
|
||||
require.NotEmpty(t, out2.Instances)
|
||||
id2 := aws.StringValue(out2.Instances[0].InstanceId)
|
||||
harness.Detail(t, "instance2", id2)
|
||||
t.Cleanup(func() {
|
||||
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id2)},
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id2, "terminated")
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id2, "running")
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//go:build e2e
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// launchGPUInstance runs a single GPU instance using the fixture's instance
|
||||
// type and AMI, registers a terminate-and-wait cleanup, and returns the
|
||||
// instance ID. The caller receives the ID before the instance is running —
|
||||
// call harness.WaitForInstanceState to block until ready.
|
||||
func launchGPUInstance(t *testing.T, fix *Fixture, subnetID string, sgIDs []string, keyName string) string {
|
||||
t.Helper()
|
||||
sgs := make([]*string, len(sgIDs))
|
||||
for i, id := range sgIDs {
|
||||
sgs[i] = aws.String(id)
|
||||
}
|
||||
out, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{
|
||||
ImageId: aws.String(fix.AMIID),
|
||||
InstanceType: aws.String(fix.GPUInstanceType),
|
||||
KeyName: aws.String(keyName),
|
||||
SubnetId: aws.String(subnetID),
|
||||
SecurityGroupIds: sgs,
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
})
|
||||
require.NoError(t, err, "RunInstances")
|
||||
require.NotEmpty(t, out.Instances, "RunInstances returned no instances")
|
||||
id := aws.StringValue(out.Instances[0].InstanceId)
|
||||
t.Cleanup(func() {
|
||||
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id)},
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id, "terminated")
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
// waitForSSH polls until an SSH handshake to tgt succeeds or timeout elapses.
|
||||
// GPU AMIs initialise the NVIDIA driver on first boot which can add latency.
|
||||
func waitForSSH(t *testing.T, tgt harness.SSHTarget, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err := exec.CommandContext(ctx, "ssh",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes",
|
||||
"-p", strconv.Itoa(tgt.Port),
|
||||
"-i", tgt.KeyPath,
|
||||
tgt.User+"@"+tgt.Host,
|
||||
"true",
|
||||
).Run()
|
||||
cancel()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
t.Fatalf("SSH to %s:%d not ready after %s", tgt.Host, tgt.Port, timeout)
|
||||
}
|
||||
|
||||
// sshRun executes cmd on tgt and returns trimmed stdout+stderr.
|
||||
func sshRun(t *testing.T, tgt harness.SSHTarget, cmd string) string {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "ssh",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes",
|
||||
"-p", strconv.Itoa(tgt.Port),
|
||||
"-i", tgt.KeyPath,
|
||||
tgt.User+"@"+tgt.Host,
|
||||
cmd,
|
||||
).CombinedOutput()
|
||||
require.NoError(t, err, "ssh %q: %s", cmd, out)
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
//go:build e2e
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
)
|
||||
|
||||
var (
|
||||
pkgFixOnce sync.Once
|
||||
pkgFix *Fixture
|
||||
pkgFixErr error
|
||||
pkgSkipReason string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
if pkgFix != nil {
|
||||
if pkgFix.Harness != nil {
|
||||
pkgFix.Harness.Close()
|
||||
}
|
||||
if pkgFix.TmpDir != "" {
|
||||
_ = os.RemoveAll(pkgFix.TmpDir)
|
||||
}
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// Fixture carries per-process state shared across GPU passthrough tests.
|
||||
type Fixture struct {
|
||||
Env *harness.Env
|
||||
AWS *harness.AWSClient
|
||||
Harness *harness.Fixture
|
||||
TmpDir string
|
||||
GPUInstanceType string // first GPU instance type advertised by the node, e.g. "g5.xlarge"
|
||||
AMIID string // AMI ID of the imported ubuntu-26.04-nvidia-gpu-x86_64 image
|
||||
}
|
||||
|
||||
func (f *Fixture) ArtifactDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
return harness.ArtifactDir(t, f.Env)
|
||||
}
|
||||
|
||||
// requireGPUFixture returns the package-scoped Fixture singleton, building it
|
||||
// on first call. Skips the calling test when:
|
||||
// - SPINIFEX_E2E is unset
|
||||
// - no GPU instance types are advertised (node has no GPU or gpu_passthrough=false)
|
||||
// - the NVIDIA GPU AMI has not been imported yet
|
||||
func requireGPUFixture(t *testing.T) *Fixture {
|
||||
t.Helper()
|
||||
pkgFixOnce.Do(func() {
|
||||
if os.Getenv("SPINIFEX_E2E") == "" {
|
||||
return
|
||||
}
|
||||
env := harness.LoadEnv(t)
|
||||
if env.Mode != harness.ModeSingle {
|
||||
pkgSkipReason = "gpu suite requires SPINIFEX_MODE=single"
|
||||
return
|
||||
}
|
||||
// Guard against harness.NewAWSClient calling t.Fatal (which exits via
|
||||
// runtime.Goexit and corrupts the Once state for subsequent tests) when
|
||||
// no Spinifex node is running. ResolveCACert uses the same candidate
|
||||
// paths, so a failure here gives a clean skip with an actionable message.
|
||||
if _, err := harness.ResolveCACert(env); err != nil {
|
||||
pkgSkipReason = "no Spinifex node running — provision first: ansible-playbook ansible/playbooks/dev-reset.yml"
|
||||
return
|
||||
}
|
||||
awsCli := harness.NewAWSClient(t, env)
|
||||
|
||||
gpuType, amiID, reason := discoverGPUResources(awsCli)
|
||||
if reason != "" {
|
||||
pkgSkipReason = reason
|
||||
return
|
||||
}
|
||||
|
||||
h, err := harness.NewProcessFixture(awsCli)
|
||||
if err != nil {
|
||||
pkgFixErr = err
|
||||
return
|
||||
}
|
||||
tmpDir, err := os.MkdirTemp("", "gpu-pkgfix-*")
|
||||
if err != nil {
|
||||
pkgFixErr = err
|
||||
return
|
||||
}
|
||||
harness.EnsureDefaultSGOpen(t, awsCli)
|
||||
pkgFix = &Fixture{
|
||||
Env: env,
|
||||
AWS: awsCli,
|
||||
Harness: h,
|
||||
TmpDir: tmpDir,
|
||||
GPUInstanceType: gpuType,
|
||||
AMIID: amiID,
|
||||
}
|
||||
})
|
||||
if pkgFixErr != nil {
|
||||
t.Fatalf("gpu fixture init: %v", pkgFixErr)
|
||||
}
|
||||
if pkgFix == nil {
|
||||
if pkgSkipReason != "" {
|
||||
t.Skip(pkgSkipReason)
|
||||
}
|
||||
t.Skip("SPINIFEX_E2E unset")
|
||||
}
|
||||
return pkgFix
|
||||
}
|
||||
|
||||
// discoverGPUResources returns the first GPU instance type and the NVIDIA GPU
|
||||
// AMI ID available on this node. A non-empty reason means one or both are
|
||||
// absent and the suite should be skipped.
|
||||
func discoverGPUResources(c *harness.AWSClient) (gpuType, amiID, reason string) {
|
||||
typesOut, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
|
||||
if err != nil {
|
||||
return "", "", "DescribeInstanceTypes: " + err.Error()
|
||||
}
|
||||
for _, it := range typesOut.InstanceTypes {
|
||||
if it.GpuInfo != nil && len(it.GpuInfo.Gpus) > 0 {
|
||||
gpuType = aws.StringValue(it.InstanceType)
|
||||
break
|
||||
}
|
||||
}
|
||||
if gpuType == "" {
|
||||
return "", "", "no GPU instance types advertised — node has no GPU or gpu_passthrough is disabled"
|
||||
}
|
||||
|
||||
imgsOut, err := c.EC2.DescribeImages(&ec2.DescribeImagesInput{
|
||||
Filters: []*ec2.Filter{
|
||||
{Name: aws.String("name"), Values: []*string{aws.String("ubuntu-26.04-nvidia-gpu-x86_64")}},
|
||||
{Name: aws.String("state"), Values: []*string{aws.String("available")}},
|
||||
},
|
||||
})
|
||||
if err != nil || len(imgsOut.Images) == 0 {
|
||||
return "", "", "ubuntu-26.04-nvidia-gpu-x86_64 AMI not imported — run: spx admin images import --name ubuntu-26.04-nvidia-gpu-x86_64"
|
||||
}
|
||||
return gpuType, aws.StringValue(imgsOut.Images[0].ImageId), ""
|
||||
}
|
||||
Reference in New Issue
Block a user