feat(gpu): multi-GPU instance types (#1099)

docs/compute/gpu-passthrough/README.md

@@ -136,6 +136,21 @@ aws ec2 run-instances \
   --key-name spinifex-key
 ```
 
+### Multi-GPU Instances
+
+The GPU count comes from the instance type, exactly as it does in EC2. AWS multi-GPU types pair eight GPUs with a very large host — `p4d.24xlarge` is 96 vCPU and 1152 GiB, — so an edge rig with eight cards and 256 GiB cannot admit them even though it has the GPUs. For those hosts Spinifex offers its own `gpu.*` family, which is not an EC2 family:
+
+```bash
+aws ec2 run-instances \
+  --image-id $GPU_AMI \
+  --instance-type gpu.8x4c \
+  --key-name spinifex-key
+```
+
+`gpu.<count>x<vcpu>c` is that many GPUs at that many vCPU each, so `gpu.8x4c` is eight GPUs on 32 vCPU and 128 GiB. Counts run in three tiers — `2c`, `4c` and `8c` — all at 4 GiB of memory per vCPU. The GPU model and VRAM reported in `DescribeInstanceTypes` are whatever the node discovered, so a `gpu.8x4c` on an 8×RTX 3090 box advertises eight RTX 3090s at 24 GiB each.
+
+A node holding more than one distinct GPU model offers single-GPU types only, and logs each withheld type by name. Two devices count as the same model when their reported name and VRAM match, which keeps a mixed-SKU H100 node fully usable.
+
 To verify the GPU is visible from inside the instance, SSH in and run:
 
 ```bash

spinifex/daemon/capacityreservation.go

@@ -7,6 +7,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 )
 
 // capacityReservation is an in-memory On-Demand Capacity Reservation pinned to
@@ -83,7 +84,8 @@ func (rm *ResourceManager) CancelReservation(id, accountID string) (*capacityRes
 // at create moves to allocated* and ConsumedCount bumps, so schedulable capacity
 // is unchanged by construction. Errors: unknown/foreign id ->
 // InvalidCapacityReservationId.NotFound; type mismatch -> InvalidParameterValue;
-// already full -> ReservationCapacityExceeded.
+// already full -> ReservationCapacityExceeded; no free GPU ->
+// InsufficientInstanceCapacity.
 func (rm *ResourceManager) AllocateFromReservation(crID, accountID string, it *ec2.InstanceTypeInfo) error {
 	rm.mu.Lock()
 	rec, ok := rm.reservations[crID]
@@ -99,6 +101,13 @@ func (rm *ResourceManager) AllocateFromReservation(crID, accountID string, it *e
 		rm.mu.Unlock()
 		return errors.New(awserrors.ErrorReservationCapacityExceeded)
 	}
+	// Re-checked under the write lock, as the general path does: two targeted
+	// launches can both clear ReservationAvailable and only one find a GPU.
+	if instancetypes.IsGPUType(it) &&
+		rm.admissibleGPUInstances(aws.StringValue(it.InstanceType)) < 1 {
+		rm.mu.Unlock()
+		return errors.New(awserrors.ErrorInsufficientInstanceCapacity)
+	}
 	rm.reservedCRVCPU -= rec.VCPUPerInstance
 	rm.reservedCRMem -= rec.MemGBPerInstance
 	rm.allocatedVCPU += rec.VCPUPerInstance
@@ -136,6 +145,8 @@ func (rm *ResourceManager) ReleaseToReservation(crID string, it *ec2.InstanceTyp
 // (Total - Consumed), or 0 when the reservation is unknown, owned by another
 // account, or for a different instance type. The daemon handler's up-front check
 // turns those zero cases into precise errors before the launch loop.
+// A GPU type is additionally gated on free GPUs: a reservation reserves no pool
+// entry, so its free slots say nothing about whether a GPU is there to claim.
 func (rm *ResourceManager) ReservationAvailable(crID, accountID string, it *ec2.InstanceTypeInfo) int {
 	rm.mu.RLock()
 	defer rm.mu.RUnlock()
@@ -143,7 +154,11 @@ func (rm *ResourceManager) ReservationAvailable(crID, accountID string, it *ec2.
 	if !ok || rec.AccountID != accountID || rec.InstanceType != aws.StringValue(it.InstanceType) {
 		return 0
 	}
-	return rec.TotalInstanceCount - rec.ConsumedCount
+	free := rec.TotalInstanceCount - rec.ConsumedCount
+	if instancetypes.IsGPUType(it) {
+		free = min(free, rm.admissibleGPUInstances(aws.StringValue(it.InstanceType)))
+	}
+	return free
 }
 
 // ValidateReservationTarget is the owning daemon's up-front semantic check for a

spinifex/daemon/daemon.go

@@ -597,15 +597,7 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 		// GPU types are capacity-gated by GPU pool size, not host CPU/memory.
 		// The GPU is the scarce resource; CPU/memory on GPU-class hardware is abundant.
 		if instancetypes.IsGPUType(it) {
-			availGPU := 0
-			if rm.gpuManager != nil {
-				availGPU = rm.gpuManager.Available()
-			}
-			gpusNeeded := instancetypes.GPUCountForType(name)
-			count := 0
-			if gpusNeeded > 0 {
-				count = availGPU / gpusNeeded
-			}
+			count := rm.admissibleGPUInstances(name)
 			if showCapacity {
 				for range count {
 					infos = append(infos, it)
@@ -696,6 +688,12 @@ func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64
 		if typeCap.VCPU == 0 || typeCap.MemoryGB == 0 {
 			continue
 		}
+		// CPU and memory never bind first on a gpu.* shape, so without this the
+		// census calls a GPU type schedulable on a node with no free GPU. EKS
+		// host selection and reservation creation both read this figure.
+		if instancetypes.IsGPUType(it) {
+			typeCap.Available = min(typeCap.Available, rm.admissibleGPUInstances(name))
+		}
 		caps = append(caps, typeCap)
 	}
 	return totalVCPU, totalMemGB, reservedVCPU, reservedMemGB, allocVCPU, allocMemGB, caps
@@ -2757,8 +2755,11 @@ func (d *Daemon) applyGPUConfig(enabled bool) {
 		}
 		mgr, models, migProfiles := buildGPUPool(probe.Devices, d.config.Daemon)
 		d.gpuManager = mgr
-		d.resourceMgr.reloadGPUTypes(models, migProfiles, mgr)
+		// Claimer before types: reloadGPUTypes starts advertising GPU types and
+		// admitting against them, and a launch landing before the claimer is
+		// wired would otherwise boot without the GPUs it was admitted for.
 		d.instanceService.SetGPUClaimer(&daemonGPUClaimer{d: d})
+		d.resourceMgr.reloadGPUTypes(models, migProfiles, mgr)
 		slog.Info("GPU passthrough enabled via config reload", "gpus", len(probe.Devices))
 		return
 	}
@@ -2914,11 +2915,8 @@ func (rm *ResourceManager) admitLocked(instanceType *ec2.InstanceTypeInfo, count
 
 	requiresGPU := instancetypes.IsGPUType(instanceType)
 	availGPU := 0
-	if requiresGPU && rm.gpuManager != nil {
-		gpusNeeded := instancetypes.GPUCountForType(instanceTypeName)
-		if gpusNeeded > 0 {
-			availGPU = rm.gpuManager.Available() / gpusNeeded
-		}
+	if requiresGPU {
+		availGPU = rm.admissibleGPUInstances(instanceTypeName)
 	}
 
 	budget := canAllocateCount(
@@ -2936,6 +2934,24 @@ func (rm *ResourceManager) admitLocked(instanceType *ec2.InstanceTypeInfo, count
 	return n, "budget"
 }
 
+// admissibleGPUInstances returns how many instances of a GPU type the free pool
+// can back. Whole GPUs and MIG slices count separately, so a node with free
+// slices and no free whole GPU refuses a whole-GPU type rather than failing it.
+func (rm *ResourceManager) admissibleGPUInstances(instanceTypeName string) int {
+	if rm.gpuManager == nil {
+		return 0
+	}
+	gpusNeeded := instancetypes.GPUCountForType(instanceTypeName)
+	if gpusNeeded <= 0 {
+		return 0
+	}
+	if instancetypes.IsMIGType(instanceTypeName) {
+		profile := instancetypes.MIGProfileFromType(instanceTypeName)
+		return rm.gpuManager.AvailableSlices(profile) / gpusNeeded
+	}
+	return rm.gpuManager.AvailableWhole() / gpusNeeded
+}
+
 // liveMemGate clamps n by current MemAvailable, catching overcommit that the
 // static -m budget misses. Returns n unchanged when disabled or on read error.
 func (rm *ResourceManager) liveMemGate(n int, instanceType *ec2.InstanceTypeInfo) int {

spinifex/daemon/daemon_gpu_handlers_test.go

@@ -133,31 +133,37 @@ func TestHandleNodeStatus_MIG(t *testing.T) {
 	assert.Empty(t, byGI[2].InstanceID, "unclaimed slice is free")
 }
 
-func TestHandleNodeVMs_GPUAttachment_WholeGPU(t *testing.T) {
+func TestHandleNodeVMs_GPUAttachments_WholeGPU(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	instanceType := getTestInstanceType(t)
 
-	dev := gpu.GPUDevice{PCIAddress: "0000:01:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
-	mgr := gpu.NewManager([]gpu.GPUDevice{dev})
-	require.NoError(t, mgr.ReclaimByAddress(dev.PCIAddress, "i-gpu-vm"))
+	first := gpu.GPUDevice{PCIAddress: "0000:01:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
+	second := gpu.GPUDevice{PCIAddress: "0000:02:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
+	mgr := gpu.NewManager([]gpu.GPUDevice{first, second})
+	require.NoError(t, mgr.ReclaimByAddress(first.PCIAddress, "i-gpu-vm"))
+	require.NoError(t, mgr.ReclaimByAddress(second.PCIAddress, "i-gpu-vm"))
 	daemon.gpuManager = mgr
 
 	daemon.vmMgr.Insert(&vm.VM{
-		ID:             "i-gpu-vm",
-		Status:         vm.StateRunning,
-		InstanceType:   instanceType,
-		GPUAttachments: []gpu.GPUAttachment{{PCIAddress: dev.PCIAddress}},
+		ID:           "i-gpu-vm",
+		Status:       vm.StateRunning,
+		InstanceType: instanceType,
+		GPUAttachments: []gpu.GPUAttachment{
+			{PCIAddress: first.PCIAddress},
+			{PCIAddress: second.PCIAddress},
+		},
 	})
 
 	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.gpuwhole")
 
 	require.Len(t, resp.VMs, 1)
-	require.NotNil(t, resp.VMs[0].GPU)
-	assert.Equal(t, "NVIDIA A10", resp.VMs[0].GPU.Model)
-	assert.Equal(t, int64(23028), resp.VMs[0].GPU.VRAMMiB)
-	assert.Equal(t, dev.PCIAddress, resp.VMs[0].GPU.PCIAddress)
-	assert.Empty(t, resp.VMs[0].GPU.Profile)
-	assert.Empty(t, resp.VMs[0].GPU.MdevPath)
+	require.Len(t, resp.VMs[0].GPUs, 2)
+	assert.Equal(t, "NVIDIA A10", resp.VMs[0].GPUs[0].Model)
+	assert.Equal(t, int64(23028), resp.VMs[0].GPUs[0].VRAMMiB)
+	assert.Equal(t, first.PCIAddress, resp.VMs[0].GPUs[0].PCIAddress)
+	assert.Equal(t, second.PCIAddress, resp.VMs[0].GPUs[1].PCIAddress)
+	assert.Empty(t, resp.VMs[0].GPUs[0].Profile)
+	assert.Empty(t, resp.VMs[0].GPUs[0].MdevPath)
 }
 
 func TestHandleNodeVMs_GPUAttachment_MIG(t *testing.T) {
@@ -185,13 +191,12 @@ func TestHandleNodeVMs_GPUAttachment_MIG(t *testing.T) {
 
 	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.gpumig")
 
-	require.Len(t, resp.VMs, 1)
-	require.NotNil(t, resp.VMs[0].GPU)
-	assert.Equal(t, dev.Model, resp.VMs[0].GPU.Model)
-	assert.Equal(t, int64(24576), resp.VMs[0].GPU.VRAMMiB)
-	assert.Equal(t, "1g.24gb", resp.VMs[0].GPU.Profile)
-	assert.Equal(t, mdevPath, resp.VMs[0].GPU.MdevPath)
-	assert.Empty(t, resp.VMs[0].GPU.PCIAddress)
+	require.Len(t, resp.VMs[0].GPUs, 1)
+	assert.Equal(t, dev.Model, resp.VMs[0].GPUs[0].Model)
+	assert.Equal(t, int64(24576), resp.VMs[0].GPUs[0].VRAMMiB)
+	assert.Equal(t, "1g.24gb", resp.VMs[0].GPUs[0].Profile)
+	assert.Equal(t, mdevPath, resp.VMs[0].GPUs[0].MdevPath)
+	assert.Empty(t, resp.VMs[0].GPUs[0].PCIAddress)
 }
 
 func TestHandleNodeVMs_GPUAttachment_Absent(t *testing.T) {
@@ -207,5 +212,5 @@ func TestHandleNodeVMs_GPUAttachment_Absent(t *testing.T) {
 	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.nogpu")
 
 	require.Len(t, resp.VMs, 1)
-	assert.Nil(t, resp.VMs[0].GPU)
+	assert.Empty(t, resp.VMs[0].GPUs)
 }

spinifex/daemon/daemon_handlers.go

@@ -600,6 +600,26 @@ func resolveVMGPU(att gpu.GPUAttachment, byMdev, byPCI map[string]gpu.PoolEntry)
 	return nil
 }
 
+// resolveVMGPUs resolves every attachment it can, reporting a short list rather
+// than passing it off as complete. One unresolved attachment used to read as no
+// GPU at all; eight make a shortened list a plausible wrong answer.
+func resolveVMGPUs(instanceID string, attachments []gpu.GPUAttachment, byMdev, byPCI map[string]gpu.PoolEntry) []types.VMGPUInfo {
+	gpus := make([]types.VMGPUInfo, 0, len(attachments))
+	for _, attachment := range attachments {
+		if info := resolveVMGPU(attachment, byMdev, byPCI); info != nil {
+			gpus = append(gpus, *info)
+			continue
+		}
+		slog.Warn("VM GPU attachment has no pool entry",
+			"instanceId", instanceID, "pci", attachment.PCIAddress, "mdev", attachment.MdevPath)
+	}
+	if len(gpus) != len(attachments) {
+		slog.Warn("VM holds GPU attachments that did not resolve",
+			"instanceId", instanceID, "attached", len(attachments), "resolved", len(gpus))
+	}
+	return gpus
+}
+
 // handleNodeVMs responds with the list of VMs running on this node.
 // Used by the CLI: spx get vms.
 func (d *Daemon) handleNodeVMs(msg *nats.Msg) string {
@@ -622,9 +642,7 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) string {
 		if v.Instance != nil && v.Instance.LaunchTime != nil {
 			info.LaunchTime = v.Instance.LaunchTime.Unix()
 		}
-		if len(v.GPUAttachments) > 0 {
-			info.GPU = resolveVMGPU(v.GPUAttachments[0], poolByMdev, poolByPCI)
-		}
+		info.GPUs = resolveVMGPUs(v.ID, v.GPUAttachments, poolByMdev, poolByPCI)
 		vms = append(vms, info)
 	})
 

spinifex/daemon/gpu_adapter.go

@@ -1,6 +1,9 @@
 package daemon
 
 import (
+	"errors"
+	"fmt"
+
 	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 )
@@ -14,20 +17,47 @@ type daemonGPUClaimer struct {
 
 var _ handlers_ec2_instance.GPUClaimer = (*daemonGPUClaimer)(nil)
 
-func (g *daemonGPUClaimer) Claim(instanceID, profileName string) (*gpu.GPUAttachment, error) {
-	dev, mig, err := g.d.gpuManager.Claim(instanceID, profileName)
-	if err != nil {
-		return nil, err
+func (g *daemonGPUClaimer) manager() *gpu.Manager {
+	g.d.mu.Lock()
+	defer g.d.mu.Unlock()
+	return g.d.gpuManager
+}
+
+func (g *daemonGPUClaimer) Claim(instanceID, profileName string, count int) ([]gpu.GPUAttachment, error) {
+	if count < 1 {
+		return nil, fmt.Errorf("GPU claim count must be positive: %d", count)
 	}
-	if mig != nil {
-		return &gpu.GPUAttachment{MdevPath: mig.MdevPath}, nil
+	mgr := g.manager()
+	if mgr == nil {
+		return nil, errors.New("GPU manager is unavailable")
 	}
-	return &gpu.GPUAttachment{
-		PCIAddress:  dev.PCIAddress,
-		XVGAEnabled: gpuXVGAEnabled(dev, g.d.config.Daemon.GPUModelOverrides),
-	}, nil
+
+	attachments := make([]gpu.GPUAttachment, 0, count)
+	for range count {
+		dev, mig, err := mgr.Claim(instanceID, profileName)
+		if err != nil {
+			if rollbackErr := mgr.Release(instanceID); rollbackErr != nil &&
+				!errors.Is(rollbackErr, gpu.ErrNoGPUClaimed) {
+				return nil, errors.Join(err, fmt.Errorf("roll back GPU claims: %w", rollbackErr))
+			}
+			return nil, err
+		}
+		if mig != nil {
+			attachments = append(attachments, gpu.GPUAttachment{MdevPath: mig.MdevPath})
+			continue
+		}
+		attachments = append(attachments, gpu.GPUAttachment{
+			PCIAddress:  dev.PCIAddress,
+			XVGAEnabled: gpuXVGAEnabled(dev, g.d.config.Daemon.GPUModelOverrides),
+		})
+	}
+	return attachments, nil
 }
 
 func (g *daemonGPUClaimer) Release(instanceID string) error {
-	return g.d.gpuManager.Release(instanceID)
+	mgr := g.manager()
+	if mgr == nil {
+		return gpu.ErrNoGPUClaimed
+	}
+	return mgr.Release(instanceID)
 }

spinifex/daemon/gpu_adapter_test.go

@@ -0,0 +1,55 @@
+//test:in-package — daemonGPUClaimer and its Daemon field are unexported;
+package daemon
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newMIGClaimManager(t *testing.T, count int) *gpu.Manager {
+	t.Helper()
+
+	dev := gpu.GPUDevice{PCIAddress: "0000:03:00.0", Model: "NVIDIA A10"}
+	instances := make([]gpu.MIGInstance, 0, count)
+	for i := range count {
+		mdevPath := filepath.Join(t.TempDir(), "mdev")
+		require.NoError(t, os.WriteFile(mdevPath, []byte("mdev"), 0o600))
+		instances = append(instances, gpu.MIGInstance{
+			GIID:     i + 1,
+			MdevPath: mdevPath,
+			Profile:  gpu.MIGProfile{Name: "1g.10gb", MemoryMiB: 10 * 1024},
+		})
+	}
+	mgr := gpu.NewManager(nil)
+	mgr.AddMIGInstances(dev, instances)
+	return mgr
+}
+
+func TestDaemonGPUClaimerClaimsRequestedCount(t *testing.T) {
+	mgr := newMIGClaimManager(t, 2)
+	claimer := &daemonGPUClaimer{d: &Daemon{gpuManager: mgr}}
+
+	attachments, err := claimer.Claim("i-two-gpu", "1g.10gb", 2)
+	require.NoError(t, err)
+	require.Len(t, attachments, 2)
+	assert.NotEqual(t, attachments[0].MdevPath, attachments[1].MdevPath)
+	assert.Zero(t, mgr.AvailableSlices("1g.10gb"))
+
+	require.NoError(t, claimer.Release("i-two-gpu"))
+	assert.Equal(t, 2, mgr.AvailableSlices("1g.10gb"))
+}
+
+func TestDaemonGPUClaimerRollsBackPartialClaim(t *testing.T) {
+	mgr := newMIGClaimManager(t, 1)
+	claimer := &daemonGPUClaimer{d: &Daemon{gpuManager: mgr}}
+
+	attachments, err := claimer.Claim("i-too-many", "1g.10gb", 2)
+	require.Error(t, err)
+	assert.Nil(t, attachments)
+	assert.Equal(t, 1, mgr.AvailableSlices("1g.10gb"))
+}

spinifex/daemon/gpu_admission_test.go

@@ -0,0 +1,234 @@
+//test:in-package — drives ResourceManager.canAllocateLocked and builds
+package daemon
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gpuTypeForTest builds an InstanceTypeInfo whose cpu/mem fit the test node
+// comfortably, so only the GPU gate can bind the admission count.
+func gpuTypeForTest(name string, gpus int64) *ec2.InstanceTypeInfo {
+	return &ec2.InstanceTypeInfo{
+		InstanceType: aws.String(name),
+		VCpuInfo:     &ec2.VCpuInfo{DefaultVCpus: aws.Int64(2)},
+		MemoryInfo:   &ec2.MemoryInfo{SizeInMiB: aws.Int64(1024)},
+		GpuInfo: &ec2.GpuInfo{
+			Gpus: []*ec2.GpuDeviceInfo{{
+				Count:        aws.Int64(gpus),
+				Manufacturer: aws.String("NVIDIA"),
+				Name:         aws.String("A10G"),
+				MemoryInfo:   &ec2.GpuDeviceMemoryInfo{SizeInMiB: aws.Int64(24576)},
+			}},
+			TotalGpuMemoryInMiB: aws.Int64(24576 * gpus),
+		},
+	}
+}
+
+// carvedMIGManager returns a manager holding only carved MIG slices — no whole
+// GPU is free.
+func carvedMIGManager(t *testing.T, slices int) *gpu.Manager {
+	t.Helper()
+	root := t.TempDir()
+	dev := gpu.GPUDevice{
+		PCIAddress: "0000:01:00.0", VendorID: "10de", DeviceID: "20b5",
+		MIGCapable: true, MIGEnabled: true, IOMMUGroup: -1,
+	}
+	instances := make([]gpu.MIGInstance, slices)
+	for i := range instances {
+		path := filepath.Join(root, string(rune('a'+i)))
+		require.NoError(t, os.MkdirAll(path, 0755))
+		instances[i] = gpu.MIGInstance{
+			GIID:     i + 1,
+			MdevPath: path,
+			Profile:  gpu.MIGProfile{Name: "1g.10gb", MemoryMiB: 10240},
+		}
+	}
+	m := gpu.NewManager(nil)
+	m.AddMIGInstances(dev, instances)
+	return m
+}
+
+// A node with free slices and no free whole GPU must refuse a whole-GPU type at
+// admission, not fail it later at Claim.
+func TestAdmission_FreeSlicesDoNotBackAWholeGPUType(t *testing.T) {
+	rm := &ResourceManager{
+		hostVCPU:   64,
+		hostMemGB:  256.0,
+		gpuManager: carvedMIGManager(t, 7),
+	}
+
+	assert.Zero(t, rm.canAllocateLocked(gpuTypeForTest("g5.xlarge", 1), 10),
+		"seven slices are not one whole GPU")
+	assert.Equal(t, 7, rm.canAllocateLocked(gpuTypeForTest("mig.1g.10gb", 1), 10),
+		"the same seven slices back seven MIG instances")
+	assert.Zero(t, rm.canAllocateLocked(gpuTypeForTest("mig.3g.40gb", 1), 10),
+		"a profile with no slices carved is not available")
+}
+
+// A multi-GPU type divides the free whole GPUs by its own count.
+func TestAdmission_MultiGPUTypeDividesTheWholePool(t *testing.T) {
+	devices := make([]gpu.GPUDevice, 8)
+	for i := range devices {
+		devices[i] = gpu.GPUDevice{PCIAddress: string(rune('a' + i)), IOMMUGroup: i}
+	}
+	rm := &ResourceManager{
+		hostVCPU:   256,
+		hostMemGB:  1024.0,
+		gpuManager: gpu.NewManager(devices),
+	}
+
+	assert.Equal(t, 8, rm.canAllocateLocked(gpuTypeForTest("gpu.1x4c", 1), 10))
+	assert.Equal(t, 2, rm.canAllocateLocked(gpuTypeForTest("gpu.4x4c", 4), 10))
+	assert.Equal(t, 1, rm.canAllocateLocked(gpuTypeForTest("g5.48xlarge", 8), 10))
+}
+
+// A reservation is a counter over vCPU and memory slots and reserves no pool
+// entry, so its free slots say nothing about whether a GPU is there to claim.
+func TestReservationAvailable_GatesOnFreeGPUs(t *testing.T) {
+	gpuType := gpuTypeForTest("gpu.4x4c", 4)
+	devices := make([]gpu.GPUDevice, 4)
+	for i := range devices {
+		devices[i] = gpu.GPUDevice{PCIAddress: string(rune('a' + i)), IOMMUGroup: i}
+	}
+	mgr := gpu.NewManager(devices)
+
+	rm := &ResourceManager{
+		hostVCPU:      64,
+		hostMemGB:     256.0,
+		gpuManager:    mgr,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{"gpu.4x4c": gpuType},
+		reservations:  make(map[string]*capacityReservation),
+	}
+
+	// Three slots reserved, but only four GPUs on the node — one instance.
+	require.NoError(t, rm.CreateReservation(&capacityReservation{
+		ID:                    "cr-gpu",
+		AccountID:             "acct-a",
+		InstanceType:          "gpu.4x4c",
+		AvailabilityZone:      "ap-southeast-2a",
+		TotalInstanceCount:    3,
+		VCPUPerInstance:       2,
+		MemGBPerInstance:      1.0,
+		InstanceMatchCriteria: "open",
+		Tenancy:               "default",
+		InstancePlatform:      "Linux/UNIX",
+		CreateDate:            time.Now(),
+	}))
+
+	assert.Equal(t, 1, rm.ReservationAvailable("cr-gpu", "acct-a", gpuType),
+		"three reserved slots cannot outrun four GPUs at four per instance")
+
+	// With the GPUs gone from the pool the reservation backs nothing at all.
+	for i := range devices {
+		mgr.MarkFailed(devices[i].PCIAddress)
+	}
+	assert.Zero(t, rm.ReservationAvailable("cr-gpu", "acct-a", gpuType))
+}
+
+// ReservationAvailable reserves nothing, so two targeted launches can both
+// clear it. Without the write-lock re-check the loser consumes a reservation
+// slot and then dies at claim time with no GPU to take.
+func TestAllocateFromReservation_RechecksFreeGPUs(t *testing.T) {
+	gpuType := gpuTypeForTest("gpu.4x4c", 4)
+	devices := make([]gpu.GPUDevice, 4)
+	for i := range devices {
+		devices[i] = gpu.GPUDevice{PCIAddress: string(rune('a' + i)), IOMMUGroup: i}
+	}
+	mgr := gpu.NewManager(devices)
+
+	rm := &ResourceManager{
+		hostVCPU:      64,
+		hostMemGB:     256.0,
+		gpuManager:    mgr,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{"gpu.4x4c": gpuType},
+		reservations:  make(map[string]*capacityReservation),
+	}
+	require.NoError(t, rm.CreateReservation(&capacityReservation{
+		ID:                    "cr-gpu",
+		AccountID:             "acct-a",
+		InstanceType:          "gpu.4x4c",
+		AvailabilityZone:      "ap-southeast-2a",
+		TotalInstanceCount:    3,
+		VCPUPerInstance:       2,
+		MemGBPerInstance:      1.0,
+		InstanceMatchCriteria: "open",
+		Tenancy:               "default",
+		InstancePlatform:      "Linux/UNIX",
+		CreateDate:            time.Now(),
+	}))
+
+	require.NoError(t, rm.AllocateFromReservation("cr-gpu", "acct-a", gpuType),
+		"the first launch takes the node's only four GPUs")
+
+	// The GPUs leave the pool between the two calls — claimed by the winner of
+	// the race, failed, or dropped by a SIGHUP rebuild. Two slots still remain.
+	for i := range devices {
+		mgr.MarkFailed(devices[i].PCIAddress)
+	}
+
+	err := rm.AllocateFromReservation("cr-gpu", "acct-a", gpuType)
+	require.Error(t, err, "free slots do not conjure a GPU")
+	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error(),
+		"the reservation is not what ran out")
+}
+
+// The node census sized GPU types on host CPU and memory alone, which for the
+// gpu.* shapes is never the binding constraint. EKS host selection and
+// CreateCapacityReservation both size themselves from this figure.
+func TestGetResourceStats_GPUTypesAreGatedOnGPUs(t *testing.T) {
+	gpuType := gpuTypeForTest("gpu.4x4c", 4)
+	devices := []gpu.GPUDevice{
+		{PCIAddress: "a", IOMMUGroup: 0},
+		{PCIAddress: "b", IOMMUGroup: 1},
+		{PCIAddress: "c", IOMMUGroup: 2},
+		{PCIAddress: "d", IOMMUGroup: 3},
+	}
+	mgr := gpu.NewManager(devices)
+	rm := &ResourceManager{
+		hostVCPU:      64,
+		hostMemGB:     256.0,
+		gpuManager:    mgr,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{"gpu.4x4c": gpuType},
+		reservations:  make(map[string]*capacityReservation),
+	}
+
+	capFor := func() types.InstanceTypeCap {
+		t.Helper()
+		_, _, _, _, _, _, caps := rm.GetResourceStats()
+		for _, c := range caps {
+			if c.Name == "gpu.4x4c" {
+				return c
+			}
+		}
+		t.Fatal("gpu.4x4c missing from the census")
+		return types.InstanceTypeCap{}
+	}
+
+	assert.Equal(t, 1, capFor().Available, "four GPUs back one four-GPU instance")
+
+	for i := range devices {
+		mgr.MarkFailed(devices[i].PCIAddress)
+	}
+	assert.Zero(t, capFor().Available,
+		"a node with no free GPU must not advertise a GPU type as schedulable")
+}
+
+// A non-GPU reservation keeps answering on slots alone.
+func TestReservationAvailable_NonGPUTypeIsUngated(t *testing.T) {
+	rm := newReservationTestRM()
+	rm.gpuManager = gpu.NewManager(nil)
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 2)))
+
+	assert.Equal(t, 2, rm.ReservationAvailable("cr-1", "acct-a", microType()))
+}

spinifex/daemon/resource_test.go

@@ -1,6 +1,7 @@
 package daemon
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -388,14 +389,19 @@ func TestCanAllocateLocked_MIGvsWholeGPU(t *testing.T) {
 	}
 
 	// Host with 8 schedulable vCPUs and 32 GiB schedulable memory (after
-	// reserve), and a GPU pool with 5 free slots — plenty for the GPU-slot
-	// gate so these assertions exercise the cpu/mem gate in isolation.
+	// reserve), and a GPU pool with 5 free whole GPUs and 5 MIG-capable ones —
+	// plenty for either GPU gate, so these assertions exercise the cpu/mem gate
+	// in isolation.
+	mgr := gpu.NewManager(make([]gpu.GPUDevice, 5))
+	for i := range 5 {
+		mgr.AddMIGGPU(gpu.GPUDevice{PCIAddress: fmt.Sprintf("0000:8%d:00.0", i)})
+	}
 	rm := &ResourceManager{
 		hostVCPU:     10,
 		hostMemGB:    34.0,
 		reservedVCPU: 2,
 		reservedMem:  2.0,
-		gpuManager:   gpu.NewManager(make([]gpu.GPUDevice, 5)),
+		gpuManager:   mgr,
 	}
 
 	migType := &ec2.InstanceTypeInfo{

spinifex/gpu/manager.go

@@ -245,6 +245,11 @@ func (m *Manager) Release(instanceID string) error {
 			continue
 		}
 
+		// Scoped to this entry, not the whole release: an instance can hold
+		// several GPUs, and one that unbound cleanly stays available however
+		// its siblings fared.
+		var entryErr error
+
 		for _, member := range entry.groupMembers {
 			orig := entry.memberDrivers[member.PCIAddress]
 			if orig == "vfio-pci" {
@@ -258,6 +263,9 @@ func (m *Manager) Release(instanceID string) error {
 			if err := unbindVFIO(m.sysfsRoot, member.PCIAddress, orig); err != nil {
 				slog.Error("GPU release failed for IOMMU group member",
 					"instance", instanceID, "pci", member.PCIAddress, "err", err)
+				if entryErr == nil {
+					entryErr = err
+				}
 				if firstErr == nil {
 					firstErr = err
 				}
@@ -268,7 +276,7 @@ func (m *Manager) Release(instanceID string) error {
 		entry.groupMembers = nil
 		entry.memberDrivers = nil
 
-		if firstErr != nil {
+		if entryErr != nil {
 			entry.Available = false
 			slog.Error("GPU marked unavailable after failed release — operator action required",
 				"gpu", entry.Device.PCIAddress)
@@ -387,7 +395,9 @@ func (m *Manager) AddMIGInstances(device GPUDevice, instances []MIGInstance) {
 	}
 }
 
-// Available returns the count of GPUs that can be claimed right now.
+// Available counts free whole GPUs and free MIG slices alike. Admission must
+// use AvailableWhole or AvailableSlices instead: two whole GPUs and fourteen
+// carved slices answer 16, which would admit two 8-GPU instances against two.
 func (m *Manager) Available() int {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -400,6 +410,37 @@ func (m *Manager) Available() int {
 	return n
 }
 
+// AvailableWhole returns the count of whole GPUs that can be claimed right now.
+// A GPU carved into MIG slices is not one of them, and neither is an un-carved
+// MIG-capable GPU held in freeMIGGPUs.
+func (m *Manager) AvailableWhole() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	n := 0
+	for _, e := range m.pool {
+		if e.Available && e.InstanceID == "" && e.MIGInstance == nil {
+			n++
+		}
+	}
+	return n
+}
+
+// AvailableSlices counts free slices of this profile plus one per un-carved
+// GPU in a single snapshot. The un-carved budget is shared across profiles:
+// this answers for one type in isolation, not a sum across different types.
+func (m *Manager) AvailableSlices(profileName string) int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	n := len(m.freeMIGGPUs)
+	for _, e := range m.pool {
+		if e.Available && e.InstanceID == "" && e.MIGInstance != nil &&
+			e.MIGInstance.Profile.Name == profileName {
+			n++
+		}
+	}
+	return n
+}
+
 // AllocatedCount returns the number of GPUs currently claimed by instances.
 func (m *Manager) AllocatedCount() int {
 	m.mu.Lock()

spinifex/gpu/manager_available_test.go

@@ -0,0 +1,104 @@
+//test:in-package — reuses the in-package sysfs and MIG fixtures
+package gpu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// A pool holding whole GPUs and carved slices together must not report one kind
+// as the other. Available counts both, which is why admission uses neither it
+// nor a division of it.
+func TestAvailableWholeAndSlicesCountSeparately(t *testing.T) {
+	root := t.TempDir()
+
+	m := NewManager([]GPUDevice{
+		{PCIAddress: "0000:01:00.0", IOMMUGroup: 1},
+		{PCIAddress: "0000:02:00.0", IOMMUGroup: 2},
+	})
+	carved := newMIGDevice("0000:03:00.0")
+	m.AddMIGInstances(carved, []MIGInstance{
+		{GIID: 1, MdevPath: makeMdevDir(t, root, "uuid-gi1"), Profile: MIGProfile{Name: "1g.10gb", MemoryMiB: 10240}},
+		{GIID: 2, MdevPath: makeMdevDir(t, root, "uuid-gi2"), Profile: MIGProfile{Name: "1g.10gb", MemoryMiB: 10240}},
+		{GIID: 3, MdevPath: makeMdevDir(t, root, "uuid-gi3"), Profile: MIGProfile{Name: "3g.40gb", MemoryMiB: 40960}},
+	})
+
+	assert.Equal(t, 5, m.Available(), "Available counts every free entry")
+	assert.Equal(t, 2, m.AvailableWhole(), "a carved GPU is not a whole GPU")
+	assert.Equal(t, 2, m.AvailableSlices("1g.10gb"))
+	assert.Equal(t, 1, m.AvailableSlices("3g.40gb"))
+	assert.Zero(t, m.AvailableSlices("7g.80gb"), "a profile with no slices carved is not available")
+}
+
+// Each un-carved GPU contributes one conservative slot to a single profile's
+// admission decision. Counts for different profiles must not be added together.
+func TestFreeMIGGPUsAreSharedAcrossProfiles(t *testing.T) {
+	m := NewManager(nil)
+	m.AddMIGGPU(newMIGDevice("0000:01:00.0"))
+	m.AddMIGGPU(newMIGDevice("0000:02:00.0"))
+
+	assert.Equal(t, 2, m.AvailableSlices("1g.10gb"), "both GPUs are carvable, once each")
+	assert.Equal(t, 2, m.AvailableSlices("7g.80gb"), "the same budget backs either profile, not both")
+	assert.Zero(t, m.AvailableWhole(), "a MIG-capable GPU held for carving is not offerable whole")
+	assert.Zero(t, m.Available(), "freeMIGGPUs are not pool entries yet")
+}
+
+func TestAvailableSlicesIncludesCarvableGPUsOnce(t *testing.T) {
+	m := NewManager(nil)
+	m.pool = []gpuEntry{
+		{Available: true, MIGInstance: &MIGInstance{Profile: MIGProfile{Name: "1g.10gb"}}},
+		{Available: true, InstanceID: "i-busy", MIGInstance: &MIGInstance{Profile: MIGProfile{Name: "1g.10gb"}}},
+		{Available: false, MIGInstance: &MIGInstance{Profile: MIGProfile{Name: "1g.10gb"}}},
+		{Available: true, MIGInstance: &MIGInstance{Profile: MIGProfile{Name: "7g.80gb"}}},
+		{Available: true},
+	}
+	m.AddMIGGPU(newMIGDevice("0000:01:00.0"))
+	assert.Equal(t, 2, m.AvailableSlices("1g.10gb"))
+	assert.Equal(t, 2, m.AvailableSlices("7g.80gb"))
+	assert.Equal(t, 1, m.AvailableSlices("3g.40gb"))
+}
+
+func TestAvailableSlicesSnapshotsPoolTransitions(t *testing.T) {
+	dev := newMIGDevice("0000:01:00.0")
+	m := NewManager(nil)
+	m.AddMIGGPU(dev)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for range 1000 {
+			m.mu.Lock()
+			m.freeMIGGPUs = nil
+			m.pool = []gpuEntry{{Device: dev, Available: true,
+				MIGInstance: &MIGInstance{Profile: MIGProfile{Name: "7g.80gb"}},
+			}}
+			m.mu.Unlock()
+			m.mu.Lock()
+			m.pool = nil
+			m.freeMIGGPUs = []GPUDevice{dev}
+			m.mu.Unlock()
+		}
+	}()
+	for range 1000 {
+		assert.Equal(t, 1, m.AvailableSlices("7g.80gb"), "a GPU must not be counted in both pools")
+	}
+	<-done
+}
+
+// The whole-GPU claim path already skips slices; only the arithmetic was wrong.
+func TestAvailableWholeDropsAClaimedGPU(t *testing.T) {
+	root, dev := buildManagerSysfs(t)
+	m := newManagerForTest([]GPUDevice{dev}, root)
+
+	assert.Equal(t, 1, m.AvailableWhole())
+
+	if _, _, err := m.Claim("i-1", ""); err != nil {
+		t.Fatalf("claim: %v", err)
+	}
+	assert.Zero(t, m.AvailableWhole())
+
+	if err := m.Release("i-1"); err != nil {
+		t.Fatalf("release: %v", err)
+	}
+	assert.Equal(t, 1, m.AvailableWhole())
+}

spinifex/gpu/manager_release_scope_test.go

@@ -0,0 +1,70 @@
+// asserts on unexported pool state that no exported accessor distinguishes.
+//
+//test:in-package — drives Release against the in-package sysfs fixtures and
+package gpu
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A multi-GPU instance releasing with one bad device must lose only that one.
+// firstErr was scoped to the whole release, so the first failure condemned
+// every GPU after it — harmless at one GPU per instance, node-emptying at eight.
+func TestManagerRelease_FailureIsScopedToTheFailingGPU(t *testing.T) {
+	root := t.TempDir()
+	buildSysfsDevice(t, root, "0000:03:00.0", "0x030200", "0x10de", "0x2236", "nvidia", 7)
+	buildSysfsDevice(t, root, "0000:04:00.0", "0x030200", "0x10de", "0x2236", "nvidia", 8)
+	makeSysfsDriverDir(t, root, "vfio-pci")
+	makeSysfsDriverDir(t, root, "nvidia")
+
+	m := newManagerForTest([]GPUDevice{
+		{PCIAddress: "0000:03:00.0", Vendor: VendorNVIDIA, VendorID: "10de", DeviceID: "2236",
+			IOMMUGroup: 7, OriginalDriver: "nvidia"},
+		{PCIAddress: "0000:04:00.0", Vendor: VendorNVIDIA, VendorID: "10de", DeviceID: "2236",
+			IOMMUGroup: 8, OriginalDriver: "nvidia"},
+	}, root)
+
+	for range 2 {
+		_, _, err := m.Claim("i-001", "")
+		require.NoError(t, err)
+	}
+	require.Zero(t, m.AvailableWhole(), "both GPUs are claimed")
+
+	// The fixture does not move the driver symlink the way the kernel does on
+	// bind, and Release skips a device that is not vfio-pci bound.
+	vfioDriverPath := filepath.Join(root, "bus/pci/drivers/vfio-pci")
+	for _, addr := range []string{"0000:03:00.0", "0000:04:00.0"} {
+		devPath := filepath.Join(root, "bus/pci/devices", addr)
+		require.NoError(t, os.Remove(filepath.Join(devPath, "driver")))
+		require.NoError(t, os.Symlink(vfioDriverPath, filepath.Join(devPath, "driver")))
+	}
+
+	// Break only the first GPU's unbind, by way of its own driver_override.
+	// A directory in the file's place fails the write whatever the test uid,
+	// where a read-only mode would not stop root.
+	override := filepath.Join(root, "bus/pci/devices/0000:03:00.0/driver_override")
+	require.NoError(t, os.Remove(override))
+	require.NoError(t, os.Mkdir(override, 0o755))
+
+	err := m.Release("i-001")
+	require.Error(t, err, "a failed unbind is still reported")
+
+	assert.Equal(t, 1, m.AvailableWhole(),
+		"the GPU that unbound cleanly must return to the pool")
+	assert.Equal(t, 2, m.TotalCount(), "both GPUs are still physically present")
+
+	for _, e := range m.pool {
+		switch e.Device.PCIAddress {
+		case "0000:03:00.0":
+			assert.False(t, e.Available, "the GPU that failed to unbind is held back")
+		case "0000:04:00.0":
+			assert.True(t, e.Available, "its sibling was never touched by the failure")
+		}
+		assert.Empty(t, e.InstanceID, "every entry is unbound from the instance either way")
+	}
+}

spinifex/handlers/ec2/instance/service.go

@@ -68,11 +68,12 @@ type InstanceTypeAllocator interface {
 	InstanceTypes() map[string]*ec2.InstanceTypeInfo
 }
 
-// GPUClaimer binds a GPU for a starting instance and returns an attachment
-// descriptor. For whole-GPU passthrough the descriptor carries the PCI address;
-// for MIG slices it carries the mdev path. nil claimer means no GPU passthrough.
+// GPUClaimer binds GPUs for a starting instance and returns their attachment
+// descriptors. For whole-GPU passthrough each descriptor carries a PCI address;
+// for MIG slices it carries an mdev path. Claims are all-or-nothing. A nil
+// claimer means no GPU passthrough.
 type GPUClaimer interface {
-	Claim(instanceID, profileName string) (*gpu.GPUAttachment, error)
+	Claim(instanceID, profileName string, count int) ([]gpu.GPUAttachment, error)
 	Release(instanceID string) error
 }
 

spinifex/handlers/ec2/instance/service_impl.go

@@ -22,7 +22,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/filterutil"
-	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_dns "github.com/mulgadc/spinifex/spinifex/handlers/dns"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
@@ -1092,17 +1091,37 @@ func (s *InstanceServiceImpl) LaunchRunInstances(ctx context.Context, instances
 			instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
 		}
 
-		if s.gpuClaimer != nil && instancetypes.IsGPUType(instanceType) {
-			profileName := instancetypes.MIGProfileFromType(aws.StringValue(instanceType.InstanceType))
-			att, gpuErr := s.gpuClaimer.Claim(instance.ID, profileName)
-			if gpuErr != nil {
-				slog.ErrorContext(ctx, "LaunchRunInstances: GPU claim failed", "instanceId", instance.ID, "err", gpuErr)
+		if instancetypes.IsGPUType(instanceType) {
+			// Admission gates on the daemon's GPU manager, which is a different
+			// field wired at a different moment. Booting a GPU type without a
+			// claimer would hand the customer a running, GPU-less instance.
+			if s.gpuClaimer == nil {
+				slog.ErrorContext(ctx, "LaunchRunInstances: GPU type admitted with no GPU claimer wired",
+					"instanceId", instance.ID, "instanceType", aws.StringValue(instanceType.InstanceType))
 				s.vmMgr.MarkFailed(ctx, instance, "gpu_claim_failed")
 				continue
 			}
-			instance.GPUAttachments = []gpu.GPUAttachment{*att}
-			slog.InfoContext(ctx, "LaunchRunInstances: GPU claimed for instance", "instanceId", instance.ID,
-				"pci", att.PCIAddress, "mdev", att.MdevPath)
+			profileName := instancetypes.MIGProfileFromType(aws.StringValue(instanceType.InstanceType))
+			gpuCount := instancetypes.GPUCountForType(aws.StringValue(instanceType.InstanceType))
+			attachments, gpuErr := s.gpuClaimer.Claim(instance.ID, profileName, gpuCount)
+			if gpuErr != nil {
+				slog.ErrorContext(ctx, "LaunchRunInstances: GPU claim failed", "instanceId", instance.ID, "count", gpuCount, "err", gpuErr)
+				s.vmMgr.MarkFailed(ctx, instance, "gpu_claim_failed")
+				continue
+			}
+			if len(attachments) != gpuCount {
+				slog.ErrorContext(ctx, "LaunchRunInstances: GPU claim returned the wrong count",
+					"instanceId", instance.ID, "requested", gpuCount, "got", len(attachments))
+				if releaseErr := s.gpuClaimer.Release(instance.ID); releaseErr != nil {
+					slog.ErrorContext(ctx, "LaunchRunInstances: GPU release after short claim failed",
+						"instanceId", instance.ID, "err", releaseErr)
+				}
+				s.vmMgr.MarkFailed(ctx, instance, "gpu_claim_failed")
+				continue
+			}
+			instance.GPUAttachments = attachments
+			slog.InfoContext(ctx, "LaunchRunInstances: GPUs claimed for instance", "instanceId", instance.ID,
+				"count", len(attachments), "attachments", attachments)
 		}
 
 		if err := s.vmMgr.Run(ctx, instance); err != nil {
@@ -2593,22 +2612,50 @@ func (s *InstanceServiceImpl) StartStoppedInstance(ctx context.Context, input *S
 	instance.DesiredState = vm.DesiredRunning
 	s.vmMgr.Insert(instance)
 
-	// Claim GPU for GPU instance types.
+	// Reclaim the GPU count the instance type advertises, which is what the
+	// instance was admitted against. A record holding a different number was
+	// written by a build that took the count from somewhere else.
 	gpuClaimed := false
-	if s.gpuClaimer != nil && instancetypes.IsGPUType(instanceType) {
-		profileName := instancetypes.MIGProfileFromType(aws.StringValue(instanceType.InstanceType))
-		att, gpuErr := s.gpuClaimer.Claim(instance.ID, profileName)
-		if gpuErr != nil {
-			slog.ErrorContext(ctx, "StartStoppedInstance: GPU claim failed", "instanceId", input.InstanceID, "err", gpuErr)
+	if instancetypes.IsGPUType(instanceType) {
+		// Admission gates on the daemon's GPU manager, which is a different
+		// field wired at a different moment. Starting a GPU type without a
+		// claimer would hand the customer a running, GPU-less instance.
+		if s.gpuClaimer == nil {
+			slog.ErrorContext(ctx, "StartStoppedInstance: GPU type admitted with no GPU claimer wired",
+				"instanceId", input.InstanceID, "instanceType", instance.InstanceType)
 			s.resourceMgr.Deallocate(instanceType)
 			s.vmMgr.Delete(instance.ID)
 			s.restoreClaimedStoppedInstance(ctx, instance)
 			return nil, errors.New(awserrors.ErrorInsufficientInstanceCapacity)
 		}
-		instance.GPUAttachments = []gpu.GPUAttachment{*att}
+		profileName := instancetypes.MIGProfileFromType(aws.StringValue(instanceType.InstanceType))
+		gpuCount := instancetypes.GPUCountForType(instance.InstanceType)
+		if recorded := len(instance.GPUAttachments); recorded > 0 && recorded != gpuCount {
+			slog.WarnContext(ctx, "StartStoppedInstance: recorded GPU count differs from the instance type",
+				"instanceId", input.InstanceID, "instanceType", instance.InstanceType,
+				"recorded", recorded, "reclaiming", gpuCount)
+		}
+		attachments, gpuErr := s.gpuClaimer.Claim(instance.ID, profileName, gpuCount)
+		if gpuErr == nil && len(attachments) != gpuCount {
+			slog.ErrorContext(ctx, "StartStoppedInstance: GPU claim returned the wrong count",
+				"instanceId", input.InstanceID, "requested", gpuCount, "got", len(attachments))
+			if relErr := s.gpuClaimer.Release(instance.ID); relErr != nil {
+				slog.ErrorContext(ctx, "StartStoppedInstance: GPU release after short claim failed",
+					"instanceId", input.InstanceID, "err", relErr)
+			}
+			gpuErr = errors.New("GPU claim returned fewer GPUs than the instance type requires")
+		}
+		if gpuErr != nil {
+			slog.ErrorContext(ctx, "StartStoppedInstance: GPU claim failed", "instanceId", input.InstanceID, "count", gpuCount, "err", gpuErr)
+			s.resourceMgr.Deallocate(instanceType)
+			s.vmMgr.Delete(instance.ID)
+			s.restoreClaimedStoppedInstance(ctx, instance)
+			return nil, errors.New(awserrors.ErrorInsufficientInstanceCapacity)
+		}
+		instance.GPUAttachments = attachments
 		gpuClaimed = true
-		slog.InfoContext(ctx, "GPU claimed for instance", "instanceId", input.InstanceID,
-			"pci", att.PCIAddress, "mdev", att.MdevPath)
+		slog.InfoContext(ctx, "GPUs claimed for instance", "instanceId", input.InstanceID,
+			"count", len(attachments), "attachments", attachments)
 	}
 
 	if err := s.vmMgr.Run(ctx, instance); err != nil {

spinifex/handlers/ec2/instance/service_impl_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/tags"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 	spxtypes "github.com/mulgadc/spinifex/spinifex/types"
@@ -2318,19 +2319,23 @@ func TestTerminateStoppedInstance_ReleaseAttachedENIs(t *testing.T) {
 // --- StartStoppedInstance tests ---
 
 type fakeGPUClaimer struct {
-	claimed    []string
-	released   []string
-	claimErr   error
-	attachment gpu.GPUAttachment
+	claimed     []string
+	claimCounts []int
+	released    []string
+	claimErr    error
+	attachments []gpu.GPUAttachment
 }
 
-func (f *fakeGPUClaimer) Claim(instanceID, _ string) (*gpu.GPUAttachment, error) {
+func (f *fakeGPUClaimer) Claim(instanceID, _ string, count int) ([]gpu.GPUAttachment, error) {
 	f.claimed = append(f.claimed, instanceID)
+	f.claimCounts = append(f.claimCounts, count)
 	if f.claimErr != nil {
 		return nil, f.claimErr
 	}
-	att := f.attachment
-	return &att, nil
+	if f.attachments != nil {
+		return append([]gpu.GPUAttachment(nil), f.attachments...), nil
+	}
+	return make([]gpu.GPUAttachment, count), nil
 }
 
 func (f *fakeGPUClaimer) Release(instanceID string) error {
@@ -2759,6 +2764,131 @@ func TestStartStoppedInstance_ConcurrentClaimRace(t *testing.T) {
 	assert.False(t, stillInMgr, "a failed Run must not leave the VM in the manager map")
 }
 
+// A restart reclaims the count the instance type advertises, which is what the
+// instance was admitted against. A record holding some other number — the only
+// source of one was the withdrawn launch override — does not move the count.
+func TestStartStoppedInstance_ReclaimsTheInstanceTypeGPUCount(t *testing.T) {
+	tests := []struct {
+		name      string
+		itype     string
+		recorded  int
+		wantClaim int
+	}{
+		{name: "multi-GPU type", itype: "g5.12xlarge", recorded: 4, wantClaim: 4},
+		{name: "record disagrees with the type", itype: "g5.2xlarge", recorded: 3, wantClaim: 1},
+		{name: "legacy record with no attachments", itype: "g5.12xlarge", recorded: 0, wantClaim: 4},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+
+			id := "i-multi-gpu"
+			attachments := make([]gpu.GPUAttachment, test.recorded)
+			for i := range attachments {
+				attachments[i] = gpu.GPUAttachment{PCIAddress: fmt.Sprintf("0000:0%d:00.0", i+1)}
+			}
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+				id: {
+					ID:             id,
+					Status:         vm.StateStopped,
+					AccountID:      "acc",
+					InstanceType:   test.itype,
+					GPUAttachments: attachments,
+				},
+			}}
+			instanceType := &ec2.InstanceTypeInfo{
+				InstanceType: aws.String(test.itype),
+				GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{{
+					Count: aws.Int64(int64(instancetypes.GPUCountForType(test.itype))),
+				}}},
+			}
+			prov := &fakeResourceCapacityProvider{
+				instanceTypes: map[string]*ec2.InstanceTypeInfo{test.itype: instanceType},
+			}
+			claimer := &fakeGPUClaimer{}
+			mgr := vm.NewManagerWithDeps(vm.Deps{VolumeMounter: raceVolumeMounter{}})
+			svc := &InstanceServiceImpl{
+				stoppedStore: store,
+				resourceMgr:  prov,
+				vmMgr:        mgr,
+				gpuClaimer:   claimer,
+			}
+
+			_, err := svc.StartStoppedInstance(
+				context.Background(),
+				&StartStoppedInstanceInput{InstanceID: id},
+				"acc",
+			)
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+			assert.Equal(t, []int{test.wantClaim}, claimer.claimCounts)
+			assert.Equal(t, []string{id}, claimer.released)
+		})
+	}
+}
+
+// A GPU type that cannot be given its GPUs must not start. Admission gates on
+// the daemon's GPU manager, a different field wired at a different moment, so
+// neither a missing claimer nor a short claim is caught anywhere else — and
+// both would otherwise return a running, GPU-less instance.
+func TestStartStoppedInstance_RefusesWhenGPUsCannotBeGuaranteed(t *testing.T) {
+	tests := []struct {
+		name         string
+		claimer      GPUClaimer
+		wantReleased bool
+	}{
+		{name: "no claimer wired", claimer: nil},
+		{
+			name:         "claim returns fewer GPUs than the type requires",
+			claimer:      &fakeGPUClaimer{attachments: []gpu.GPUAttachment{{PCIAddress: "0000:01:00.0"}}},
+			wantReleased: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+
+			const id, itype = "i-multi-gpu", "g5.12xlarge"
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{
+				id: {ID: id, Status: vm.StateStopped, AccountID: "acc", InstanceType: itype},
+			}}
+			instanceType := &ec2.InstanceTypeInfo{
+				InstanceType: aws.String(itype),
+				GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{{
+					Count: aws.Int64(int64(instancetypes.GPUCountForType(itype))),
+				}}},
+			}
+			prov := &fakeResourceCapacityProvider{
+				instanceTypes: map[string]*ec2.InstanceTypeInfo{itype: instanceType},
+			}
+			svc := &InstanceServiceImpl{
+				stoppedStore: store,
+				resourceMgr:  prov,
+				vmMgr:        vm.NewManagerWithDeps(vm.Deps{VolumeMounter: raceVolumeMounter{}}),
+				gpuClaimer:   test.claimer,
+			}
+
+			_, err := svc.StartStoppedInstance(
+				context.Background(),
+				&StartStoppedInstanceInput{InstanceID: id},
+				"acc",
+			)
+			require.Error(t, err)
+			assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error(),
+				"a GPU that cannot be claimed is a capacity refusal, not a launch")
+
+			if test.wantReleased {
+				claimer, ok := test.claimer.(*fakeGPUClaimer)
+				require.True(t, ok)
+				assert.Equal(t, []string{id}, claimer.released,
+					"the partial claim must not be left holding pool entries")
+			}
+		})
+	}
+}
+
 // --- PrepareRunInstances / ec2.cmd dispatch tests ---------------------------
 
 type fakeAMILoader struct {
@@ -2946,6 +3076,46 @@ func TestPrepareRunInstances_InsufficientCapacity(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error())
 }
 
+// ElasticInferenceAccelerators is a real EC2 parameter for Elastic Inference,
+// whose types are eia1.*/eia2.*. It carries no Spinifex meaning: the GPU count
+// comes from the instance type, so Type=gpu,Count=3 neither succeeds as an
+// override nor gates capacity.
+func TestPrepareRunInstances_ElasticInferenceAcceleratorIsNotAGPUOverride(t *testing.T) {
+	instanceType := &ec2.InstanceTypeInfo{
+		InstanceType: aws.String("g5.2xlarge"),
+		GpuInfo: &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{{
+			Count: aws.Int64(1),
+		}}},
+	}
+	types := map[string]*ec2.InstanceTypeInfo{"g5.2xlarge": instanceType}
+	prov := &fakeResourceCapacityProvider{
+		instanceTypes: types,
+		canAllocFn:    func(_ *ec2.InstanceTypeInfo, count int) int { return count },
+	}
+	svc := &InstanceServiceImpl{
+		instanceTypes: types,
+		amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{
+			"ami-1": {ImageOwnerAlias: "acc"},
+		}},
+		resourceMgr: prov,
+		gpuClaimer:  &fakeGPUClaimer{},
+	}
+
+	_, instances, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
+		InstanceType: aws.String("g5.2xlarge"),
+		ImageId:      aws.String("ami-1"),
+		MinCount:     aws.Int64(2),
+		MaxCount:     aws.Int64(2),
+		ElasticInferenceAccelerators: []*ec2.ElasticInferenceAccelerator{{
+			Type: aws.String("gpu"), Count: aws.Int64(3),
+		}},
+	}, "acc", "")
+	require.NoError(t, err)
+	assert.Len(t, instances, 2)
+	assert.Equal(t, 1, instancetypes.GPUCountForType("g5.2xlarge"),
+		"the count must stay a lookup on the type name")
+}
+
 func TestPrepareRunInstances_HappyPathNoENI(t *testing.T) {
 	types, _ := defaultPrepareInstanceTypes()
 	prov := &fakeResourceCapacityProvider{

spinifex/handlers/ecs/capacity.go

@@ -107,7 +107,14 @@ func (s *Service) ProvisionCapacity(ctx context.Context, input *ProvisionCapacit
 
 	var amiID string
 	if instancetypes.IsGPUTypeName(instanceType) {
-		amiID, err = lookupECSGPUNodeAMI(ctx, s.deps.Images, accountID, instancetypes.GPUVendorForType(instanceType))
+		// Refusing beats the non-GPU AMI: that boots a container instance with
+		// the GPUs passed through and no driver, which registers no GPU
+		// resource and strands every GPU task as PENDING with nothing logged.
+		vendor := instancetypes.GPUVendorForType(instanceType)
+		if vendor == "" {
+			return nil, fmt.Errorf("ecs: instance type %s does not name a GPU vendor, so its GPU node AMI cannot be resolved", instanceType)
+		}
+		amiID, err = lookupECSGPUNodeAMI(ctx, s.deps.Images, accountID, vendor)
 	} else {
 		amiID, err = lookupECSNodeAMI(ctx, s.deps.Images, accountID)
 	}

spinifex/handlers/eks/nodegroup.go

@@ -253,6 +253,14 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 	minSize, maxSize, desired := scalingFromInput(input.ScalingConfig)
 
 	gpuEnabled, gpuVendor := gpuFieldsForInstanceTypes(instanceTypes)
+	if gpuEnabled && gpuVendor == "" {
+		// Caught here rather than at launch: without a vendor there is no GPU
+		// AMI to resolve, and the nodegroup would otherwise reach Ready with
+		// the GPUs passed through, no driver, and every GPU pod Pending.
+		slog.ErrorContext(ctx, "createNodegroup: GPU instance types name no GPU vendor",
+			"cluster", cluster, "nodegroup", ng, "instanceTypes", instanceTypes)
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
 
 	amiType := aws.StringValue(input.AmiType)
 	if amiType == "" {

spinifex/instancetypes/definitions.go

@@ -1,6 +1,8 @@
 package instancetypes
 
 import (
+	"fmt"
+	"maps"
 	"slices"
 	"strings"
 )
@@ -69,6 +71,8 @@ func GPUModelForVendorDevice(vendorID, deviceID string) *GPUModel {
 // a GPU instance type's family prefix (e.g. "g5.xlarge" -> "g5"), or "" if
 // instanceType is not a known GPU family.
 func GPUVendorForType(instanceType string) string {
+	// gpu.* and mig.* fall through to "": neither names a family, so no
+	// knownGPUModels entry matches and the vendor comes from discovery instead.
 	family, _, _ := strings.Cut(instanceType, ".")
 	for i := range knownGPUModels {
 		if knownGPUModels[i].Family == family {
@@ -78,9 +82,12 @@ func GPUVendorForType(instanceType string) string {
 	return ""
 }
 
-// IsGPUTypeName reports whether instanceType belongs to a known GPU family.
+// IsGPUTypeName reports whether instanceType is GPU-backed. gpu.* and mig.*
+// qualify even though GPUVendorForType cannot name their vendor: the vendor
+// comes from discovery, so callers needing one must handle an empty answer.
 func IsGPUTypeName(instanceType string) bool {
-	return GPUVendorForType(instanceType) != ""
+	return IsBuiltinGPUType(instanceType) || IsMIGType(instanceType) ||
+		GPUVendorForType(instanceType) != ""
 }
 
 // cpuGeneration represents a specific CPU microarchitecture generation
@@ -199,38 +206,50 @@ var memorySizes = []instanceSize{
 // memorySizesSmall is memorySizes without 12xlarge and 24xlarge (older/ARM families).
 var memorySizesSmall = slices.Clone(memorySizes[:6])
 
-// g4dnSizes are the single-GPU G4dn instance sizes (1x NVIDIA T4 each).
+// g4dnSizes are the G4dn instance sizes (NVIDIA T4). 12xlarge carries 4 GPUs,
+// every other size one. g4dn.metal is bare metal and is not offered.
 var g4dnSizes = []instanceSize{
 	{"xlarge", 4, 16},
 	{"2xlarge", 8, 32},
 	{"4xlarge", 16, 64},
 	{"8xlarge", 32, 128},
+	{"12xlarge", 48, 192},
 	{"16xlarge", 64, 256},
 }
 
-// g4adSizes are the single-GPU G4ad instance sizes (1x AMD Radeon Pro V520 each).
+// g4adSizes are the G4ad instance sizes (AMD Radeon Pro V520). 8xlarge carries
+// 2 GPUs and 16xlarge 4.
 var g4adSizes = []instanceSize{
 	{"xlarge", 4, 16},
 	{"2xlarge", 8, 32},
 	{"4xlarge", 16, 64},
+	{"8xlarge", 32, 128},
+	{"16xlarge", 64, 256},
 }
 
-// g5Sizes are the single-GPU G5 instance sizes (1x NVIDIA A10G each).
+// g5Sizes are the G5 instance sizes (NVIDIA A10G). 12xlarge and 24xlarge carry
+// 4 GPUs and 48xlarge 8; every other size one.
 var g5Sizes = []instanceSize{
 	{"xlarge", 4, 16},
 	{"2xlarge", 8, 32},
 	{"4xlarge", 16, 64},
 	{"8xlarge", 32, 128},
+	{"12xlarge", 48, 192},
 	{"16xlarge", 64, 256},
+	{"24xlarge", 96, 384},
+	{"48xlarge", 192, 768},
 }
 
-// g6Sizes are the single-GPU G6 instance sizes (1x NVIDIA L4 each).
+// g6Sizes are the G6 instance sizes (NVIDIA L4), which mirror g5 exactly.
 var g6Sizes = []instanceSize{
 	{"xlarge", 4, 16},
 	{"2xlarge", 8, 32},
 	{"4xlarge", 16, 64},
 	{"8xlarge", 32, 128},
+	{"12xlarge", 48, 192},
 	{"16xlarge", 64, 256},
+	{"24xlarge", 96, 384},
+	{"48xlarge", 192, 768},
 }
 
 // gr6Sizes are the single-GPU Gr6 instance sizes (1x NVIDIA L4, memory-optimized).
@@ -248,11 +267,70 @@ var g7eSizes = []instanceSize{
 	{"12xlarge", 48, 512},
 }
 
-// gpuCountPerType overrides the default of 1 GPU for multi-GPU instance sizes.
-var gpuCountPerType = map[string]int{
-	"g7e.12xlarge": 2,
+// builtinGPUFamily is the one non-AWS GPU family Spinifex defines, for the host
+// shapes AWS does not sell. A bare "gpu" is not an EC2 family name, so the
+// prefix is reserved and cannot collide with a real type.
+const builtinGPUFamily = "gpu"
+const builtinGPUPrefix = builtinGPUFamily + "."
+
+const MaxGPUsPerInstance = 8
+
+// builtinGPUTiers are the vCPU-per-GPU ratios the gpu.* family offers.
+var builtinGPUTiers = []int{2, 4, 8}
+
+// builtinGPUMemPerVCPUGB is AWS's general-purpose vCPU:memory ratio, applied to
+// every tier so no gpu.* shape is one EC2 would find unfamiliar.
+const builtinGPUMemPerVCPUGB = 4
+
+func IsBuiltinGPUType(instanceType string) bool {
+	return strings.HasPrefix(instanceType, builtinGPUPrefix)
 }
 
+// builtinGPUSizes and builtinGPUCounts describe the gpu.* family: the name
+// gpu.<count>x<vcpu>c is that many GPUs at that many vCPU each.
+var builtinGPUSizes, builtinGPUCounts = func() ([]instanceSize, map[string]int) {
+	sizes := make([]instanceSize, 0, MaxGPUsPerInstance*len(builtinGPUTiers))
+	counts := make(map[string]int, MaxGPUsPerInstance*len(builtinGPUTiers))
+	for _, perGPU := range builtinGPUTiers {
+		for n := 1; n <= MaxGPUsPerInstance; n++ {
+			vcpus := n * perGPU
+			suffix := fmt.Sprintf("%dx%dc", n, perGPU)
+			sizes = append(sizes, instanceSize{suffix, vcpus, float64(vcpus * builtinGPUMemPerVCPUGB)})
+			counts[builtinGPUPrefix+suffix] = n
+		}
+	}
+	return sizes, counts
+}()
+
+// gpuCountPerType overrides the default of 1 GPU for multi-GPU instance sizes.
+// Every figure for an AWS family is that family's real EC2 GPU count.
+var gpuCountPerType = func() map[string]int {
+	m := map[string]int{
+		"g4dn.12xlarge": 4,
+		"g4ad.8xlarge":  2,
+		"g4ad.16xlarge": 4,
+		"g5.12xlarge":   4,
+		"g5.24xlarge":   4,
+		"g5.48xlarge":   8,
+		"g6.12xlarge":   4,
+		"g6.24xlarge":   4,
+		"g6.48xlarge":   8,
+		"g6e.12xlarge":  4,
+		"g6e.24xlarge":  4,
+		"g6e.48xlarge":  8,
+		"g7e.12xlarge":  2,
+		"p3.8xlarge":    4,
+		"p3.16xlarge":   8,
+		"p3dn.24xlarge": 8,
+		"p4d.24xlarge":  8,
+		"p4de.24xlarge": 8,
+		"p5.48xlarge":   8,
+		"p5e.48xlarge":  8,
+	}
+	maps.Copy(m, builtinGPUCounts)
+	return m
+}()
+
 // GPUCountForType returns the number of GPUs required by the given instance type.
 // Returns 1 for all single-GPU and non-GPU types.
 func GPUCountForType(instanceType string) int {
@@ -262,49 +340,63 @@ func GPUCountForType(instanceType string) int {
 	return 1
 }
 
-// g6eSizes are the single-GPU G6e instance sizes (1x NVIDIA L40S each).
+// g6eSizes are the G6e instance sizes (NVIDIA L40S). 12xlarge and 24xlarge
+// carry 4 GPUs and 48xlarge 8; every other size one.
 var g6eSizes = []instanceSize{
 	{"xlarge", 4, 32},
 	{"2xlarge", 8, 64},
 	{"4xlarge", 16, 128},
 	{"8xlarge", 32, 256},
+	{"12xlarge", 48, 384},
 	{"16xlarge", 64, 512},
+	{"24xlarge", 96, 768},
+	{"48xlarge", 192, 1536},
 }
 
-// p3Sizes are the single-GPU P3 instance sizes (1x NVIDIA V100 16 GiB each).
-// P3 8xlarge and larger have multiple GPUs and are excluded.
+// p3Sizes are the P3 instance sizes (NVIDIA V100 SXM2 16 GiB). 8xlarge carries
+// 4 GPUs and 16xlarge 8.
 var p3Sizes = []instanceSize{
 	{"2xlarge", 8, 61},
+	{"8xlarge", 32, 244},
+	{"16xlarge", 64, 488},
 }
 
-// p3dnSizes are the single-GPU P3dn instance sizes (1x NVIDIA V100 32 GiB each).
-// P3dn 24xlarge has 8 GPUs and is excluded.
+// p3dnSizes are the P3dn instance sizes (NVIDIA V100 SXM2 32 GiB). 24xlarge is
+// the only size AWS sells and carries 8 GPUs; 2xlarge is a Spinifex single-GPU
+// size kept for hosts that cannot meet the 24xlarge shape.
 var p3dnSizes = []instanceSize{
 	{"2xlarge", 8, 61},
+	{"24xlarge", 96, 768},
 }
 
-// p4dSizes are the single-GPU P4d instance sizes (1x NVIDIA A100 SXM4 40 GiB each).
-// P4d 24xlarge has 8 GPUs and is excluded.
+// p4dSizes are the P4d instance sizes (NVIDIA A100 SXM4 40 GiB). 24xlarge is
+// the only size AWS sells and carries 8 GPUs; xlarge is a Spinifex single-GPU
+// size kept for hosts that cannot meet the 24xlarge shape.
 var p4dSizes = []instanceSize{
 	{"xlarge", 4, 32},
+	{"24xlarge", 96, 1152},
 }
 
-// p4deSizes are the single-GPU P4de instance sizes (1x NVIDIA A100 SXM4 80 GiB each).
-// P4de 24xlarge has 8 GPUs and is excluded.
+// p4deSizes are the P4de instance sizes (NVIDIA A100 SXM4 80 GiB), shaped
+// exactly as p4d; xlarge is the Spinifex single-GPU size.
 var p4deSizes = []instanceSize{
 	{"xlarge", 4, 32},
+	{"24xlarge", 96, 1152},
 }
 
-// p5Sizes are the single-GPU P5 instance sizes (1x NVIDIA H100 SXM5 80 GiB each).
-// P5 48xlarge has 8 GPUs and is excluded.
+// p5Sizes are the P5 instance sizes (NVIDIA H100 SXM5 80 GiB). 48xlarge is the
+// only size AWS sells and carries 8 GPUs; 4xlarge is the Spinifex single-GPU
+// size.
 var p5Sizes = []instanceSize{
 	{"4xlarge", 16, 256},
+	{"48xlarge", 192, 2048},
 }
 
-// p5eSizes are the single-GPU P5e instance sizes (1x NVIDIA H200 SXM5 141 GiB each).
-// P5e 48xlarge has 8 GPUs and is excluded.
+// p5eSizes are the P5e instance sizes (NVIDIA H200 SXM5 141 GiB), shaped
+// exactly as p5; 4xlarge is the Spinifex single-GPU size.
 var p5eSizes = []instanceSize{
 	{"4xlarge", 16, 256},
+	{"48xlarge", 192, 2048},
 }
 
 // systemSizes defines internal-only instance types for system VMs (LB, NAT GW, etc.).
@@ -370,6 +462,10 @@ var instanceFamilyDefs = []instanceFamilyDef{
 	{name: "p5", sizes: p5Sizes, currentGen: true},
 	{name: "p5e", sizes: p5eSizes, currentGen: true},
 
+	// Non-AWS GPU family for host shapes AWS does not sell; emitted by
+	// GenerateBuiltinGPUTypes rather than by matching a model's family.
+	{name: builtinGPUFamily, sizes: builtinGPUSizes, currentGen: true},
+
 	// Memory Optimized (1:8 vCPU:memory)
 	{name: "r4", sizes: memorySizesSmall, currentGen: false},
 	{name: "r5", sizes: memorySizes, currentGen: true},

spinifex/instancetypes/generate.go

@@ -158,9 +158,22 @@ func generateSystemTypes(arch string) map[string]*ec2.InstanceTypeInfo {
 	return types
 }
 
-// GenerateGPUTypes returns InstanceTypeInfo entries for each GPU model with GpuInfo populated.
+// GenerateGPUTypes returns InstanceTypeInfo entries for each GPU model plus the
+// built-in gpu.* family. models carries one entry per whole GPU, so a node
+// resolving more than one distinct model offers no type above a count of 1.
 func GenerateGPUTypes(models []GPUModel, arch string) map[string]*ec2.InstanceTypeInfo {
 	types := make(map[string]*ec2.InstanceTypeInfo)
+	if len(models) == 0 {
+		return types
+	}
+
+	maxGPUsPerInstance := MaxGPUsPerInstance
+	if !gpuModelsHomogeneous(models) {
+		maxGPUsPerInstance = 1
+		slog.Warn("node holds more than one distinct GPU model, offering single-GPU types only",
+			"models", len(models))
+	}
+
 	seen := make(map[string]bool)
 
 	for _, model := range models {
@@ -175,48 +188,103 @@ func GenerateGPUTypes(models []GPUModel, arch string) map[string]*ec2.InstanceTy
 			}
 			for _, size := range def.sizes {
 				name := fmt.Sprintf("%s.%s", def.name, size.suffix)
-				gpuCount := int64(GPUCountForType(name))
-				types[name] = &ec2.InstanceTypeInfo{
-					InstanceType: aws.String(name),
-					VCpuInfo: &ec2.VCpuInfo{
-						DefaultVCpus: aws.Int64(int64(size.vcpus)),
-					},
-					MemoryInfo: &ec2.MemoryInfo{
-						SizeInMiB: aws.Int64(int64(size.memoryGB * 1024)),
-					},
-					ProcessorInfo: &ec2.ProcessorInfo{
-						SupportedArchitectures: []*string{aws.String(arch)},
-					},
-					GpuInfo: &ec2.GpuInfo{
-						Gpus: []*ec2.GpuDeviceInfo{{
-							Count:        aws.Int64(gpuCount),
-							Manufacturer: aws.String(model.Manufacturer),
-							Name:         aws.String(model.Name),
-							MemoryInfo: &ec2.GpuDeviceMemoryInfo{
-								SizeInMiB: aws.Int64(model.MemoryMiB),
-							},
-						}},
-						TotalGpuMemoryInMiB: aws.Int64(model.MemoryMiB * gpuCount),
-					},
-					CurrentGeneration:             aws.Bool(def.currentGen),
-					BurstablePerformanceSupported: aws.Bool(false),
-					Hypervisor:                    aws.String("kvm"),
-					SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
-					SupportedRootDeviceTypes:      []*string{aws.String("ebs")},
-					NetworkInfo:                   NetworkInfoForType(name),
-					PlacementGroupInfo: &ec2.PlacementGroupInfo{
-						SupportedStrategies: []*string{
-							aws.String("cluster"),
-							aws.String("spread"),
-						},
-					},
+				if !gpuTypeOffered(name, maxGPUsPerInstance) {
+					continue
 				}
+				types[name] = gpuInstanceTypeInfo(name, size, model, arch, def.currentGen)
 			}
 		}
 	}
+
+	// gpu.* is no model's family, so GenerateGPUTypes cannot reach it by the
+	// match above. It advertises whichever model the node resolved.
+	maps.Copy(types, generateBuiltinGPUTypes(models[0], arch, maxGPUsPerInstance))
 	return types
 }
 
+// gpuModelsHomogeneous reports whether every GPU advertises the same name and
+// VRAM. Those reach GpuInfo, so devices advertising identically are
+// interchangeable — which spares a mixed-SKU H100 node differing only by PCI ID.
+func gpuModelsHomogeneous(models []GPUModel) bool {
+	for _, m := range models[1:] {
+		if m.Name != models[0].Name || m.MemoryMiB != models[0].MemoryMiB {
+			return false
+		}
+	}
+	return true
+}
+
+// gpuTypeOffered reports whether a node handing at most maxGPUsPerInstance GPUs
+// to one instance can offer this type, logging by name when it cannot. An
+// operator who sees a type silently absent has no thread to pull.
+func gpuTypeOffered(name string, maxGPUsPerInstance int) bool {
+	required := GPUCountForType(name)
+	if required <= maxGPUsPerInstance {
+		return true
+	}
+	slog.Info("GPU instance type not offered",
+		"instanceType", name, "gpus_required", required,
+		"gpus_offerable", maxGPUsPerInstance)
+	return false
+}
+
+// generateBuiltinGPUTypes emits the gpu.* family for the node's resolved GPU
+// model. Every figure but the model itself is static, so the family is
+// identical on every node that discovers the same hardware.
+func generateBuiltinGPUTypes(model GPUModel, arch string, maxGPUsPerInstance int) map[string]*ec2.InstanceTypeInfo {
+	types := make(map[string]*ec2.InstanceTypeInfo, len(builtinGPUSizes))
+	for _, size := range builtinGPUSizes {
+		name := builtinGPUPrefix + size.suffix
+		if !gpuTypeOffered(name, maxGPUsPerInstance) {
+			continue
+		}
+		types[name] = gpuInstanceTypeInfo(name, size, model, arch, true)
+	}
+	return types
+}
+
+// gpuInstanceTypeInfo builds one GPU InstanceTypeInfo. The vCPU and memory
+// figures come from the size, the GPU count from the type name, and the model,
+// VRAM and manufacturer from discovery.
+func gpuInstanceTypeInfo(name string, size instanceSize, model GPUModel, arch string, currentGen bool) *ec2.InstanceTypeInfo {
+	gpuCount := int64(GPUCountForType(name))
+	return &ec2.InstanceTypeInfo{
+		InstanceType: aws.String(name),
+		VCpuInfo: &ec2.VCpuInfo{
+			DefaultVCpus: aws.Int64(int64(size.vcpus)),
+		},
+		MemoryInfo: &ec2.MemoryInfo{
+			SizeInMiB: aws.Int64(int64(size.memoryGB * 1024)),
+		},
+		ProcessorInfo: &ec2.ProcessorInfo{
+			SupportedArchitectures: []*string{aws.String(arch)},
+		},
+		GpuInfo: &ec2.GpuInfo{
+			Gpus: []*ec2.GpuDeviceInfo{{
+				Count:        aws.Int64(gpuCount),
+				Manufacturer: aws.String(model.Manufacturer),
+				Name:         aws.String(model.Name),
+				MemoryInfo: &ec2.GpuDeviceMemoryInfo{
+					SizeInMiB: aws.Int64(model.MemoryMiB),
+				},
+			}},
+			TotalGpuMemoryInMiB: aws.Int64(model.MemoryMiB * gpuCount),
+		},
+		CurrentGeneration:             aws.Bool(currentGen),
+		BurstablePerformanceSupported: aws.Bool(false),
+		Hypervisor:                    aws.String("kvm"),
+		SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
+		SupportedRootDeviceTypes:      []*string{aws.String("ebs")},
+		NetworkInfo:                   NetworkInfoForType(name),
+		PlacementGroupInfo: &ec2.PlacementGroupInfo{
+			SupportedStrategies: []*string{
+				aws.String("cluster"),
+				aws.String("spread"),
+			},
+		},
+	}
+}
+
 // IsGPUType returns true if the instance type has GPU resources.
 func IsGPUType(info *ec2.InstanceTypeInfo) bool {
 	return info.GpuInfo != nil && len(info.GpuInfo.Gpus) > 0

spinifex/instancetypes/generate_test.go

@@ -461,9 +461,10 @@ func TestGPUModelForVendorDevice_Unknown(t *testing.T) {
 func TestGenerateGPUTypes_NVIDIAa10g(t *testing.T) {
 	types := GenerateGPUTypes([]GPUModel{NVIDIAa10g}, "x86_64")
 
-	// g5 has 5 single-GPU sizes: xlarge, 2xlarge, 4xlarge, 8xlarge, 16xlarge
-	assert.Len(t, types, 5)
-	for _, name := range []string{"g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge", "g5.16xlarge"} {
+	// Every g5 size, plus the built-in gpu.* family.
+	assert.Len(t, types, len(g5Sizes)+len(builtinGPUSizes))
+	for _, name := range []string{"g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge",
+		"g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge"} {
 		assert.True(t, hasFamily(types, name), "expected %s", name)
 	}
 
@@ -485,7 +486,8 @@ func TestGenerateGPUTypes_NVIDIAa10g(t *testing.T) {
 func TestGenerateGPUTypes_DeduplicatesSameFamily(t *testing.T) {
 	// Two GPUs of the same model (same family) should produce only one set of types.
 	types := GenerateGPUTypes([]GPUModel{NVIDIAa10g, NVIDIAa10g}, "x86_64")
-	assert.Len(t, types, 5, "duplicate GPU model should not double the type count")
+	assert.Len(t, types, len(g5Sizes)+len(builtinGPUSizes),
+		"duplicate GPU model should not double the type count")
 }
 
 func TestGenerateGPUTypes_EmptyModels(t *testing.T) {

spinifex/instancetypes/gpu_type_name_test.go

@@ -15,6 +15,9 @@ func TestGPUVendorForType(t *testing.T) {
 		{"p4d.xlarge", "nvidia"},
 		{"m5.large", ""},
 		{"", ""},
+		// Both name a shape, not a family; the vendor comes from discovery.
+		{"gpu.8x4c", ""},
+		{"mig.1g.10gb", ""},
 	}
 	for _, tc := range tests {
 		t.Run(tc.instanceType, func(t *testing.T) {
@@ -23,6 +26,9 @@ func TestGPUVendorForType(t *testing.T) {
 	}
 }
 
+// A false answer routes ECS and EKS to the non-GPU AMI, which boots the node
+// with its GPUs passed through and no driver. The vendor being unknowable from
+// the name is not a reason to call these types CPU-only.
 func TestIsGPUTypeName(t *testing.T) {
 	tests := []struct {
 		instanceType string
@@ -32,6 +38,9 @@ func TestIsGPUTypeName(t *testing.T) {
 		{"p4d.xlarge", true},
 		{"m5.large", false},
 		{"", false},
+		{"gpu.1x2c", true},
+		{"gpu.8x8c", true},
+		{"mig.1g.10gb", true},
 	}
 	for _, tc := range tests {
 		t.Run(tc.instanceType, func(t *testing.T) {

spinifex/instancetypes/multi_gpu_test.go

@@ -0,0 +1,279 @@
+//test:in-package — pins instanceFamilyDefs and gpuCountPerType against each
+package instancetypes
+
+import (
+	"fmt"
+	"slices"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gpuFamilies are every family in instanceFamilyDefs that GenerateGPUTypes can
+// emit. Listed here rather than derived so a new GPU family has to be added
+// deliberately, with its counts.
+var gpuFamilies = []string{
+	"g4dn", "g4ad", "g5", "g6", "gr6", "g6e", "g7e",
+	"p3", "p3dn", "p4d", "p4de", "p5", "p5e",
+	builtinGPUFamily,
+}
+
+// wantGPUCounts pins the GPU count of every size of every GPU family. A size
+// added without a count, or a count without a size, fails here — otherwise a
+// wrong figure surfaces only when a customer sizes a workload from it.
+var wantGPUCounts = map[string]int{
+	"g4dn.xlarge":   1,
+	"g4dn.2xlarge":  1,
+	"g4dn.4xlarge":  1,
+	"g4dn.8xlarge":  1,
+	"g4dn.12xlarge": 4,
+	"g4dn.16xlarge": 1,
+
+	"g4ad.xlarge":   1,
+	"g4ad.2xlarge":  1,
+	"g4ad.4xlarge":  1,
+	"g4ad.8xlarge":  2,
+	"g4ad.16xlarge": 4,
+
+	"g5.xlarge":   1,
+	"g5.2xlarge":  1,
+	"g5.4xlarge":  1,
+	"g5.8xlarge":  1,
+	"g5.12xlarge": 4,
+	"g5.16xlarge": 1,
+	"g5.24xlarge": 4,
+	"g5.48xlarge": 8,
+
+	"g6.xlarge":   1,
+	"g6.2xlarge":  1,
+	"g6.4xlarge":  1,
+	"g6.8xlarge":  1,
+	"g6.12xlarge": 4,
+	"g6.16xlarge": 1,
+	"g6.24xlarge": 4,
+	"g6.48xlarge": 8,
+
+	"gr6.4xlarge": 1,
+	"gr6.8xlarge": 1,
+
+	"g6e.xlarge":   1,
+	"g6e.2xlarge":  1,
+	"g6e.4xlarge":  1,
+	"g6e.8xlarge":  1,
+	"g6e.12xlarge": 4,
+	"g6e.16xlarge": 1,
+	"g6e.24xlarge": 4,
+	"g6e.48xlarge": 8,
+
+	// g7e is a Spinifex family, grandfathered and not extended.
+	"g7e.2xlarge":  1,
+	"g7e.4xlarge":  1,
+	"g7e.8xlarge":  1,
+	"g7e.12xlarge": 2,
+
+	"p3.2xlarge":  1,
+	"p3.8xlarge":  4,
+	"p3.16xlarge": 8,
+
+	"p3dn.2xlarge":  1, // Spinifex single-GPU size
+	"p3dn.24xlarge": 8,
+
+	"p4d.xlarge":   1, // Spinifex single-GPU size
+	"p4d.24xlarge": 8,
+
+	"p4de.xlarge":   1, // Spinifex single-GPU size
+	"p4de.24xlarge": 8,
+
+	"p5.4xlarge":  1, // Spinifex single-GPU size
+	"p5.48xlarge": 8,
+
+	"p5e.4xlarge":  1, // Spinifex single-GPU size
+	"p5e.48xlarge": 8,
+}
+
+func init() {
+	// The gpu.* family is generated, so its counts are pinned by the shape
+	// assertions in TestBuiltinGPUFamily rather than repeated by hand.
+	for n := 1; n <= MaxGPUsPerInstance; n++ {
+		for _, perGPU := range builtinGPUTiers {
+			wantGPUCounts[fmt.Sprintf("%s%dx%dc", builtinGPUPrefix, n, perGPU)] = n
+		}
+	}
+}
+
+func TestGPUCountForType_PinsEveryGPUSize(t *testing.T) {
+	defined := make(map[string]bool, len(wantGPUCounts))
+
+	for _, def := range instanceFamilyDefs {
+		if !slices.Contains(gpuFamilies, def.name) {
+			continue
+		}
+		for _, size := range def.sizes {
+			name := def.name + "." + size.suffix
+			defined[name] = true
+			want, pinned := wantGPUCounts[name]
+			if !assert.True(t, pinned, "%s has no pinned GPU count", name) {
+				continue
+			}
+			assert.Equal(t, want, GPUCountForType(name), "GPU count for %s", name)
+		}
+	}
+
+	for name := range wantGPUCounts {
+		assert.True(t, defined[name], "%s is pinned but no longer defined — removing a type is breaking", name)
+	}
+}
+
+// Every name in gpuCountPerType must be a type that exists. An entry for a size
+// that was never defined silently does nothing.
+func TestGPUCountPerType_HasNoOrphanEntries(t *testing.T) {
+	for name := range gpuCountPerType {
+		_, ok := DefaultVCPUs(name)
+		assert.True(t, ok, "gpuCountPerType names %s, which is not a defined instance type", name)
+	}
+}
+
+func TestBuiltinGPUFamily(t *testing.T) {
+	assert.Len(t, builtinGPUSizes, MaxGPUsPerInstance*len(builtinGPUTiers))
+
+	for n := 1; n <= MaxGPUsPerInstance; n++ {
+		for _, perGPU := range builtinGPUTiers {
+			name := fmt.Sprintf("%s%dx%dc", builtinGPUPrefix, n, perGPU)
+			t.Run(name, func(t *testing.T) {
+				wantVCPUs := n * perGPU
+
+				vcpus, ok := DefaultVCPUs(name)
+				require.True(t, ok, "%s must be a defined type", name)
+				assert.Equal(t, wantVCPUs, vcpus)
+
+				memMiB, ok := DefaultMemoryMiB(name)
+				require.True(t, ok)
+				assert.Equal(t, int64(wantVCPUs*builtinGPUMemPerVCPUGB*1024), memMiB,
+					"every tier carries 4 GiB per vCPU")
+
+				assert.Equal(t, n, GPUCountForType(name))
+				assert.True(t, IsBuiltinGPUType(name))
+			})
+		}
+	}
+}
+
+// The prefix is reserved and self-announcing, following sys. and mig.
+func TestIsBuiltinGPUType(t *testing.T) {
+	assert.True(t, IsBuiltinGPUType("gpu.8x4c"))
+	assert.True(t, IsBuiltinGPUType("gpu.1x2c"))
+	assert.False(t, IsBuiltinGPUType("g5.xlarge"))
+	assert.False(t, IsBuiltinGPUType("gpu"))
+	assert.False(t, IsBuiltinGPUType(""))
+}
+
+// gpu.* and mig.* name a shape and a profile, so neither carries a vendor.
+func TestGPUVendorForType_BuiltinAndMIGCarryNoVendor(t *testing.T) {
+	assert.Empty(t, GPUVendorForType("gpu.8x4c"))
+	assert.Empty(t, GPUVendorForType("mig.1g.10gb"))
+	assert.Equal(t, "nvidia", GPUVendorForType("g5.xlarge"))
+}
+
+// A node's gpu.* types advertise whatever GPU it discovered, whichever AWS
+// family that model maps to.
+func TestGenerateGPUTypes_BuiltinFamilyReportsTheDiscoveredModel(t *testing.T) {
+	rtx3090 := GPUModel{
+		VendorID: "10de", DeviceID: "2204", Family: "g5",
+		Manufacturer: "NVIDIA", Name: "GeForce RTX 3090", MemoryMiB: 24576,
+	}
+	models := make([]GPUModel, 8)
+	for i := range models {
+		models[i] = rtx3090
+	}
+
+	types := GenerateGPUTypes(models, "x86_64")
+
+	it, ok := types["gpu.8x4c"]
+	require.True(t, ok, "a homogeneous 8-GPU node must offer gpu.8x4c")
+	assert.Equal(t, int64(32), *it.VCpuInfo.DefaultVCpus)
+	assert.Equal(t, int64(128*1024), *it.MemoryInfo.SizeInMiB)
+	require.Len(t, it.GpuInfo.Gpus, 1)
+	assert.Equal(t, int64(8), *it.GpuInfo.Gpus[0].Count)
+	assert.Equal(t, "GeForce RTX 3090", *it.GpuInfo.Gpus[0].Name)
+	assert.Equal(t, "NVIDIA", *it.GpuInfo.Gpus[0].Manufacturer)
+	assert.Equal(t, int64(24576), *it.GpuInfo.Gpus[0].MemoryInfo.SizeInMiB)
+	assert.Equal(t, int64(8*24576), *it.GpuInfo.TotalGpuMemoryInMiB)
+
+	// All three tiers, counts 1 through 8, with no gaps.
+	for n := 1; n <= MaxGPUsPerInstance; n++ {
+		for _, perGPU := range builtinGPUTiers {
+			name := fmt.Sprintf("%s%dx%dc", builtinGPUPrefix, n, perGPU)
+			assert.Contains(t, types, name)
+		}
+	}
+}
+
+// Two devices advertising the same name and VRAM are interchangeable, so a
+// mixed-SKU node of one model (H100 SXM and PCIe differ only by PCI ID) still
+// offers multi-GPU types.
+func TestGenerateGPUTypes_SameNameAndVRAMIsHomogeneous(t *testing.T) {
+	types := GenerateGPUTypes([]GPUModel{NVIDIAh100sxm, NVIDIAh100pcie}, "x86_64")
+	assert.Contains(t, types, "gpu.2x4c")
+	assert.Contains(t, types, "p5.48xlarge")
+}
+
+// Handing one instance a mixture of models tends to make NCCL hang rather than
+// fail, so a node that resolved more than one offers single-GPU types only.
+func TestGenerateGPUTypes_MixedModelsOfferSingleGPUTypesOnly(t *testing.T) {
+	types := GenerateGPUTypes([]GPUModel{NVIDIAa10g, NVIDIAl40s}, "x86_64")
+
+	require.NotEmpty(t, types)
+	for name, it := range types {
+		require.Len(t, it.GpuInfo.Gpus, 1)
+		assert.Equal(t, int64(1), *it.GpuInfo.Gpus[0].Count,
+			"%s must not be offered on a node with more than one GPU model", name)
+		assert.Equal(t, 1, GPUCountForType(name))
+	}
+
+	// The single-GPU sizes of both families survive, and so does gpu.1x*.
+	assert.Contains(t, types, "g5.xlarge")
+	assert.Contains(t, types, "g6e.xlarge")
+	assert.Contains(t, types, "gpu.1x4c")
+	assert.NotContains(t, types, "gpu.2x4c")
+	assert.NotContains(t, types, "g5.48xlarge")
+}
+
+// MIG types stay at count 1: EC2 has no multi-slice type, and an accidental
+// capability is awkward to withdraw once someone depends on it.
+func TestGenerateMIGTypes_CountIsAlwaysOne(t *testing.T) {
+	profiles := []MIGProfileSpec{
+		{Name: "1g.10gb", MemoryMiB: 10240},
+		{Name: "3g.40gb", MemoryMiB: 40960},
+		{Name: "7g.80gb", MemoryMiB: 81920},
+	}
+	types := GenerateMIGTypes(profiles, "x86_64")
+	require.Len(t, types, len(profiles))
+
+	for name, it := range types {
+		require.Len(t, it.GpuInfo.Gpus, 1)
+		assert.Equal(t, int64(1), *it.GpuInfo.Gpus[0].Count, "%s", name)
+		assert.Equal(t, 1, GPUCountForType(name), "%s", name)
+		assert.Equal(t, *it.GpuInfo.Gpus[0].MemoryInfo.SizeInMiB, *it.GpuInfo.TotalGpuMemoryInMiB,
+			"%s: one slice is the whole allocation", name)
+	}
+}
+
+// A multi-GPU AWS size must report its real GPU count and the aggregate VRAM
+// that follows from it.
+func TestGenerateGPUTypes_MultiGPUAWSSizes(t *testing.T) {
+	models := make([]GPUModel, 8)
+	for i := range models {
+		models[i] = NVIDIAa10g
+	}
+	types := GenerateGPUTypes(models, "x86_64")
+
+	it, ok := types["g5.48xlarge"]
+	require.True(t, ok)
+	assert.Equal(t, int64(192), *it.VCpuInfo.DefaultVCpus)
+	assert.Equal(t, int64(768*1024), *it.MemoryInfo.SizeInMiB)
+	assert.Equal(t, int64(8), *it.GpuInfo.Gpus[0].Count)
+	assert.Equal(t, int64(8*24576), *it.GpuInfo.TotalGpuMemoryInMiB)
+	assert.Equal(t, "g5.48xlarge", aws.StringValue(it.InstanceType))
+}

spinifex/services/spinifexui/frontend/dist/assets/_id-U4BZhcdn.js

@@ -1,2 +1,2 @@
-const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_id-x-YmyORB.js","assets/compiler-runtime-D_vZfn79.js","assets/useQuery-2EKf7AyV.js","assets/useBaseQuery-Dqg0ViLa.js","assets/removable-Dwr9otKB.js","assets/query-BPr_oSEg.js","assets/useSuspenseQuery-Bc7E1P-f.js","assets/link-Dhy4xOXd.js","assets/useSelector-CRh8wCTv.js","assets/with-selector-DrM041uv.js","assets/cluster-config-xw_Q7X1e.js","assets/hex-encoding-B19xAT_7.js","assets/createLucideIcon-BQgzNCBR.js","assets/back-link-CjKy1JE5.js","assets/select-B01jpDaX.js","assets/useValueAsRef-F56m8WbP.js","assets/dist-BKgR2qk6.js","assets/button-BarTr35m.js","assets/createBaseUIEventDetails-DL-8egB9.js","assets/check-BO7W8OSs.js","assets/chevron-down-Dt4q6d24.js","assets/useValueChanged-B2TUO8IO.js","assets/useScrollLock-NX6x9Pv8.js","assets/usePositioner-DUsvphCt.js","assets/DirectionContext-C9rdm8xn.js","assets/useControlled-DwxdYHYK.js","assets/useCompositeListItem-CEsaEeXb.js","assets/rotate-cw-FTUcK_f-.js","assets/trash-DVab4pEz.js","assets/input-6kdUb_DH.js","assets/admin-context-CXmiyIcc.js","assets/sts-B5ZEON3v.js","assets/signed-fetch-B-FFW93q.js","assets/ec2-CWEplLq4.js","assets/queryOptions-Dfvzj6n2.js","assets/awsClient-b0hcP1Iz.js","assets/index.browser-C3WbWNjZ.js","assets/ec2-Ctto2xjU.js","assets/useMutation-6rdxVQpi.js","assets/mutation-BziiWz5u.js","assets/subnet-calculator-BlCTyGtY.js","assets/iam-CVg27y22.js","assets/commandBuilder-CoLHZwsC.js","assets/json-DsgWHaZY.js","assets/page-heading-vksh_67w.js","assets/admin-BUkrFIlI.js","assets/error-banner-DeOeVBKS.js","assets/field-c7GHI5st.js","assets/separator-B3CHPrlt.js","assets/textarea-n04EavUm.js","assets/alert-dialog-kCQ1W4Rz.js","assets/DialogTitle-DiInIGjk.js","assets/delete-confirmation-dialog-SdDN4LiF.js","assets/detail-row-DNBRlVeK.js","assets/state-badge-D1Z6q_OA.js","assets/ami-details-ChpQ4w1b.js","assets/gpu-instance-type-select-BD2r8UcO.js"])))=>i.map(i=>d[i]);
-import{n as e,t}from"./lazyRouteComponent-CiZDS8hZ.js";import{lt as n}from"./cluster-config-xw_Q7X1e.js";import{a as r,c as i,s as a}from"./ec2-CWEplLq4.js";var o=e(`/_auth/ec2/(instances)/describe-instances/$id`)({loader:async({context:e,params:t})=>{let[n]=await Promise.all([e.queryClient.query({...a(t.id),staleTime:`static`}),e.queryClient.query({...i,staleTime:`static`})]),o=n.Reservations?.[0]?.Instances?.[0]?.ImageId;return await e.queryClient.query({...r(o),staleTime:`static`}),n},head:({loaderData:e})=>({meta:[{title:`${e?.Reservations?.[0]?.Instances?.[0]?.InstanceId??`Instance`} | EC2 | Mulga`}]}),component:t(()=>n(()=>import(`./_id-x-YmyORB.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{o as t};
+const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_id-BnUjyn0E.js","assets/compiler-runtime-D_vZfn79.js","assets/useQuery-2EKf7AyV.js","assets/useBaseQuery-Dqg0ViLa.js","assets/removable-Dwr9otKB.js","assets/query-BPr_oSEg.js","assets/useSuspenseQuery-Bc7E1P-f.js","assets/link-Dhy4xOXd.js","assets/useSelector-CRh8wCTv.js","assets/with-selector-DrM041uv.js","assets/cluster-config-xw_Q7X1e.js","assets/hex-encoding-B19xAT_7.js","assets/createLucideIcon-BQgzNCBR.js","assets/back-link-CjKy1JE5.js","assets/select-B01jpDaX.js","assets/useValueAsRef-F56m8WbP.js","assets/dist-BKgR2qk6.js","assets/button-BarTr35m.js","assets/createBaseUIEventDetails-DL-8egB9.js","assets/check-BO7W8OSs.js","assets/chevron-down-Dt4q6d24.js","assets/useValueChanged-B2TUO8IO.js","assets/useScrollLock-NX6x9Pv8.js","assets/usePositioner-DUsvphCt.js","assets/DirectionContext-C9rdm8xn.js","assets/useControlled-DwxdYHYK.js","assets/useCompositeListItem-CEsaEeXb.js","assets/rotate-cw-FTUcK_f-.js","assets/trash-DVab4pEz.js","assets/input-6kdUb_DH.js","assets/admin-context-CXmiyIcc.js","assets/sts-B5ZEON3v.js","assets/signed-fetch-B-FFW93q.js","assets/ec2-CWEplLq4.js","assets/queryOptions-Dfvzj6n2.js","assets/awsClient-b0hcP1Iz.js","assets/index.browser-C3WbWNjZ.js","assets/ec2-Ctto2xjU.js","assets/useMutation-6rdxVQpi.js","assets/mutation-BziiWz5u.js","assets/subnet-calculator-BlCTyGtY.js","assets/iam-CVg27y22.js","assets/commandBuilder-CoLHZwsC.js","assets/json-DsgWHaZY.js","assets/page-heading-vksh_67w.js","assets/admin-BUkrFIlI.js","assets/error-banner-DeOeVBKS.js","assets/field-c7GHI5st.js","assets/separator-B3CHPrlt.js","assets/textarea-n04EavUm.js","assets/alert-dialog-kCQ1W4Rz.js","assets/DialogTitle-DiInIGjk.js","assets/delete-confirmation-dialog-SdDN4LiF.js","assets/detail-row-DNBRlVeK.js","assets/state-badge-D1Z6q_OA.js","assets/ami-details-ChpQ4w1b.js","assets/gpu-instance-type-select-BD2r8UcO.js"])))=>i.map(i=>d[i]);
+import{n as e,t}from"./lazyRouteComponent-CiZDS8hZ.js";import{lt as n}from"./cluster-config-xw_Q7X1e.js";import{a as r,c as i,s as a}from"./ec2-CWEplLq4.js";var o=e(`/_auth/ec2/(instances)/describe-instances/$id`)({loader:async({context:e,params:t})=>{let[n]=await Promise.all([e.queryClient.query({...a(t.id),staleTime:`static`}),e.queryClient.query({...i,staleTime:`static`})]),o=n.Reservations?.[0]?.Instances?.[0]?.ImageId;return await e.queryClient.query({...r(o),staleTime:`static`}),n},head:({loaderData:e})=>({meta:[{title:`${e?.Reservations?.[0]?.Instances?.[0]?.InstanceId??`Instance`} | EC2 | Mulga`}]}),component:t(()=>n(()=>import(`./_id-BnUjyn0E.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{o as t};

spinifex/services/spinifexui/frontend/dist/index.html

@@ -7,7 +7,7 @@
     <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
     <meta name="theme-color" content="#000000" />
     <meta name="description" content="Spinifex Control Panel" />
-    <script type="module" crossorigin src="/assets/index-CJ3qP1oV.js"></script>
+    <script type="module" crossorigin src="/assets/index-BrcYJ9RS.js"></script>
     <link rel="modulepreload" crossorigin href="/assets/compiler-runtime-D_vZfn79.js">
     <link rel="modulepreload" crossorigin href="/assets/removable-Dwr9otKB.js">
     <link rel="modulepreload" crossorigin href="/assets/query-BPr_oSEg.js">
@@ -74,7 +74,7 @@
     <link rel="modulepreload" crossorigin href="/assets/_knowledgeBaseId-7saNV50A.js">
     <link rel="modulepreload" crossorigin href="/assets/_id-FFfcJrCW.js">
     <link rel="modulepreload" crossorigin href="/assets/_id-Df06xbgH.js">
-    <link rel="modulepreload" crossorigin href="/assets/_id-DHGMf_p-.js">
+    <link rel="modulepreload" crossorigin href="/assets/_id-U4BZhcdn.js">
     <link rel="modulepreload" crossorigin href="/assets/_id-DpCiHWsF.js">
     <link rel="modulepreload" crossorigin href="/assets/_id-CXZivkIZ.js">
     <link rel="modulepreload" crossorigin href="/assets/_id-UPy9yakk.js">

spinifex/services/spinifexui/frontend/src/queries/admin.ts

@@ -73,7 +73,7 @@ interface VMInfo {
   node: string
   launch_time: number
   managed_by?: string
-  gpu?: VMGPUInfo
+  gpus?: VMGPUInfo[]
 }
 
 interface GetVMsOutput {

spinifex/services/spinifexui/frontend/src/routes/_auth/ec2/(instances)/describe-instances/$id.tsx

@@ -331,7 +331,7 @@ function InstanceDetail() {
         <IamRolePanel instanceId={instance.InstanceId} />
 
         {/* GPU */}
-        {isAdmin && <GpuDetailCard gpu={vmInfo?.gpu} />}
+        {isAdmin && <GpuDetailCard gpus={vmInfo?.gpus} />}
 
         {/* Block Device Mappings */}
         {instance.BlockDeviceMappings &&

spinifex/services/spinifexui/frontend/src/routes/_auth/ec2/-components/gpu-detail-card.test.tsx

@@ -6,8 +6,11 @@ import type { VMGPUInfo } from "@/queries/admin"
 import { GpuDetailCard } from "./gpu-detail-card"
 
 describe("GpuDetailCard", () => {
-  it("renders nothing when no gpu is attached", () => {
-    const { container } = render(<GpuDetailCard gpu={undefined} />)
+  it("renders nothing when no GPUs are attached", () => {
+    const { container, rerender } = render(<GpuDetailCard gpus={undefined} />)
+    expect(container).toBeEmptyDOMElement()
+
+    rerender(<GpuDetailCard gpus={[]} />)
     expect(container).toBeEmptyDOMElement()
   })
 
@@ -18,7 +21,7 @@ describe("GpuDetailCard", () => {
       profile: "1g.24gb",
       mdev_path: "/sys/bus/mdev/devices/abc-123",
     }
-    render(<GpuDetailCard gpu={gpu} />)
+    render(<GpuDetailCard gpus={[gpu]} />)
 
     expect(screen.getByText("GPU")).toBeInTheDocument()
     expect(screen.getByText("RTX Pro 6000 Blackwell SE")).toBeInTheDocument()
@@ -36,10 +39,33 @@ describe("GpuDetailCard", () => {
       vram_mib: 24_576,
       pci_address: "0000:01:00.0",
     }
-    render(<GpuDetailCard gpu={gpu} />)
+    render(<GpuDetailCard gpus={[gpu]} />)
 
     expect(screen.getByText("PCIe passthrough")).toBeInTheDocument()
     expect(screen.getByText("0000:01:00.0")).toBeInTheDocument()
     expect(screen.queryByText("Profile")).not.toBeInTheDocument()
   })
+
+  it("renders every attached GPU in attachment order", () => {
+    const gpus: VMGPUInfo[] = [
+      {
+        model: "RTX 3090",
+        vram_mib: 24_576,
+        pci_address: "0000:01:00.0",
+      },
+      {
+        model: "RTX 3090",
+        vram_mib: 24_576,
+        pci_address: "0000:02:00.0",
+      },
+    ]
+    render(<GpuDetailCard gpus={gpus} />)
+
+    expect(screen.getByText("GPUs (2)")).toBeInTheDocument()
+    expect(screen.getByText("GPU 1")).toBeInTheDocument()
+    expect(screen.getByText("GPU 2")).toBeInTheDocument()
+    expect(screen.getByText("0000:01:00.0")).toBeInTheDocument()
+    expect(screen.getByText("0000:02:00.0")).toBeInTheDocument()
+    expect(screen.getAllByText("RTX 3090")).toHaveLength(2)
+  })
 })

spinifex/services/spinifexui/frontend/src/routes/_auth/ec2/-components/gpu-detail-card.tsx

@@ -3,27 +3,43 @@ import { DetailRow } from "@/components/detail-row"
 import { formatVRAMMiB } from "@/lib/utils"
 import type { VMGPUInfo } from "@/queries/admin"
 
-export function GpuDetailCard({ gpu }: { gpu?: VMGPUInfo }) {
-  if (!gpu) {
+export function GpuDetailCard({ gpus }: { gpus?: VMGPUInfo[] }) {
+  if (!gpus?.length) {
     return null
   }
 
   return (
     <DetailCard>
-      <DetailCard.Header>GPU</DetailCard.Header>
-      <DetailCard.Content>
-        <DetailRow label="Model" value={gpu.model} />
-        <DetailRow label="VRAM" value={formatVRAMMiB(gpu.vram_mib)} />
-        <DetailRow
-          label="Attachment"
-          value={gpu.profile ? "MIG slice" : "PCIe passthrough"}
-        />
-        {gpu.profile && <DetailRow label="Profile" value={gpu.profile} />}
-        {gpu.mdev_path && <DetailRow label="Mdev path" value={gpu.mdev_path} />}
-        {gpu.pci_address && (
-          <DetailRow label="PCI address" value={gpu.pci_address} />
-        )}
-      </DetailCard.Content>
+      <DetailCard.Header>
+        {gpus.length === 1 ? "GPU" : `GPUs (${gpus.length})`}
+      </DetailCard.Header>
+      {gpus.map((gpu, index) => (
+        <div
+          className="border-b last:border-b-0"
+          key={gpu.mdev_path ?? gpu.pci_address ?? `${gpu.model}-${index}`}
+        >
+          {gpus.length > 1 && (
+            <h3 className="bg-muted/30 px-4 py-2 text-sm font-medium">
+              GPU {index + 1}
+            </h3>
+          )}
+          <DetailCard.Content>
+            <DetailRow label="Model" value={gpu.model} />
+            <DetailRow label="VRAM" value={formatVRAMMiB(gpu.vram_mib)} />
+            <DetailRow
+              label="Attachment"
+              value={gpu.profile ? "MIG slice" : "PCIe passthrough"}
+            />
+            {gpu.profile && <DetailRow label="Profile" value={gpu.profile} />}
+            {gpu.mdev_path && (
+              <DetailRow label="Mdev path" value={gpu.mdev_path} />
+            )}
+            {gpu.pci_address && (
+              <DetailRow label="PCI address" value={gpu.pci_address} />
+            )}
+          </DetailCard.Content>
+        </div>
+      ))}
     </DetailCard>
   )
 }

spinifex/services/spinifexui/frontend/src/routes/_auth/nodes.tsx

@@ -240,14 +240,17 @@ function NodesTable({ nodes }: { nodes: NodeInfo[] }) {
   )
 }
 
-export function formatVMGPU(gpu: VMInfo["gpu"]): string {
-  if (!gpu) {
+export function formatVMGPUs(gpus: VMInfo["gpus"]): string {
+  if (!gpus?.length) {
     return "-"
   }
-  if (gpu.profile) {
-    return `${gpu.profile} ${formatVRAMMiB(gpu.vram_mib)}`
-  }
-  return `${gpu.model} ${formatVRAMMiB(gpu.vram_mib)}`
+  return gpus
+    .map((gpu) =>
+      gpu.profile
+        ? `${gpu.profile} ${formatVRAMMiB(gpu.vram_mib)}`
+        : `${gpu.model} ${formatVRAMMiB(gpu.vram_mib)}`,
+    )
+    .join(", ")
 }
 
 function VMsTable({ vms }: { vms: VMInfo[] }) {
@@ -281,7 +284,7 @@ function VMsTable({ vms }: { vms: VMInfo[] }) {
               <td className="py-1.5 pr-4 font-mono">{vm.instance_type}</td>
               <td className="py-1.5 pr-4">{vm.vcpu}</td>
               <td className="py-1.5 pr-4">{formatMemory(vm.memory_gb)}</td>
-              <td className="py-1.5 pr-4 font-mono">{formatVMGPU(vm.gpu)}</td>
+              <td className="py-1.5 pr-4 font-mono">{formatVMGPUs(vm.gpus)}</td>
               <td className="py-1.5 pr-4 font-mono">{vm.node}</td>
               <td className="py-1.5">
                 {vm.launch_time > 0 ? formatAge(vm.launch_time) : "-"}

spinifex/types/node.go

@@ -27,7 +27,7 @@ type GPUInfo struct {
 	Slices     []GPUSliceInfo `json:"slices,omitempty"`
 }
 
-// VMGPUInfo describes the GPU attached to a VM.
+// VMGPUInfo describes one GPU attached to a VM.
 type VMGPUInfo struct {
 	Model      string `json:"model"`
 	VRAMMiB    int64  `json:"vram_mib"`
@@ -93,8 +93,8 @@ type VMInfo struct {
 	// ManagedBy is the Spinifex platform component that owns this VM
 	// (e.g. "elbv2"). Empty for customer VMs. The UI uses this to filter
 	// system-managed resources out of customer-facing listings.
-	ManagedBy string     `json:"managed_by,omitempty"`
-	GPU       *VMGPUInfo `json:"gpu,omitempty"`
+	ManagedBy string      `json:"managed_by,omitempty"`
+	GPUs      []VMGPUInfo `json:"gpus,omitempty"`
 	// Health is a display label for instance health: "ok", "impaired",
 	// "recovering", or "-" for non-running VMs. CrashCount is the lifetime
 	// crash tally within the current restart window.