gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: correct the GPU model catalog and stop unknown VRAM reading as zero (#688)

Browse Source 
The GPU model table was only ever used to report VRAM, so wrong or missing
entries were cosmetic. Bedrock's capacity check is the first code that rejects
on VRAM, which turns every latent data error into an admission decision.

Correct seven entries verified against pci.ids — most seriously 1002:7448,
labelled Instinct MI300X at 192GB when it is a Radeon Pro W7900 at 48GB, a
4x over-report that admits a model which then OOMs at weight load. Record the
smallest capacity where one PCI ID spans several cards, since under-reporting
refuses an endpoint an operator can see and override while over-reporting
fails later and looks like a vLLM bug.

Widen the table so common datacenter, workstation and consumer cards resolve
without an operator-supplied override, and apply GPUModelOverrides to the pool
devices as well as the advertised model, so the override reaches the code that
actually gates. An override with no memory_mib is left alone rather than
wiping discovered VRAM, since the config cannot distinguish it from a
deliberate zero.

MemoryMiB 0 stays the "undiscoverable" sentinel, but checkCapacity no longer
counts it as a device with no VRAM: an otherwise-free device now reports which
card is unknown and how to fix it.
This commit is contained in:
Josh Siviour
2026-08-04 13:36:54 +10:00
committed by GitHub
parent 5a3f5e60b2
commit 12a1e4aedc
7 changed files with 333 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spinifex/daemon/daemon.go
+27
View File
@@ -2855,6 +2855,11 @@ func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manage
models = append(models, resolveGPUModel(dev, cfg.GPUModelOverrides))
}
// Apply MemoryMiB overrides to the raw devices too, not just the advertised
// model above, so a configured override reaches admission checks (e.g.
// Bedrock capacity) that read Device.MemoryMiB straight from the pool.
applyGPUModelOverrides(wholeGPU, cfg.GPUModelOverrides)
mgr := gpu.NewManager(wholeGPU)
for _, me := range migEntries {
if len(me.existing) > 0 {
@@ -2870,6 +2875,28 @@ func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manage
return mgr, models, migProfiles
}
// applyGPUModelOverrides sets MemoryMiB on each device matched by a
// GPUModelOverride, in place, so a corrected VRAM figure reaches admission
// checks and not just the advertised model.
func applyGPUModelOverrides(devices []gpu.GPUDevice, overrides []config.GPUModelOverride) {
for i := range devices {
dev := &devices[i]
for _, o := range overrides {
if o.VendorID != dev.VendorID || o.DeviceID != dev.DeviceID {
continue
}
// Only a positive value applies. An override set purely for
// xvga_off or mig_profile leaves memory_mib at 0, which the struct
// cannot distinguish from a deliberate 0 — so treat it as "not
// specified" rather than wiping discovered VRAM.
if o.MemoryMiB > 0 {
dev.MemoryMiB = o.MemoryMiB
}
break
}
}
}
// resolveGPUModel maps a GPU device to an instance type model. Overrides take
// priority, then the production model list, then a g5 default.
func resolveGPUModel(dev gpu.GPUDevice, overrides []config.GPUModelOverride) instancetypes.GPUModel {
spinifex/daemon/daemon_test.go
+51
View File
@@ -4115,6 +4115,57 @@ func TestResolveGPUModel_OverrideCustomisesConsumerGPU(t *testing.T) {
assert.Equal(t, int64(12288), m.MemoryMiB)
}
// TestApplyGPUModelOverrides_SetsDeviceMemory pins that a GPUModelOverride
// reaches the raw device passed into gpu.NewManager, not just the advertised
// instance type, so admission checks that read Device.MemoryMiB directly
// (e.g. Bedrock capacity) see the override too.
func TestApplyGPUModelOverrides_SetsDeviceMemory(t *testing.T) {
devices := []gpu.GPUDevice{
{VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 0},
{VendorID: "10de", DeviceID: "2236", PCIAddress: "0000:04:00.0", MemoryMiB: 23028},
}
overrides := []config.GPUModelOverride{
{VendorID: "10de", DeviceID: "25b0", Name: "RTX A1000", MemoryMiB: 8188},
}
applyGPUModelOverrides(devices, overrides)
assert.Equal(t, int64(8188), devices[0].MemoryMiB, "overridden device should carry the corrected VRAM")
assert.Equal(t, int64(23028), devices[1].MemoryMiB, "device with no matching override is untouched")
}
// TestApplyGPUModelOverrides_ZeroMemoryDoesNotWipe covers an override set for
// a non-VRAM reason: memory_mib is absent, so it decodes to 0, and applying
// that would erase discovered VRAM and fail admission on a device that is fine.
func TestApplyGPUModelOverrides_ZeroMemoryDoesNotWipe(t *testing.T) {
devices := []gpu.GPUDevice{
{VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 8188},
}
overrides := []config.GPUModelOverride{
{VendorID: "10de", DeviceID: "25b0", XVGAOff: true},
}
applyGPUModelOverrides(devices, overrides)
assert.Equal(t, int64(8188), devices[0].MemoryMiB, "override without memory_mib must leave discovered VRAM intact")
}
// TestBuildGPUPool_OverrideReachesManagerSnapshot exercises the full path:
// buildGPUPool must apply GPUModelOverrides.MemoryMiB to the device before
// constructing the gpu.Manager, so Manager.Snapshot (what checkCapacity
// reads) reports the overridden VRAM rather than the raw discovered value.
func TestBuildGPUPool_OverrideReachesManagerSnapshot(t *testing.T) {
devices := []gpu.GPUDevice{
{VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 0},
}
cfg := config.DaemonConfig{
GPUModelOverrides: []config.GPUModelOverride{
{VendorID: "10de", DeviceID: "25b0", Name: "RTX A1000", MemoryMiB: 8188},
},
}
mgr, _, _ := buildGPUPool(devices, cfg)
require.NotNil(t, mgr)
snapshot := mgr.Snapshot()
require.Len(t, snapshot, 1)
assert.Equal(t, int64(8188), snapshot[0].Device.MemoryMiB)
}
// --- initServiceWithRetry ---
// stubInitRetrySleep replaces the package-level sleep seam with a recorder
spinifex/gpu/mig_parsers_test.go
+6 -5
View File
@@ -645,14 +645,14 @@ func TestEnrichMdevPaths_NoMatchingUUID_ReturnsError(t *testing.T) {
func TestIsMIGCapable(t *testing.T) {
capable := [][2]string{
{"10de", "20b2"}, // A100 SXM 40GB
{"10de", "20b5"}, // A100 SXM 80GB
{"10de", "20b2"}, // A100 SXM4 80GB
{"10de", "20b5"}, // A100 PCIe 80GB
{"10de", "20f1"}, // A100 PCIe 40GB
{"10de", "20f3"}, // A100 PCIe 80GB
{"10de", "2233"}, // A30
{"10de", "20f3"}, // A800-SXM4-80GB
{"10de", "20b7"}, // A30 PCIe
{"10de", "2330"}, // H100 SXM
{"10de", "2331"}, // H100 PCIe
{"10de", "233a"}, // H100 NVL
{"10de", "233a"}, // H800L 94GB
{"10de", "2335"}, // H200 SXM
{"10de", "2bb5"}, // RTX Pro 6000 Blackwell Server Edition
}
@@ -663,6 +663,7 @@ func TestIsMIGCapable(t *testing.T) {
notCapable := [][2]string{
{"10de", "2236"}, // A10 (not MIG capable)
{"10de", "2233"}, // RTX A5500 (workstation, has display)
{"10de", "1e04"}, // RTX 2080 Ti
{"1002", "687f"}, // AMD Vega 10
{"8086", "0000"}, // Intel
spinifex/gpu/models.go
+84 -24
View File
@@ -7,31 +7,75 @@ type modelInfo struct {
// knownModels maps "vendorID:deviceID" to human-readable model info.
// Used as a fallback when nvidia-smi/rocm-smi is unavailable.
//
// PCI IDs are checked against /usr/share/misc/pci.ids; do not add or change
// one without a matching line there. VRAM is not carried by pci.ids and must
// come from vendor documentation or an observed report, entry by entry.
//
// Ambiguity rule: several PCI IDs cover more than one card at different VRAM
// sizes (e.g. one Navi 31 ID spans four Radeon RX 7900 SKUs from 16 to 24GB).
// Record the SMALLEST VRAM among the covered cards. Under-reporting refuses
// an endpoint that would have fit, which an operator can see and override;
// over-reporting admits a model that then OOMs during weight load, on a host
// where the failure looks unrelated to the GPU table.
var knownModels = map[string]modelInfo{
// NVIDIA datacenter
// NVIDIA datacenter (compute)
"10de:2236": {"NVIDIA A10", 23028},
"10de:20b2": {"NVIDIA A100 SXM 40GB", 40960},
"10de:20b5": {"NVIDIA A100 SXM 80GB", 81920},
"10de:2237": {"NVIDIA A10G", 23028}, // same GA102 silicon/board as A10
"10de:2235": {"NVIDIA A40", 45634},
"10de:20b0": {"NVIDIA A100 SXM4 40GB", 40960},
"10de:20b2": {"NVIDIA A100 SXM4 80GB", 81920},
"10de:20b5": {"NVIDIA A100 PCIe 80GB", 81920},
"10de:20f1": {"NVIDIA A100 PCIe 40GB", 40960},
"10de:20f3": {"NVIDIA A100 PCIe 80GB", 81920},
"10de:20f3": {"NVIDIA A800-SXM4-80GB", 81920},
"10de:20b7": {"NVIDIA A30 PCIe", 24576},
"10de:25b6": {"NVIDIA A2 / A16", 15356}, // shared ID; both SKUs carry 16GB per physical GPU
"10de:27b8": {"NVIDIA L4", 23034},
"10de:1eb8": {"NVIDIA Tesla T4", 15109},
"10de:2330": {"NVIDIA H100 SXM", 81920},
"10de:2331": {"NVIDIA H100 PCIe", 81920},
"10de:233a": {"NVIDIA H100 NVL", 94208},
"10de:2337": {"NVIDIA H100 SXM5 64GB", 65536},
"10de:2321": {"NVIDIA H100 NVL 94GB", 94208},
"10de:233a": {"NVIDIA H800L 94GB", 94208},
"10de:2335": {"NVIDIA H200 SXM", 144384},
"10de:2233": {"NVIDIA A30", 24576},
"10de:233b": {"NVIDIA H200 NVL", 144384},
"10de:26b5": {"NVIDIA L40", 46068},
"10de:26b9": {"NVIDIA L40S", 46068},
"10de:2bb5": {"NVIDIA RTX Pro 6000 Blackwell Server Edition", 98304},
// NVIDIA workstation (display output; not compute, not MIG)
"10de:2233": {"NVIDIA RTX A5500", 24576},
"10de:2230": {"NVIDIA RTX A6000", 49140},
"10de:2231": {"NVIDIA RTX A5000", 24576},
"10de:24b0": {"NVIDIA RTX A4000", 16376},
"10de:2531": {"NVIDIA RTX A2000", 6138},
"10de:2571": {"NVIDIA RTX A2000 12GB", 12288},
"10de:25b0": {"NVIDIA RTX A1000", 8188}, // hardware-verified on wattle
"10de:26b1": {"NVIDIA RTX 6000 Ada Generation", 49152},
"10de:26b2": {"NVIDIA RTX 5000 Ada Generation", 32768},
"10de:27b2": {"NVIDIA RTX 4000 Ada Generation", 20480},
// NVIDIA consumer
"10de:2684": {"NVIDIA GeForce RTX 4090", 24576},
"10de:2782": {"NVIDIA GeForce RTX 4070", 12288},
"10de:2782": {"NVIDIA GeForce RTX 4070 Ti", 12288},
"10de:2204": {"NVIDIA GeForce RTX 3090", 24576},
// AMD datacenter
"1002:75a0": {"AMD Instinct MI350X", 294896},
"1002:7448": {"AMD Instinct MI300X", 196608},
"1002:740c": {"AMD Instinct MI250X", 65536},
"10de:2203": {"NVIDIA GeForce RTX 3090 Ti", 24576},
"10de:2208": {"NVIDIA GeForce RTX 3080 Ti", 12288},
"10de:2206": {"NVIDIA GeForce RTX 3080", 10240},
"10de:2487": {"NVIDIA GeForce RTX 3060", 12288},
"10de:2486": {"NVIDIA GeForce RTX 3060 Ti", 8192},
"10de:2484": {"NVIDIA GeForce RTX 3070", 8192},
"10de:1e04": {"NVIDIA GeForce RTX 2080 Ti", 11264},
// AMD datacenter (compute)
"1002:75a0": {"AMD Instinct MI350X", 294896}, // absent from local pci.ids snapshot; left unverified per plan
"1002:74a1": {"AMD Instinct MI300X", 196608},
"1002:74a0": {"AMD Instinct MI300A", 131072},
"1002:740c": {"AMD Instinct MI250X", 65536}, // per-GCD VRAM; card exposes 2 PCI devices, 64GB each
"1002:740f": {"AMD Instinct MI210", 65536},
"1002:738c": {"AMD Instinct MI100", 32768},
// AMD workstation (display output; not compute)
"1002:7448": {"AMD Radeon Pro W7900", 49152},
"1002:745e": {"AMD Radeon Pro W7800", 32768},
// AMD consumer
"1002:744c": {"AMD Radeon RX 7900 XTX", 24576},
"1002:744c": {"AMD Radeon RX 7900 XT/XTX/GRE/7900M", 16384}, // shared ID spans 16/16/20/24GB SKUs; smallest recorded
"1002:73bf": {"AMD Radeon RX 6900 XT", 16384},
}
@@ -41,37 +85,53 @@ var knownModels = map[string]modelInfo{
var computeModels = map[string]bool{
// NVIDIA datacenter
"10de:2236": true, // A10
"10de:20b2": true, // A100 SXM 40GB
"10de:20b5": true, // A100 SXM 80GB
"10de:2237": true, // A10G
"10de:2235": true, // A40
"10de:20b0": true, // A100 SXM4 40GB
"10de:20b2": true, // A100 SXM4 80GB
"10de:20b5": true, // A100 PCIe 80GB
"10de:20f1": true, // A100 PCIe 40GB
"10de:20f3": true, // A100 PCIe 80GB
"10de:20f3": true, // A800-SXM4-80GB
"10de:20b7": true, // A30 PCIe
"10de:25b6": true, // A2 / A16
"10de:27b8": true, // L4
"10de:1eb8": true, // Tesla T4
"10de:2330": true, // H100 SXM
"10de:2331": true, // H100 PCIe
"10de:233a": true, // H100 NVL
"10de:2337": true, // H100 SXM5 64GB
"10de:2321": true, // H100 NVL 94GB
"10de:233a": true, // H800L 94GB
"10de:2335": true, // H200 SXM
"10de:2233": true, // A30
"10de:233b": true, // H200 NVL
"10de:26b5": true, // L40
"10de:26b9": true, // L40S
"10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition
// AMD datacenter
"1002:75a0": true, // MI350X
"1002:7448": true, // MI300X
"1002:74a1": true, // MI300X
"1002:74a0": true, // MI300A
"1002:740c": true, // MI250X
"1002:740f": true, // MI210
"1002:738c": true, // MI100
}
// migCapableModels is the set of "vendorID:deviceID" keys for NVIDIA GPUs that
// support MIG partitioning. All are also in computeModels (headless datacenter cards).
// A10, L4, L40S are compute GPUs but do NOT support MIG.
// A10, A10G, L4, L40S and T4 are compute GPUs but do NOT support MIG.
var migCapableModels = map[string]bool{
"10de:20b2": true, // A100 SXM 40GB
"10de:20b5": true, // A100 SXM 80GB
"10de:20b0": true, // A100 SXM4 40GB
"10de:20b2": true, // A100 SXM4 80GB
"10de:20b5": true, // A100 PCIe 80GB
"10de:20f1": true, // A100 PCIe 40GB
"10de:20f3": true, // A100 PCIe 80GB
"10de:20f3": true, // A800-SXM4-80GB
"10de:20b7": true, // A30 PCIe
"10de:2330": true, // H100 SXM
"10de:2331": true, // H100 PCIe
"10de:233a": true, // H100 NVL
"10de:2337": true, // H100 SXM5 64GB
"10de:2321": true, // H100 NVL 94GB
"10de:233a": true, // H800L 94GB
"10de:2335": true, // H200 SXM
"10de:2233": true, // A30
"10de:233b": true, // H200 NVL
"10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition
}
spinifex/gpu/models_test.go
+100 -5
View File
@@ -23,12 +23,107 @@ func TestLookupModel_RTXPro6000BlackwellSE(t *testing.T) {
}
func TestLookupModel_KnownAMD(t *testing.T) {
name, mem := lookupModel("1002", "744c")
if name != "AMD Radeon RX 7900 XTX" {
t.Errorf("name = %q, want AMD Radeon RX 7900 XTX", name)
name, mem := lookupModel("1002", "740f")
if name != "AMD Instinct MI210" {
t.Errorf("name = %q, want AMD Instinct MI210", name)
}
if mem != 24576 {
t.Errorf("memoryMiB = %d, want 24576", mem)
if mem != 65536 {
t.Errorf("memoryMiB = %d, want 65536", mem)
}
}
// TestLookupModel_AmbiguityRule pins 1002:744c to the smallest of the four
// Radeon RX 7900 SKUs (16/16/20/24GB) the ID covers, per the table's
// documented under-report convention for shared PCI IDs.
func TestLookupModel_AmbiguityRule(t *testing.T) {
name, mem := lookupModel("1002", "744c")
if name != "AMD Radeon RX 7900 XT/XTX/GRE/7900M" {
t.Errorf("name = %q, want AMD Radeon RX 7900 XT/XTX/GRE/7900M", name)
}
if mem != 16384 {
t.Errorf("memoryMiB = %d, want 16384 (smallest covered SKU)", mem)
}
}
// TestLookupModel_CorrectedEntries pins the corrected table entries by name
// and VRAM, so a future edit cannot silently reintroduce a card labelled as
// hardware it is not.
func TestLookupModel_CorrectedEntries(t *testing.T) {
tests := []struct {
vendorID, deviceID string
wantName string
wantMemMiB int64
}{
{"1002", "7448", "AMD Radeon Pro W7900", 49152},
{"1002", "74a1", "AMD Instinct MI300X", 196608},
{"10de", "2233", "NVIDIA RTX A5500", 24576},
{"10de", "20b7", "NVIDIA A30 PCIe", 24576},
{"10de", "20b2", "NVIDIA A100 SXM4 80GB", 81920},
{"10de", "20b0", "NVIDIA A100 SXM4 40GB", 40960},
{"10de", "20b5", "NVIDIA A100 PCIe 80GB", 81920},
{"10de", "20f3", "NVIDIA A800-SXM4-80GB", 81920},
{"10de", "233a", "NVIDIA H800L 94GB", 94208},
{"10de", "25b0", "NVIDIA RTX A1000", 8188},
}
for _, tc := range tests {
t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) {
name, mem := lookupModel(tc.vendorID, tc.deviceID)
if name != tc.wantName {
t.Errorf("name = %q, want %q", name, tc.wantName)
}
if mem != tc.wantMemMiB {
t.Errorf("memoryMiB = %d, want %d", mem, tc.wantMemMiB)
}
})
}
}
// TestModels_CorrectedMembership asserts the corrected entries landed in the
// right compute/MIG sets: the wattle A1000 and the newly-real A30 are
// compute+MIG where applicable, while the corrected W7900/A5500/RX7900 stay
// out of both since they're display-capable workstation/consumer cards.
func TestModels_CorrectedMembership(t *testing.T) {
tests := []struct {
vendorID, deviceID string
wantCompute, wantMIG bool
}{
{"1002", "7448", false, false}, // W7900: workstation, has display
{"1002", "74a1", true, false}, // MI300X: compute, AMD has no MIG
{"10de", "2233", false, false}, // A5500: workstation, has display
{"10de", "20b7", true, true}, // A30 PCIe: compute + MIG-capable
{"10de", "20b2", true, true}, // A100 SXM4 80GB: compute + MIG-capable
{"10de", "25b0", false, false}, // A1000 (wattle): workstation, has display
{"1002", "744c", false, false}, // RX 7900 family: consumer, has display
}
for _, tc := range tests {
t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) {
if got := IsComputeGPU(tc.vendorID, tc.deviceID); got != tc.wantCompute {
t.Errorf("IsComputeGPU(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantCompute)
}
if got := IsMIGCapable(tc.vendorID, tc.deviceID); got != tc.wantMIG {
t.Errorf("IsMIGCapable(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantMIG)
}
})
}
}
// TestModels_NonMIGCapableCompute pins the plan's explicit exception list:
// A10, A10G, L4, L40S and T4 are compute GPUs but do not support MIG.
func TestModels_NonMIGCapableCompute(t *testing.T) {
nonMIG := []struct{ vendorID, deviceID string }{
{"10de", "2236"}, // A10
{"10de", "2237"}, // A10G
{"10de", "27b8"}, // L4
{"10de", "26b9"}, // L40S
{"10de", "1eb8"}, // T4
}
for _, tc := range nonMIG {
if !IsComputeGPU(tc.vendorID, tc.deviceID) {
t.Errorf("%s:%s expected to be a compute GPU", tc.vendorID, tc.deviceID)
}
if IsMIGCapable(tc.vendorID, tc.deviceID) {
t.Errorf("%s:%s expected NOT to be MIG-capable", tc.vendorID, tc.deviceID)
}
}
}
spinifex/handlers/bedrock/capacity.go
+16 -1
View File
@@ -24,7 +24,12 @@ func checkCapacity(snapshot []gpu.PoolEntry, minVRAMMiB int) error {
if minVRAMMiB <= 0 {
return fmt.Errorf("bedrock: invalid MinVRAMMiB %d", minVRAMMiB)
}
for _, entry := range snapshot {
// An available device with 0 MiB reported is undiscoverable VRAM, not an
// exhausted or tiny device — memMiB >= minVRAMMiB below would otherwise
// read it as "too small" and blame capacity for a data-gap problem.
var unknownVRAM *gpu.PoolEntry
for i := range snapshot {
entry := &snapshot[i]
if !entry.Available {
continue
}
@@ -35,6 +40,16 @@ func checkCapacity(snapshot []gpu.PoolEntry, minVRAMMiB int) error {
if memMiB >= int64(minVRAMMiB) {
return nil
}
if memMiB == 0 && unknownVRAM == nil {
unknownVRAM = entry
}
}
if unknownVRAM != nil {
return awserrors.Errorf(awserrors.ErrorModelNotReadyException,
"bedrock: GPU %s (%s, vendor:device %s:%s) has undiscoverable VRAM (reported 0 MiB); "+
"add it to the gpu model catalog or set memory_mib in a GPUModelOverride for it",
unknownVRAM.Device.PCIAddress, unknownVRAM.Device.Model,
unknownVRAM.Device.VendorID, unknownVRAM.Device.DeviceID)
}
return awserrors.Errorf(awserrors.ErrorModelNotReadyException,
"bedrock: no free GPU device has %d MiB of VRAM free", minVRAMMiB)
spinifex/handlers/bedrock/capacity_test.go
+49
View File
@@ -6,6 +6,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/awserrors"
"github.com/mulgadc/spinifex/spinifex/gpu"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckCapacity(t *testing.T) {
@@ -83,6 +84,54 @@ func TestCheckCapacity_ErrorCode(t *testing.T) {
assert.Equal(t, awserrors.ErrorModelNotReadyException, awserrors.ValidErrorCodeFromError(err))
}
// TestCheckCapacity_UnknownVRAM pins that an available device
// reporting 0 MiB (undiscoverable, not exhausted) must produce a distinct,
// actionable error naming the device — not the generic "no free GPU device
// has N MiB free" exhaustion message, which would mislead an operator into
// thinking the card is genuinely too small or already claimed.
func TestCheckCapacity_UnknownVRAM(t *testing.T) {
snapshot := []gpu.PoolEntry{
{
Device: gpu.GPUDevice{PCIAddress: "0000:03:00.0", Model: "NVIDIA RTX A1000", VendorID: "10de", DeviceID: "25b0", MemoryMiB: 0},
Available: true,
},
}
err := checkCapacity(snapshot, 5120)
require.Error(t, err)
assert.Contains(t, err.Error(), "0000:03:00.0")
assert.Contains(t, err.Error(), "10de:25b0")
assert.Contains(t, err.Error(), "gpu model catalog")
assert.Contains(t, err.Error(), "GPUModelOverride")
assert.NotContains(t, err.Error(), "no free GPU device has")
}
// TestCheckCapacity_UnknownVRAM_DistinctFromExhaustion proves the two
// failure modes produce different messages for the same minVRAMMiB, so an
// operator (or a caller matching on message text) can tell "data gap" apart
// from "genuinely too small / already claimed".
func TestCheckCapacity_UnknownVRAM_DistinctFromExhaustion(t *testing.T) {
unknownErr := checkCapacity([]gpu.PoolEntry{
{Device: gpu.GPUDevice{MemoryMiB: 0}, Available: true},
}, 5120)
exhaustedErr := checkCapacity([]gpu.PoolEntry{
{Device: gpu.GPUDevice{MemoryMiB: 2048}, Available: true},
}, 5120)
require.Error(t, unknownErr)
require.Error(t, exhaustedErr)
assert.NotEqual(t, unknownErr.Error(), exhaustedErr.Error())
}
// TestCheckCapacity_UnknownVRAM_SkippedWhenAnotherDeviceSuffices ensures an
// unknown-VRAM device never blocks admission when a different available
// device in the same snapshot already satisfies minVRAMMiB.
func TestCheckCapacity_UnknownVRAM_SkippedWhenAnotherDeviceSuffices(t *testing.T) {
snapshot := []gpu.PoolEntry{
{Device: gpu.GPUDevice{MemoryMiB: 0}, Available: true},
{Device: gpu.GPUDevice{MemoryMiB: 16384}, Available: true},
}
assert.NoError(t, checkCapacity(snapshot, 5120))
}
type stubSnapshotter struct {
entries []gpu.PoolEntry
}