diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index fb2bcef56..6ca5f121c 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -2855,6 +2855,11 @@ func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manage models = append(models, resolveGPUModel(dev, cfg.GPUModelOverrides)) } + // Apply MemoryMiB overrides to the raw devices too, not just the advertised + // model above, so a configured override reaches admission checks (e.g. + // Bedrock capacity) that read Device.MemoryMiB straight from the pool. + applyGPUModelOverrides(wholeGPU, cfg.GPUModelOverrides) + mgr := gpu.NewManager(wholeGPU) for _, me := range migEntries { if len(me.existing) > 0 { @@ -2870,6 +2875,28 @@ func buildGPUPool(devices []gpu.GPUDevice, cfg config.DaemonConfig) (*gpu.Manage return mgr, models, migProfiles } +// applyGPUModelOverrides sets MemoryMiB on each device matched by a +// GPUModelOverride, in place, so a corrected VRAM figure reaches admission +// checks and not just the advertised model. +func applyGPUModelOverrides(devices []gpu.GPUDevice, overrides []config.GPUModelOverride) { + for i := range devices { + dev := &devices[i] + for _, o := range overrides { + if o.VendorID != dev.VendorID || o.DeviceID != dev.DeviceID { + continue + } + // Only a positive value applies. An override set purely for + // xvga_off or mig_profile leaves memory_mib at 0, which the struct + // cannot distinguish from a deliberate 0 — so treat it as "not + // specified" rather than wiping discovered VRAM. + if o.MemoryMiB > 0 { + dev.MemoryMiB = o.MemoryMiB + } + break + } + } +} + // resolveGPUModel maps a GPU device to an instance type model. Overrides take // priority, then the production model list, then a g5 default. func resolveGPUModel(dev gpu.GPUDevice, overrides []config.GPUModelOverride) instancetypes.GPUModel { diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index e65e0607e..ddfe2b581 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -4115,6 +4115,57 @@ func TestResolveGPUModel_OverrideCustomisesConsumerGPU(t *testing.T) { assert.Equal(t, int64(12288), m.MemoryMiB) } +// TestApplyGPUModelOverrides_SetsDeviceMemory pins that a GPUModelOverride +// reaches the raw device passed into gpu.NewManager, not just the advertised +// instance type, so admission checks that read Device.MemoryMiB directly +// (e.g. Bedrock capacity) see the override too. +func TestApplyGPUModelOverrides_SetsDeviceMemory(t *testing.T) { + devices := []gpu.GPUDevice{ + {VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 0}, + {VendorID: "10de", DeviceID: "2236", PCIAddress: "0000:04:00.0", MemoryMiB: 23028}, + } + overrides := []config.GPUModelOverride{ + {VendorID: "10de", DeviceID: "25b0", Name: "RTX A1000", MemoryMiB: 8188}, + } + applyGPUModelOverrides(devices, overrides) + assert.Equal(t, int64(8188), devices[0].MemoryMiB, "overridden device should carry the corrected VRAM") + assert.Equal(t, int64(23028), devices[1].MemoryMiB, "device with no matching override is untouched") +} + +// TestApplyGPUModelOverrides_ZeroMemoryDoesNotWipe covers an override set for +// a non-VRAM reason: memory_mib is absent, so it decodes to 0, and applying +// that would erase discovered VRAM and fail admission on a device that is fine. +func TestApplyGPUModelOverrides_ZeroMemoryDoesNotWipe(t *testing.T) { + devices := []gpu.GPUDevice{ + {VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 8188}, + } + overrides := []config.GPUModelOverride{ + {VendorID: "10de", DeviceID: "25b0", XVGAOff: true}, + } + applyGPUModelOverrides(devices, overrides) + assert.Equal(t, int64(8188), devices[0].MemoryMiB, "override without memory_mib must leave discovered VRAM intact") +} + +// TestBuildGPUPool_OverrideReachesManagerSnapshot exercises the full path: +// buildGPUPool must apply GPUModelOverrides.MemoryMiB to the device before +// constructing the gpu.Manager, so Manager.Snapshot (what checkCapacity +// reads) reports the overridden VRAM rather than the raw discovered value. +func TestBuildGPUPool_OverrideReachesManagerSnapshot(t *testing.T) { + devices := []gpu.GPUDevice{ + {VendorID: "10de", DeviceID: "25b0", PCIAddress: "0000:03:00.0", MemoryMiB: 0}, + } + cfg := config.DaemonConfig{ + GPUModelOverrides: []config.GPUModelOverride{ + {VendorID: "10de", DeviceID: "25b0", Name: "RTX A1000", MemoryMiB: 8188}, + }, + } + mgr, _, _ := buildGPUPool(devices, cfg) + require.NotNil(t, mgr) + snapshot := mgr.Snapshot() + require.Len(t, snapshot, 1) + assert.Equal(t, int64(8188), snapshot[0].Device.MemoryMiB) +} + // --- initServiceWithRetry --- // stubInitRetrySleep replaces the package-level sleep seam with a recorder diff --git a/spinifex/gpu/mig_parsers_test.go b/spinifex/gpu/mig_parsers_test.go index 926132fcc..1e9d2aacc 100644 --- a/spinifex/gpu/mig_parsers_test.go +++ b/spinifex/gpu/mig_parsers_test.go @@ -645,14 +645,14 @@ func TestEnrichMdevPaths_NoMatchingUUID_ReturnsError(t *testing.T) { func TestIsMIGCapable(t *testing.T) { capable := [][2]string{ - {"10de", "20b2"}, // A100 SXM 40GB - {"10de", "20b5"}, // A100 SXM 80GB + {"10de", "20b2"}, // A100 SXM4 80GB + {"10de", "20b5"}, // A100 PCIe 80GB {"10de", "20f1"}, // A100 PCIe 40GB - {"10de", "20f3"}, // A100 PCIe 80GB - {"10de", "2233"}, // A30 + {"10de", "20f3"}, // A800-SXM4-80GB + {"10de", "20b7"}, // A30 PCIe {"10de", "2330"}, // H100 SXM {"10de", "2331"}, // H100 PCIe - {"10de", "233a"}, // H100 NVL + {"10de", "233a"}, // H800L 94GB {"10de", "2335"}, // H200 SXM {"10de", "2bb5"}, // RTX Pro 6000 Blackwell Server Edition } @@ -663,6 +663,7 @@ func TestIsMIGCapable(t *testing.T) { notCapable := [][2]string{ {"10de", "2236"}, // A10 (not MIG capable) + {"10de", "2233"}, // RTX A5500 (workstation, has display) {"10de", "1e04"}, // RTX 2080 Ti {"1002", "687f"}, // AMD Vega 10 {"8086", "0000"}, // Intel diff --git a/spinifex/gpu/models.go b/spinifex/gpu/models.go index 5352f6796..85f3de0b2 100644 --- a/spinifex/gpu/models.go +++ b/spinifex/gpu/models.go @@ -7,31 +7,75 @@ type modelInfo struct { // knownModels maps "vendorID:deviceID" to human-readable model info. // Used as a fallback when nvidia-smi/rocm-smi is unavailable. +// +// PCI IDs are checked against /usr/share/misc/pci.ids; do not add or change +// one without a matching line there. VRAM is not carried by pci.ids and must +// come from vendor documentation or an observed report, entry by entry. +// +// Ambiguity rule: several PCI IDs cover more than one card at different VRAM +// sizes (e.g. one Navi 31 ID spans four Radeon RX 7900 SKUs from 16 to 24GB). +// Record the SMALLEST VRAM among the covered cards. Under-reporting refuses +// an endpoint that would have fit, which an operator can see and override; +// over-reporting admits a model that then OOMs during weight load, on a host +// where the failure looks unrelated to the GPU table. var knownModels = map[string]modelInfo{ - // NVIDIA datacenter + // NVIDIA datacenter (compute) "10de:2236": {"NVIDIA A10", 23028}, - "10de:20b2": {"NVIDIA A100 SXM 40GB", 40960}, - "10de:20b5": {"NVIDIA A100 SXM 80GB", 81920}, + "10de:2237": {"NVIDIA A10G", 23028}, // same GA102 silicon/board as A10 + "10de:2235": {"NVIDIA A40", 45634}, + "10de:20b0": {"NVIDIA A100 SXM4 40GB", 40960}, + "10de:20b2": {"NVIDIA A100 SXM4 80GB", 81920}, + "10de:20b5": {"NVIDIA A100 PCIe 80GB", 81920}, "10de:20f1": {"NVIDIA A100 PCIe 40GB", 40960}, - "10de:20f3": {"NVIDIA A100 PCIe 80GB", 81920}, + "10de:20f3": {"NVIDIA A800-SXM4-80GB", 81920}, + "10de:20b7": {"NVIDIA A30 PCIe", 24576}, + "10de:25b6": {"NVIDIA A2 / A16", 15356}, // shared ID; both SKUs carry 16GB per physical GPU + "10de:27b8": {"NVIDIA L4", 23034}, + "10de:1eb8": {"NVIDIA Tesla T4", 15109}, "10de:2330": {"NVIDIA H100 SXM", 81920}, "10de:2331": {"NVIDIA H100 PCIe", 81920}, - "10de:233a": {"NVIDIA H100 NVL", 94208}, + "10de:2337": {"NVIDIA H100 SXM5 64GB", 65536}, + "10de:2321": {"NVIDIA H100 NVL 94GB", 94208}, + "10de:233a": {"NVIDIA H800L 94GB", 94208}, "10de:2335": {"NVIDIA H200 SXM", 144384}, - "10de:2233": {"NVIDIA A30", 24576}, + "10de:233b": {"NVIDIA H200 NVL", 144384}, "10de:26b5": {"NVIDIA L40", 46068}, "10de:26b9": {"NVIDIA L40S", 46068}, "10de:2bb5": {"NVIDIA RTX Pro 6000 Blackwell Server Edition", 98304}, + // NVIDIA workstation (display output; not compute, not MIG) + "10de:2233": {"NVIDIA RTX A5500", 24576}, + "10de:2230": {"NVIDIA RTX A6000", 49140}, + "10de:2231": {"NVIDIA RTX A5000", 24576}, + "10de:24b0": {"NVIDIA RTX A4000", 16376}, + "10de:2531": {"NVIDIA RTX A2000", 6138}, + "10de:2571": {"NVIDIA RTX A2000 12GB", 12288}, + "10de:25b0": {"NVIDIA RTX A1000", 8188}, // hardware-verified on wattle + "10de:26b1": {"NVIDIA RTX 6000 Ada Generation", 49152}, + "10de:26b2": {"NVIDIA RTX 5000 Ada Generation", 32768}, + "10de:27b2": {"NVIDIA RTX 4000 Ada Generation", 20480}, // NVIDIA consumer "10de:2684": {"NVIDIA GeForce RTX 4090", 24576}, - "10de:2782": {"NVIDIA GeForce RTX 4070", 12288}, + "10de:2782": {"NVIDIA GeForce RTX 4070 Ti", 12288}, "10de:2204": {"NVIDIA GeForce RTX 3090", 24576}, - // AMD datacenter - "1002:75a0": {"AMD Instinct MI350X", 294896}, - "1002:7448": {"AMD Instinct MI300X", 196608}, - "1002:740c": {"AMD Instinct MI250X", 65536}, + "10de:2203": {"NVIDIA GeForce RTX 3090 Ti", 24576}, + "10de:2208": {"NVIDIA GeForce RTX 3080 Ti", 12288}, + "10de:2206": {"NVIDIA GeForce RTX 3080", 10240}, + "10de:2487": {"NVIDIA GeForce RTX 3060", 12288}, + "10de:2486": {"NVIDIA GeForce RTX 3060 Ti", 8192}, + "10de:2484": {"NVIDIA GeForce RTX 3070", 8192}, + "10de:1e04": {"NVIDIA GeForce RTX 2080 Ti", 11264}, + // AMD datacenter (compute) + "1002:75a0": {"AMD Instinct MI350X", 294896}, // absent from local pci.ids snapshot; left unverified per plan + "1002:74a1": {"AMD Instinct MI300X", 196608}, + "1002:74a0": {"AMD Instinct MI300A", 131072}, + "1002:740c": {"AMD Instinct MI250X", 65536}, // per-GCD VRAM; card exposes 2 PCI devices, 64GB each + "1002:740f": {"AMD Instinct MI210", 65536}, + "1002:738c": {"AMD Instinct MI100", 32768}, + // AMD workstation (display output; not compute) + "1002:7448": {"AMD Radeon Pro W7900", 49152}, + "1002:745e": {"AMD Radeon Pro W7800", 32768}, // AMD consumer - "1002:744c": {"AMD Radeon RX 7900 XTX", 24576}, + "1002:744c": {"AMD Radeon RX 7900 XT/XTX/GRE/7900M", 16384}, // shared ID spans 16/16/20/24GB SKUs; smallest recorded "1002:73bf": {"AMD Radeon RX 6900 XT", 16384}, } @@ -41,37 +85,53 @@ var knownModels = map[string]modelInfo{ var computeModels = map[string]bool{ // NVIDIA datacenter "10de:2236": true, // A10 - "10de:20b2": true, // A100 SXM 40GB - "10de:20b5": true, // A100 SXM 80GB + "10de:2237": true, // A10G + "10de:2235": true, // A40 + "10de:20b0": true, // A100 SXM4 40GB + "10de:20b2": true, // A100 SXM4 80GB + "10de:20b5": true, // A100 PCIe 80GB "10de:20f1": true, // A100 PCIe 40GB - "10de:20f3": true, // A100 PCIe 80GB + "10de:20f3": true, // A800-SXM4-80GB + "10de:20b7": true, // A30 PCIe + "10de:25b6": true, // A2 / A16 + "10de:27b8": true, // L4 + "10de:1eb8": true, // Tesla T4 "10de:2330": true, // H100 SXM "10de:2331": true, // H100 PCIe - "10de:233a": true, // H100 NVL + "10de:2337": true, // H100 SXM5 64GB + "10de:2321": true, // H100 NVL 94GB + "10de:233a": true, // H800L 94GB "10de:2335": true, // H200 SXM - "10de:2233": true, // A30 + "10de:233b": true, // H200 NVL "10de:26b5": true, // L40 "10de:26b9": true, // L40S "10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition // AMD datacenter "1002:75a0": true, // MI350X - "1002:7448": true, // MI300X + "1002:74a1": true, // MI300X + "1002:74a0": true, // MI300A "1002:740c": true, // MI250X + "1002:740f": true, // MI210 + "1002:738c": true, // MI100 } // migCapableModels is the set of "vendorID:deviceID" keys for NVIDIA GPUs that // support MIG partitioning. All are also in computeModels (headless datacenter cards). -// A10, L4, L40S are compute GPUs but do NOT support MIG. +// A10, A10G, L4, L40S and T4 are compute GPUs but do NOT support MIG. var migCapableModels = map[string]bool{ - "10de:20b2": true, // A100 SXM 40GB - "10de:20b5": true, // A100 SXM 80GB + "10de:20b0": true, // A100 SXM4 40GB + "10de:20b2": true, // A100 SXM4 80GB + "10de:20b5": true, // A100 PCIe 80GB "10de:20f1": true, // A100 PCIe 40GB - "10de:20f3": true, // A100 PCIe 80GB + "10de:20f3": true, // A800-SXM4-80GB + "10de:20b7": true, // A30 PCIe "10de:2330": true, // H100 SXM "10de:2331": true, // H100 PCIe - "10de:233a": true, // H100 NVL + "10de:2337": true, // H100 SXM5 64GB + "10de:2321": true, // H100 NVL 94GB + "10de:233a": true, // H800L 94GB "10de:2335": true, // H200 SXM - "10de:2233": true, // A30 + "10de:233b": true, // H200 NVL "10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition } diff --git a/spinifex/gpu/models_test.go b/spinifex/gpu/models_test.go index 134ac5847..c8e78214c 100644 --- a/spinifex/gpu/models_test.go +++ b/spinifex/gpu/models_test.go @@ -23,12 +23,107 @@ func TestLookupModel_RTXPro6000BlackwellSE(t *testing.T) { } func TestLookupModel_KnownAMD(t *testing.T) { - name, mem := lookupModel("1002", "744c") - if name != "AMD Radeon RX 7900 XTX" { - t.Errorf("name = %q, want AMD Radeon RX 7900 XTX", name) + name, mem := lookupModel("1002", "740f") + if name != "AMD Instinct MI210" { + t.Errorf("name = %q, want AMD Instinct MI210", name) } - if mem != 24576 { - t.Errorf("memoryMiB = %d, want 24576", mem) + if mem != 65536 { + t.Errorf("memoryMiB = %d, want 65536", mem) + } +} + +// TestLookupModel_AmbiguityRule pins 1002:744c to the smallest of the four +// Radeon RX 7900 SKUs (16/16/20/24GB) the ID covers, per the table's +// documented under-report convention for shared PCI IDs. +func TestLookupModel_AmbiguityRule(t *testing.T) { + name, mem := lookupModel("1002", "744c") + if name != "AMD Radeon RX 7900 XT/XTX/GRE/7900M" { + t.Errorf("name = %q, want AMD Radeon RX 7900 XT/XTX/GRE/7900M", name) + } + if mem != 16384 { + t.Errorf("memoryMiB = %d, want 16384 (smallest covered SKU)", mem) + } +} + +// TestLookupModel_CorrectedEntries pins the corrected table entries by name +// and VRAM, so a future edit cannot silently reintroduce a card labelled as +// hardware it is not. +func TestLookupModel_CorrectedEntries(t *testing.T) { + tests := []struct { + vendorID, deviceID string + wantName string + wantMemMiB int64 + }{ + {"1002", "7448", "AMD Radeon Pro W7900", 49152}, + {"1002", "74a1", "AMD Instinct MI300X", 196608}, + {"10de", "2233", "NVIDIA RTX A5500", 24576}, + {"10de", "20b7", "NVIDIA A30 PCIe", 24576}, + {"10de", "20b2", "NVIDIA A100 SXM4 80GB", 81920}, + {"10de", "20b0", "NVIDIA A100 SXM4 40GB", 40960}, + {"10de", "20b5", "NVIDIA A100 PCIe 80GB", 81920}, + {"10de", "20f3", "NVIDIA A800-SXM4-80GB", 81920}, + {"10de", "233a", "NVIDIA H800L 94GB", 94208}, + {"10de", "25b0", "NVIDIA RTX A1000", 8188}, + } + for _, tc := range tests { + t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) { + name, mem := lookupModel(tc.vendorID, tc.deviceID) + if name != tc.wantName { + t.Errorf("name = %q, want %q", name, tc.wantName) + } + if mem != tc.wantMemMiB { + t.Errorf("memoryMiB = %d, want %d", mem, tc.wantMemMiB) + } + }) + } +} + +// TestModels_CorrectedMembership asserts the corrected entries landed in the +// right compute/MIG sets: the wattle A1000 and the newly-real A30 are +// compute+MIG where applicable, while the corrected W7900/A5500/RX7900 stay +// out of both since they're display-capable workstation/consumer cards. +func TestModels_CorrectedMembership(t *testing.T) { + tests := []struct { + vendorID, deviceID string + wantCompute, wantMIG bool + }{ + {"1002", "7448", false, false}, // W7900: workstation, has display + {"1002", "74a1", true, false}, // MI300X: compute, AMD has no MIG + {"10de", "2233", false, false}, // A5500: workstation, has display + {"10de", "20b7", true, true}, // A30 PCIe: compute + MIG-capable + {"10de", "20b2", true, true}, // A100 SXM4 80GB: compute + MIG-capable + {"10de", "25b0", false, false}, // A1000 (wattle): workstation, has display + {"1002", "744c", false, false}, // RX 7900 family: consumer, has display + } + for _, tc := range tests { + t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) { + if got := IsComputeGPU(tc.vendorID, tc.deviceID); got != tc.wantCompute { + t.Errorf("IsComputeGPU(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantCompute) + } + if got := IsMIGCapable(tc.vendorID, tc.deviceID); got != tc.wantMIG { + t.Errorf("IsMIGCapable(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantMIG) + } + }) + } +} + +// TestModels_NonMIGCapableCompute pins the plan's explicit exception list: +// A10, A10G, L4, L40S and T4 are compute GPUs but do not support MIG. +func TestModels_NonMIGCapableCompute(t *testing.T) { + nonMIG := []struct{ vendorID, deviceID string }{ + {"10de", "2236"}, // A10 + {"10de", "2237"}, // A10G + {"10de", "27b8"}, // L4 + {"10de", "26b9"}, // L40S + {"10de", "1eb8"}, // T4 + } + for _, tc := range nonMIG { + if !IsComputeGPU(tc.vendorID, tc.deviceID) { + t.Errorf("%s:%s expected to be a compute GPU", tc.vendorID, tc.deviceID) + } + if IsMIGCapable(tc.vendorID, tc.deviceID) { + t.Errorf("%s:%s expected NOT to be MIG-capable", tc.vendorID, tc.deviceID) + } } } diff --git a/spinifex/handlers/bedrock/capacity.go b/spinifex/handlers/bedrock/capacity.go index aca100f4c..0e6819f74 100644 --- a/spinifex/handlers/bedrock/capacity.go +++ b/spinifex/handlers/bedrock/capacity.go @@ -24,7 +24,12 @@ func checkCapacity(snapshot []gpu.PoolEntry, minVRAMMiB int) error { if minVRAMMiB <= 0 { return fmt.Errorf("bedrock: invalid MinVRAMMiB %d", minVRAMMiB) } - for _, entry := range snapshot { + // An available device with 0 MiB reported is undiscoverable VRAM, not an + // exhausted or tiny device — memMiB >= minVRAMMiB below would otherwise + // read it as "too small" and blame capacity for a data-gap problem. + var unknownVRAM *gpu.PoolEntry + for i := range snapshot { + entry := &snapshot[i] if !entry.Available { continue } @@ -35,6 +40,16 @@ func checkCapacity(snapshot []gpu.PoolEntry, minVRAMMiB int) error { if memMiB >= int64(minVRAMMiB) { return nil } + if memMiB == 0 && unknownVRAM == nil { + unknownVRAM = entry + } + } + if unknownVRAM != nil { + return awserrors.Errorf(awserrors.ErrorModelNotReadyException, + "bedrock: GPU %s (%s, vendor:device %s:%s) has undiscoverable VRAM (reported 0 MiB); "+ + "add it to the gpu model catalog or set memory_mib in a GPUModelOverride for it", + unknownVRAM.Device.PCIAddress, unknownVRAM.Device.Model, + unknownVRAM.Device.VendorID, unknownVRAM.Device.DeviceID) } return awserrors.Errorf(awserrors.ErrorModelNotReadyException, "bedrock: no free GPU device has %d MiB of VRAM free", minVRAMMiB) diff --git a/spinifex/handlers/bedrock/capacity_test.go b/spinifex/handlers/bedrock/capacity_test.go index 0d579428b..42e84d90f 100644 --- a/spinifex/handlers/bedrock/capacity_test.go +++ b/spinifex/handlers/bedrock/capacity_test.go @@ -6,6 +6,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/gpu" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCheckCapacity(t *testing.T) { @@ -83,6 +84,54 @@ func TestCheckCapacity_ErrorCode(t *testing.T) { assert.Equal(t, awserrors.ErrorModelNotReadyException, awserrors.ValidErrorCodeFromError(err)) } +// TestCheckCapacity_UnknownVRAM pins that an available device +// reporting 0 MiB (undiscoverable, not exhausted) must produce a distinct, +// actionable error naming the device — not the generic "no free GPU device +// has N MiB free" exhaustion message, which would mislead an operator into +// thinking the card is genuinely too small or already claimed. +func TestCheckCapacity_UnknownVRAM(t *testing.T) { + snapshot := []gpu.PoolEntry{ + { + Device: gpu.GPUDevice{PCIAddress: "0000:03:00.0", Model: "NVIDIA RTX A1000", VendorID: "10de", DeviceID: "25b0", MemoryMiB: 0}, + Available: true, + }, + } + err := checkCapacity(snapshot, 5120) + require.Error(t, err) + assert.Contains(t, err.Error(), "0000:03:00.0") + assert.Contains(t, err.Error(), "10de:25b0") + assert.Contains(t, err.Error(), "gpu model catalog") + assert.Contains(t, err.Error(), "GPUModelOverride") + assert.NotContains(t, err.Error(), "no free GPU device has") +} + +// TestCheckCapacity_UnknownVRAM_DistinctFromExhaustion proves the two +// failure modes produce different messages for the same minVRAMMiB, so an +// operator (or a caller matching on message text) can tell "data gap" apart +// from "genuinely too small / already claimed". +func TestCheckCapacity_UnknownVRAM_DistinctFromExhaustion(t *testing.T) { + unknownErr := checkCapacity([]gpu.PoolEntry{ + {Device: gpu.GPUDevice{MemoryMiB: 0}, Available: true}, + }, 5120) + exhaustedErr := checkCapacity([]gpu.PoolEntry{ + {Device: gpu.GPUDevice{MemoryMiB: 2048}, Available: true}, + }, 5120) + require.Error(t, unknownErr) + require.Error(t, exhaustedErr) + assert.NotEqual(t, unknownErr.Error(), exhaustedErr.Error()) +} + +// TestCheckCapacity_UnknownVRAM_SkippedWhenAnotherDeviceSuffices ensures an +// unknown-VRAM device never blocks admission when a different available +// device in the same snapshot already satisfies minVRAMMiB. +func TestCheckCapacity_UnknownVRAM_SkippedWhenAnotherDeviceSuffices(t *testing.T) { + snapshot := []gpu.PoolEntry{ + {Device: gpu.GPUDevice{MemoryMiB: 0}, Available: true}, + {Device: gpu.GPUDevice{MemoryMiB: 16384}, Available: true}, + } + assert.NoError(t, checkCapacity(snapshot, 5120)) +} + type stubSnapshotter struct { entries []gpu.PoolEntry }