fix: correct the GPU model catalog and stop unknown VRAM reading as zero (#688)

The GPU model table was only ever used to report VRAM, so wrong or missing entries were cosmetic. Bedrock's capacity check is the first code that rejects on VRAM, which turns every latent data error into an admission decision. Correct seven entries verified against pci.ids — most seriously 1002:7448, labelled Instinct MI300X at 192GB when it is a Radeon Pro W7900 at 48GB, a 4x over-report that admits a model which then OOMs at weight load. Record the smallest capacity where one PCI ID spans several cards, since under-reporting refuses an endpoint an operator can see and override while over-reporting fails later and looks like a vLLM bug. Widen the table so common datacenter, workstation and consumer cards resolve without an operator-supplied override, and apply GPUModelOverrides to the pool devices as well as the advertised model, so the override reaches the code that actually gates. An override with no memory_mib is left alone rather than wiping discovered VRAM, since the config cannot distinguish it from a deliberate zero. MemoryMiB 0 stays the "undiscoverable" sentinel, but checkCapacity no longer counts it as a device with no VRAM: an otherwise-free device now reports which card is unknown and how to fix it.
2026-10-22 04:07:04 +00:00 · 2026-08-04 13:36:54 +10:00
parent 5a3f5e60b2
commit 12a1e4aedc
7 changed files with 333 additions and 35 deletions
@@ -645,14 +645,14 @@ func TestEnrichMdevPaths_NoMatchingUUID_ReturnsError(t *testing.T) {

 func TestIsMIGCapable(t *testing.T) {
 	capable := [][2]string{
-		{"10de", "20b2"}, // A100 SXM 40GB
-		{"10de", "20b5"}, // A100 SXM 80GB
+		{"10de", "20b2"}, // A100 SXM4 80GB
+		{"10de", "20b5"}, // A100 PCIe 80GB
 		{"10de", "20f1"}, // A100 PCIe 40GB
-		{"10de", "20f3"}, // A100 PCIe 80GB
-		{"10de", "2233"}, // A30
+		{"10de", "20f3"}, // A800-SXM4-80GB
+		{"10de", "20b7"}, // A30 PCIe
 		{"10de", "2330"}, // H100 SXM
 		{"10de", "2331"}, // H100 PCIe
-		{"10de", "233a"}, // H100 NVL
+		{"10de", "233a"}, // H800L 94GB
 		{"10de", "2335"}, // H200 SXM
 		{"10de", "2bb5"}, // RTX Pro 6000 Blackwell Server Edition
 	}
@@ -663,6 +663,7 @@ func TestIsMIGCapable(t *testing.T) {

 	notCapable := [][2]string{
 		{"10de", "2236"}, // A10 (not MIG capable)
+		{"10de", "2233"}, // RTX A5500 (workstation, has display)
 		{"10de", "1e04"}, // RTX 2080 Ti
 		{"1002", "687f"}, // AMD Vega 10
 		{"8086", "0000"}, // Intel
@@ -7,31 +7,75 @@ type modelInfo struct {

 // knownModels maps "vendorID:deviceID" to human-readable model info.
 // Used as a fallback when nvidia-smi/rocm-smi is unavailable.
+//
+// PCI IDs are checked against /usr/share/misc/pci.ids; do not add or change
+// one without a matching line there. VRAM is not carried by pci.ids and must
+// come from vendor documentation or an observed report, entry by entry.
+//
+// Ambiguity rule: several PCI IDs cover more than one card at different VRAM
+// sizes (e.g. one Navi 31 ID spans four Radeon RX 7900 SKUs from 16 to 24GB).
+// Record the SMALLEST VRAM among the covered cards. Under-reporting refuses
+// an endpoint that would have fit, which an operator can see and override;
+// over-reporting admits a model that then OOMs during weight load, on a host
+// where the failure looks unrelated to the GPU table.
 var knownModels = map[string]modelInfo{
-	// NVIDIA datacenter
+	// NVIDIA datacenter (compute)
 	"10de:2236": {"NVIDIA A10", 23028},
-	"10de:20b2": {"NVIDIA A100 SXM 40GB", 40960},
-	"10de:20b5": {"NVIDIA A100 SXM 80GB", 81920},
+	"10de:2237": {"NVIDIA A10G", 23028}, // same GA102 silicon/board as A10
+	"10de:2235": {"NVIDIA A40", 45634},
+	"10de:20b0": {"NVIDIA A100 SXM4 40GB", 40960},
+	"10de:20b2": {"NVIDIA A100 SXM4 80GB", 81920},
+	"10de:20b5": {"NVIDIA A100 PCIe 80GB", 81920},
 	"10de:20f1": {"NVIDIA A100 PCIe 40GB", 40960},
-	"10de:20f3": {"NVIDIA A100 PCIe 80GB", 81920},
+	"10de:20f3": {"NVIDIA A800-SXM4-80GB", 81920},
+	"10de:20b7": {"NVIDIA A30 PCIe", 24576},
+	"10de:25b6": {"NVIDIA A2 / A16", 15356}, // shared ID; both SKUs carry 16GB per physical GPU
+	"10de:27b8": {"NVIDIA L4", 23034},
+	"10de:1eb8": {"NVIDIA Tesla T4", 15109},
 	"10de:2330": {"NVIDIA H100 SXM", 81920},
 	"10de:2331": {"NVIDIA H100 PCIe", 81920},
-	"10de:233a": {"NVIDIA H100 NVL", 94208},
+	"10de:2337": {"NVIDIA H100 SXM5 64GB", 65536},
+	"10de:2321": {"NVIDIA H100 NVL 94GB", 94208},
+	"10de:233a": {"NVIDIA H800L 94GB", 94208},
 	"10de:2335": {"NVIDIA H200 SXM", 144384},
-	"10de:2233": {"NVIDIA A30", 24576},
+	"10de:233b": {"NVIDIA H200 NVL", 144384},
 	"10de:26b5": {"NVIDIA L40", 46068},
 	"10de:26b9": {"NVIDIA L40S", 46068},
 	"10de:2bb5": {"NVIDIA RTX Pro 6000 Blackwell Server Edition", 98304},
+	// NVIDIA workstation (display output; not compute, not MIG)
+	"10de:2233": {"NVIDIA RTX A5500", 24576},
+	"10de:2230": {"NVIDIA RTX A6000", 49140},
+	"10de:2231": {"NVIDIA RTX A5000", 24576},
+	"10de:24b0": {"NVIDIA RTX A4000", 16376},
+	"10de:2531": {"NVIDIA RTX A2000", 6138},
+	"10de:2571": {"NVIDIA RTX A2000 12GB", 12288},
+	"10de:25b0": {"NVIDIA RTX A1000", 8188}, // hardware-verified on wattle
+	"10de:26b1": {"NVIDIA RTX 6000 Ada Generation", 49152},
+	"10de:26b2": {"NVIDIA RTX 5000 Ada Generation", 32768},
+	"10de:27b2": {"NVIDIA RTX 4000 Ada Generation", 20480},
 	// NVIDIA consumer
 	"10de:2684": {"NVIDIA GeForce RTX 4090", 24576},
-	"10de:2782": {"NVIDIA GeForce RTX 4070", 12288},
+	"10de:2782": {"NVIDIA GeForce RTX 4070 Ti", 12288},
 	"10de:2204": {"NVIDIA GeForce RTX 3090", 24576},
-	// AMD datacenter
-	"1002:75a0": {"AMD Instinct MI350X", 294896},
-	"1002:7448": {"AMD Instinct MI300X", 196608},
-	"1002:740c": {"AMD Instinct MI250X", 65536},
+	"10de:2203": {"NVIDIA GeForce RTX 3090 Ti", 24576},
+	"10de:2208": {"NVIDIA GeForce RTX 3080 Ti", 12288},
+	"10de:2206": {"NVIDIA GeForce RTX 3080", 10240},
+	"10de:2487": {"NVIDIA GeForce RTX 3060", 12288},
+	"10de:2486": {"NVIDIA GeForce RTX 3060 Ti", 8192},
+	"10de:2484": {"NVIDIA GeForce RTX 3070", 8192},
+	"10de:1e04": {"NVIDIA GeForce RTX 2080 Ti", 11264},
+	// AMD datacenter (compute)
+	"1002:75a0": {"AMD Instinct MI350X", 294896}, // absent from local pci.ids snapshot; left unverified per plan
+	"1002:74a1": {"AMD Instinct MI300X", 196608},
+	"1002:74a0": {"AMD Instinct MI300A", 131072},
+	"1002:740c": {"AMD Instinct MI250X", 65536}, // per-GCD VRAM; card exposes 2 PCI devices, 64GB each
+	"1002:740f": {"AMD Instinct MI210", 65536},
+	"1002:738c": {"AMD Instinct MI100", 32768},
+	// AMD workstation (display output; not compute)
+	"1002:7448": {"AMD Radeon Pro W7900", 49152},
+	"1002:745e": {"AMD Radeon Pro W7800", 32768},
 	// AMD consumer
-	"1002:744c": {"AMD Radeon RX 7900 XTX", 24576},
+	"1002:744c": {"AMD Radeon RX 7900 XT/XTX/GRE/7900M", 16384}, // shared ID spans 16/16/20/24GB SKUs; smallest recorded
 	"1002:73bf": {"AMD Radeon RX 6900 XT", 16384},
 }

@@ -41,37 +85,53 @@ var knownModels = map[string]modelInfo{
 var computeModels = map[string]bool{
 	// NVIDIA datacenter
 	"10de:2236": true, // A10
-	"10de:20b2": true, // A100 SXM 40GB
-	"10de:20b5": true, // A100 SXM 80GB
+	"10de:2237": true, // A10G
+	"10de:2235": true, // A40
+	"10de:20b0": true, // A100 SXM4 40GB
+	"10de:20b2": true, // A100 SXM4 80GB
+	"10de:20b5": true, // A100 PCIe 80GB
 	"10de:20f1": true, // A100 PCIe 40GB
-	"10de:20f3": true, // A100 PCIe 80GB
+	"10de:20f3": true, // A800-SXM4-80GB
+	"10de:20b7": true, // A30 PCIe
+	"10de:25b6": true, // A2 / A16
+	"10de:27b8": true, // L4
+	"10de:1eb8": true, // Tesla T4
 	"10de:2330": true, // H100 SXM
 	"10de:2331": true, // H100 PCIe
-	"10de:233a": true, // H100 NVL
+	"10de:2337": true, // H100 SXM5 64GB
+	"10de:2321": true, // H100 NVL 94GB
+	"10de:233a": true, // H800L 94GB
 	"10de:2335": true, // H200 SXM
-	"10de:2233": true, // A30
+	"10de:233b": true, // H200 NVL
 	"10de:26b5": true, // L40
 	"10de:26b9": true, // L40S
 	"10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition
 	// AMD datacenter
 	"1002:75a0": true, // MI350X
-	"1002:7448": true, // MI300X
+	"1002:74a1": true, // MI300X
+	"1002:74a0": true, // MI300A
 	"1002:740c": true, // MI250X
+	"1002:740f": true, // MI210
+	"1002:738c": true, // MI100
 }

 // migCapableModels is the set of "vendorID:deviceID" keys for NVIDIA GPUs that
 // support MIG partitioning. All are also in computeModels (headless datacenter cards).
-// A10, L4, L40S are compute GPUs but do NOT support MIG.
+// A10, A10G, L4, L40S and T4 are compute GPUs but do NOT support MIG.
 var migCapableModels = map[string]bool{
-	"10de:20b2": true, // A100 SXM 40GB
-	"10de:20b5": true, // A100 SXM 80GB
+	"10de:20b0": true, // A100 SXM4 40GB
+	"10de:20b2": true, // A100 SXM4 80GB
+	"10de:20b5": true, // A100 PCIe 80GB
 	"10de:20f1": true, // A100 PCIe 40GB
-	"10de:20f3": true, // A100 PCIe 80GB
+	"10de:20f3": true, // A800-SXM4-80GB
+	"10de:20b7": true, // A30 PCIe
 	"10de:2330": true, // H100 SXM
 	"10de:2331": true, // H100 PCIe
-	"10de:233a": true, // H100 NVL
+	"10de:2337": true, // H100 SXM5 64GB
+	"10de:2321": true, // H100 NVL 94GB
+	"10de:233a": true, // H800L 94GB
 	"10de:2335": true, // H200 SXM
-	"10de:2233": true, // A30
+	"10de:233b": true, // H200 NVL
 	"10de:2bb5": true, // RTX Pro 6000 Blackwell Server Edition
 }

@@ -23,12 +23,107 @@ func TestLookupModel_RTXPro6000BlackwellSE(t *testing.T) {
 }

 func TestLookupModel_KnownAMD(t *testing.T) {
-	name, mem := lookupModel("1002", "744c")
-	if name != "AMD Radeon RX 7900 XTX" {
-		t.Errorf("name = %q, want AMD Radeon RX 7900 XTX", name)
+	name, mem := lookupModel("1002", "740f")
+	if name != "AMD Instinct MI210" {
+		t.Errorf("name = %q, want AMD Instinct MI210", name)
 	}
-	if mem != 24576 {
-		t.Errorf("memoryMiB = %d, want 24576", mem)
+	if mem != 65536 {
+		t.Errorf("memoryMiB = %d, want 65536", mem)
+	}
+}
+
+// TestLookupModel_AmbiguityRule pins 1002:744c to the smallest of the four
+// Radeon RX 7900 SKUs (16/16/20/24GB) the ID covers, per the table's
+// documented under-report convention for shared PCI IDs.
+func TestLookupModel_AmbiguityRule(t *testing.T) {
+	name, mem := lookupModel("1002", "744c")
+	if name != "AMD Radeon RX 7900 XT/XTX/GRE/7900M" {
+		t.Errorf("name = %q, want AMD Radeon RX 7900 XT/XTX/GRE/7900M", name)
+	}
+	if mem != 16384 {
+		t.Errorf("memoryMiB = %d, want 16384 (smallest covered SKU)", mem)
+	}
+}
+
+// TestLookupModel_CorrectedEntries pins the corrected table entries by name
+// and VRAM, so a future edit cannot silently reintroduce a card labelled as
+// hardware it is not.
+func TestLookupModel_CorrectedEntries(t *testing.T) {
+	tests := []struct {
+		vendorID, deviceID string
+		wantName           string
+		wantMemMiB         int64
+	}{
+		{"1002", "7448", "AMD Radeon Pro W7900", 49152},
+		{"1002", "74a1", "AMD Instinct MI300X", 196608},
+		{"10de", "2233", "NVIDIA RTX A5500", 24576},
+		{"10de", "20b7", "NVIDIA A30 PCIe", 24576},
+		{"10de", "20b2", "NVIDIA A100 SXM4 80GB", 81920},
+		{"10de", "20b0", "NVIDIA A100 SXM4 40GB", 40960},
+		{"10de", "20b5", "NVIDIA A100 PCIe 80GB", 81920},
+		{"10de", "20f3", "NVIDIA A800-SXM4-80GB", 81920},
+		{"10de", "233a", "NVIDIA H800L 94GB", 94208},
+		{"10de", "25b0", "NVIDIA RTX A1000", 8188},
+	}
+	for _, tc := range tests {
+		t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) {
+			name, mem := lookupModel(tc.vendorID, tc.deviceID)
+			if name != tc.wantName {
+				t.Errorf("name = %q, want %q", name, tc.wantName)
+			}
+			if mem != tc.wantMemMiB {
+				t.Errorf("memoryMiB = %d, want %d", mem, tc.wantMemMiB)
+			}
+		})
+	}
+}
+
+// TestModels_CorrectedMembership asserts the corrected entries landed in the
+// right compute/MIG sets: the wattle A1000 and the newly-real A30 are
+// compute+MIG where applicable, while the corrected W7900/A5500/RX7900 stay
+// out of both since they're display-capable workstation/consumer cards.
+func TestModels_CorrectedMembership(t *testing.T) {
+	tests := []struct {
+		vendorID, deviceID   string
+		wantCompute, wantMIG bool
+	}{
+		{"1002", "7448", false, false}, // W7900: workstation, has display
+		{"1002", "74a1", true, false},  // MI300X: compute, AMD has no MIG
+		{"10de", "2233", false, false}, // A5500: workstation, has display
+		{"10de", "20b7", true, true},   // A30 PCIe: compute + MIG-capable
+		{"10de", "20b2", true, true},   // A100 SXM4 80GB: compute + MIG-capable
+		{"10de", "25b0", false, false}, // A1000 (wattle): workstation, has display
+		{"1002", "744c", false, false}, // RX 7900 family: consumer, has display
+	}
+	for _, tc := range tests {
+		t.Run(tc.vendorID+":"+tc.deviceID, func(t *testing.T) {
+			if got := IsComputeGPU(tc.vendorID, tc.deviceID); got != tc.wantCompute {
+				t.Errorf("IsComputeGPU(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantCompute)
+			}
+			if got := IsMIGCapable(tc.vendorID, tc.deviceID); got != tc.wantMIG {
+				t.Errorf("IsMIGCapable(%s,%s) = %v, want %v", tc.vendorID, tc.deviceID, got, tc.wantMIG)
+			}
+		})
+	}
+}
+
+// TestModels_NonMIGCapableCompute pins the plan's explicit exception list:
+// A10, A10G, L4, L40S and T4 are compute GPUs but do not support MIG.
+func TestModels_NonMIGCapableCompute(t *testing.T) {
+	nonMIG := []struct{ vendorID, deviceID string }{
+		{"10de", "2236"}, // A10
+		{"10de", "2237"}, // A10G
+		{"10de", "27b8"}, // L4
+		{"10de", "26b9"}, // L40S
+		{"10de", "1eb8"}, // T4
+	}
+	for _, tc := range nonMIG {
+		if !IsComputeGPU(tc.vendorID, tc.deviceID) {
+			t.Errorf("%s:%s expected to be a compute GPU", tc.vendorID, tc.deviceID)
+		}
+		if IsMIGCapable(tc.vendorID, tc.deviceID) {
+			t.Errorf("%s:%s expected NOT to be MIG-capable", tc.vendorID, tc.deviceID)
+		}
 	}
 }