mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-22 20:26:44 +00:00
fix(gpu): Blackwell GPU support fixes for EKS/ECS image builds and VFIO (#559)
* fix(eks): GPU node AMI declares uefi boot mode The spinifex-eks-node-gpu catalog entry declared BootMode: bios, but the Ubuntu 26.04 GPU appliance image is built GPT/EFI-only. Under legacy BIOS no OVMF/pflash firmware is attached, so the guest hangs at firmware with zero serial console output. * fix(images): install open-kernel NVIDIA DKMS driver for Blackwell GPUs Blackwell-generation GPUs (e.g. RTX Pro 6000 Blackwell) refuse to initialize under the closed/proprietary kernel module: dmesg shows RmInitAdapter failing with "requires use of the NVIDIA open kernel modules" (NVRM error 0x22). Switch from nvidia-dkms-<ver>-server to nvidia-dkms-<ver>-server-open in both the shared ubuntu-gpu-nvidia recipe and its verbatim duplicate in eks-node-gpu. nvidia-utils has no separate open variant (userspace tools only), so it is unchanged. * fix(eks): tolerate CP and GPU taints in ebs-csi-controller/lbc addons A cluster with a GPU-only nodegroup has no untainted node: the CP carries CriticalAddonsOnly (EKS parity keeps user workloads off it) and every worker carries nvidia.com/gpu. Without tolerations for both, ebs-csi-controller and aws-load-balancer-controller have nowhere to schedule and stay Pending forever. Real EKS never needs this since its control plane isn't a schedulable node, so the stock upstream manifests carry no such toleration. * fix(images): give the libguestfs appliance a working resolver The guest's /etc/resolv.conf symlinks to systemd-resolved's stub (127.0.0.53), which isn't running inside the appliance chroot, so DNS is dead until package installs need it. Point it at a real resolver before package installs, then restore the stub symlink afterward so the shipped AMI still gets its DNS from systemd-resolved at boot like a stock cloud image. * fix(gpu): skip upstream PCIe bridge devices during VFIO bind/unbind A GPU's IOMMU group can include an upstream PCIe bridge when the platform lacks ACS isolation between them. vfio-pci refuses to bind non-endpoint bridge devices, so Claim/ReclaimByAddress must filter bridge-class (0x06) members out of the group before touching driver bindings, rather than failing the whole claim or leaving the bridge rebound incorrectly. * fix(ecs): GPU node AMI declares uefi boot mode The spinifex-ecs-node-gpu catalog entry declared BootMode: bios, but the Ubuntu 26.04 GPU appliance image is built GPT/EFI-only. Under legacy BIOS no OVMF/pflash firmware is attached, so the guest hangs at firmware with zero serial console output. Same defect and same fix as the eks-node-gpu entry: both images are built by the same script from the same stock Ubuntu cloudimg, so both were EFI-only and only one was corrected. * fix(images): open-kernel NVIDIA module on ecs-agent-gpu, pin driver branch ecs-agent-gpu still installed nvidia-dkms-<N>-server, the proprietary kernel module. NVIDIA ships no proprietary module for Blackwell (GB202) and later, so ECS GPU container-instances could not bind those cards on any driver branch. It is the third verbatim copy of the routine and was missed when the other two were fixed. Also pins the driver branch across all three copies. The branch was resolved at build time via apt-cache search | sort -rn | head -1, which sorts on the number in the package name — but that number is a branch alias, not a driver version (nvidia-dkms-535-server installs 580.126.20), so the ordering key need not match what installs. Resolving also let identical source produce different drivers on different days. The pin lives in setup.sh rather than manifest.conf because manifest variables are not passed into the guest: the setup script is uploaded and executed by virt-customize --run, with no environment carried across. * fix(eks): tolerate CP and GPU taints in argocd addon All six argocd Deployments and its StatefulSet carried no tolerations. A cluster with a GPU-only nodegroup has no untainted node to place them on: the control plane carries CriticalAddonsOnly and every worker carries nvidia.com/gpu, so the pods stay Pending forever. Same defect the ebs-csi-controller and load-balancer-controller addons already had. Real EKS never hits it because its control plane is not a schedulable node, so the stock upstream manifests carry no such toleration and the divergence is ours to maintain. --------- Co-authored-by: joshsiv-mulga <mrk@mulgadc.com>
This commit is contained in:
tomnewton-mulga
@@ -265,6 +265,15 @@ else
|
||||
CUST+=(--memsize 2048)
|
||||
fi
|
||||
|
||||
# The guest's /etc/resolv.conf symlinks to systemd-resolved's stub
|
||||
# (127.0.0.53), which isn't running inside the appliance chroot, so DNS is
|
||||
# dead until package installs need it. Point it at a real resolver first, then
|
||||
# restore the stub symlink after package installs so the shipped AMI still
|
||||
# gets its DNS from systemd-resolved at boot like a stock cloud image.
|
||||
if [[ -n "${APK_PACKAGES:-}${APT_PACKAGES:-}" ]]; then
|
||||
CUST+=(--run-command 'rm -f /etc/resolv.conf; printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolv.conf')
|
||||
fi
|
||||
|
||||
# Packages
|
||||
if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
|
||||
echo "Will install packages: ${APK_PACKAGES}"
|
||||
@@ -338,6 +347,12 @@ else
|
||||
CUST+=(--run-command 'export DEBIAN_FRONTEND=noninteractive; apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/* 2>/dev/null || true')
|
||||
fi
|
||||
|
||||
# Restore the stock systemd-resolved stub symlink so the shipped AMI gets its
|
||||
# DNS from the instance's own resolver at boot, not the static appliance one.
|
||||
if [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APK_PACKAGES:-}${APT_PACKAGES:-}${SETUP_SCRIPT:-}" ]]; then
|
||||
CUST+=(--run-command 'rm -f /etc/resolv.conf; ln -sf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf')
|
||||
fi
|
||||
|
||||
echo "Customizing image (libguestfs appliance)..."
|
||||
"${CUST[@]}"
|
||||
|
||||
|
||||
@@ -71,18 +71,21 @@ EOF
|
||||
|
||||
apt-get install -y --no-install-recommends "linux-headers-${KVER}"
|
||||
|
||||
# Ubuntu 26.04+ no longer ships unversioned meta-packages; detect the
|
||||
# latest available versioned NVIDIA server driver.
|
||||
NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
|
||||
| grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
|
||||
if [ -z "${NVIDIA_VER}" ]; then
|
||||
echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
|
||||
apt-cache search nvidia-dkms || true
|
||||
exit 1
|
||||
fi
|
||||
echo "[nvidia-gpu-stack] installing NVIDIA server driver version: ${NVIDIA_VER}"
|
||||
# Ubuntu 26.04+ no longer ships unversioned meta-packages, so the branch is
|
||||
# named explicitly. Pinned rather than resolved at build time: the number in
|
||||
# the package name is a branch alias, not a driver version (nvidia-dkms-535-server
|
||||
# installs 580.126.20), so picking the highest number sorts on a value that
|
||||
# need not match what gets installed. Resolving also let identical source
|
||||
# produce different drivers on different days. Bumping is a deliberate edit.
|
||||
NVIDIA_VER=595
|
||||
echo "[nvidia-gpu-stack] installing NVIDIA server driver branch: ${NVIDIA_VER}"
|
||||
# The -open (GSP/open-kernel-module) DKMS variant, not the closed/proprietary
|
||||
# one: Blackwell-generation GPUs (e.g. RTX Pro 6000 Blackwell) refuse to
|
||||
# initialize under the closed kernel module — RmInitAdapter fails with
|
||||
# "requires use of the NVIDIA open kernel modules" (dmesg NVRM error 0x22).
|
||||
# nvidia-utils has no separate open variant (userspace tools only).
|
||||
apt-get install -y --no-install-recommends \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server" \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server-open" \
|
||||
"nvidia-utils-${NVIDIA_VER}-server"
|
||||
|
||||
NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
|
||||
|
||||
@@ -90,18 +90,21 @@ EOF
|
||||
|
||||
apt-get install -y --no-install-recommends "linux-headers-${KVER}"
|
||||
|
||||
# Ubuntu 26.04+ no longer ships unversioned meta-packages; detect the
|
||||
# latest available versioned NVIDIA server driver.
|
||||
NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
|
||||
| grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
|
||||
if [ -z "${NVIDIA_VER}" ]; then
|
||||
echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
|
||||
apt-cache search nvidia-dkms || true
|
||||
exit 1
|
||||
fi
|
||||
echo "[nvidia-gpu-stack] installing NVIDIA server driver version: ${NVIDIA_VER}"
|
||||
# Ubuntu 26.04+ no longer ships unversioned meta-packages, so the branch is
|
||||
# named explicitly. Pinned rather than resolved at build time: the number in
|
||||
# the package name is a branch alias, not a driver version (nvidia-dkms-535-server
|
||||
# installs 580.126.20), so picking the highest number sorts on a value that
|
||||
# need not match what gets installed. Resolving also let identical source
|
||||
# produce different drivers on different days. Bumping is a deliberate edit.
|
||||
NVIDIA_VER=595
|
||||
echo "[nvidia-gpu-stack] installing NVIDIA server driver branch: ${NVIDIA_VER}"
|
||||
# The -open (GSP/open-kernel-module) DKMS variant, not the closed/proprietary
|
||||
# one: Blackwell-generation GPUs (e.g. RTX Pro 6000 Blackwell) refuse to
|
||||
# initialize under the closed kernel module — RmInitAdapter fails with
|
||||
# "requires use of the NVIDIA open kernel modules" (dmesg NVRM error 0x22).
|
||||
# nvidia-utils has no separate open variant (userspace tools only).
|
||||
apt-get install -y --no-install-recommends \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server" \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server-open" \
|
||||
"nvidia-utils-${NVIDIA_VER}-server"
|
||||
|
||||
NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
|
||||
|
||||
@@ -24939,6 +24939,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-applicationset-controller
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- args:
|
||||
- /usr/local/bin/argocd-applicationset-controller
|
||||
@@ -25180,6 +25191,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-dex-server
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
@@ -25304,6 +25326,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-notifications-controller
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- args:
|
||||
- /usr/local/bin/argocd-notifications
|
||||
@@ -25403,6 +25436,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-redis
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
@@ -25486,6 +25530,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-repo-server
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
@@ -25840,6 +25895,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-server
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
@@ -26251,6 +26317,17 @@ spec:
|
||||
labels:
|
||||
app.kubernetes.io/name: argocd-application-controller
|
||||
spec:
|
||||
# Local delta over the vendored upstream manifest: a cluster with a
|
||||
# GPU-only nodegroup has no untainted node — the CP carries
|
||||
# CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
|
||||
# these the pod stays Pending forever. Upstream never needs it because
|
||||
# real EKS control planes are not schedulable nodes.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
|
||||
@@ -44,6 +44,18 @@ spec:
|
||||
kubernetes.io/os: linux
|
||||
serviceAccountName: ebs-csi-controller-sa
|
||||
priorityClassName: system-cluster-critical
|
||||
# A cluster with a GPU-only nodegroup has no untainted node: the CP carries
|
||||
# CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
|
||||
# worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
|
||||
# tolerations the controller has nowhere to schedule and stays Pending
|
||||
# forever. Real EKS never needs this: its control plane isn't a schedulable
|
||||
# node, so the stock upstream manifest carries no such toleration.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
|
||||
+12
@@ -348,6 +348,18 @@ spec:
|
||||
readOnly: true
|
||||
{{IRSA_VOLUME_MOUNT}}
|
||||
priorityClassName: system-cluster-critical
|
||||
# A cluster with a GPU-only nodegroup has no untainted node: the CP carries
|
||||
# CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
|
||||
# worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
|
||||
# tolerations the controller has nowhere to schedule and stays Pending
|
||||
# forever. Real EKS never needs this: its control plane isn't a schedulable
|
||||
# node, so the stock upstream manifest carries no such toleration.
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
securityContext:
|
||||
fsGroup: 1337
|
||||
serviceAccountName: aws-load-balancer-controller
|
||||
|
||||
@@ -60,19 +60,22 @@ EOF
|
||||
apt-get install -y --no-install-recommends \
|
||||
"linux-headers-${KVER}"
|
||||
|
||||
# Detect the latest available versioned NVIDIA server driver. Ubuntu 26.04+
|
||||
# no longer ships unversioned meta-packages (nvidia-dkms-server); the packages
|
||||
# are now versioned (e.g. nvidia-dkms-570-server).
|
||||
NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
|
||||
| grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
|
||||
if [[ -z "${NVIDIA_VER}" ]]; then
|
||||
echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
|
||||
apt-cache search nvidia-dkms || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Installing NVIDIA server driver version: ${NVIDIA_VER}"
|
||||
# Ubuntu 26.04+ no longer ships unversioned meta-packages (nvidia-dkms-server);
|
||||
# the packages are versioned (e.g. nvidia-dkms-570-server), so the branch is
|
||||
# named explicitly. Pinned rather than resolved at build time: the number in the
|
||||
# package name is a branch alias, not a driver version (nvidia-dkms-535-server
|
||||
# installs 580.126.20), so picking the highest number sorts on a value that need
|
||||
# not match what gets installed. Resolving also let identical source produce
|
||||
# different drivers on different days. Bumping is a deliberate edit.
|
||||
NVIDIA_VER=595
|
||||
echo "Installing NVIDIA server driver branch: ${NVIDIA_VER}"
|
||||
# The -open (GSP/open-kernel-module) DKMS variant, not the closed/proprietary
|
||||
# one: Blackwell-generation GPUs (e.g. RTX Pro 6000 Blackwell) refuse to
|
||||
# initialize under the closed kernel module — RmInitAdapter fails with
|
||||
# "requires use of the NVIDIA open kernel modules" (dmesg NVRM error 0x22).
|
||||
# nvidia-utils has no separate open variant (userspace tools only).
|
||||
apt-get install -y --no-install-recommends \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server" \
|
||||
"nvidia-dkms-${NVIDIA_VER}-server-open" \
|
||||
"nvidia-utils-${NVIDIA_VER}-server"
|
||||
|
||||
# Detect the installed NVIDIA DKMS module name + version for explicit build.
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// groupMembers returns all PCI devices in the given IOMMU group.
|
||||
@@ -40,3 +41,26 @@ func groupMembers(sysfsRoot string, groupNum int) ([]IOMMUGroupMember, error) {
|
||||
|
||||
return members, nil
|
||||
}
|
||||
|
||||
// isBridgeClass reports whether a PCI class string identifies a bridge device
|
||||
// (base class 0x06 — host, ISA, PCI-to-PCI, etc). A passthrough endpoint's
|
||||
// IOMMU group can include an upstream bridge when the platform lacks ACS
|
||||
// isolation between them; the bridge itself never performs DMA and vfio-pci
|
||||
// refuses to bind non-endpoint devices, so it must stay off the VFIO
|
||||
// bind/unbind lifecycle while the endpoint is claimed.
|
||||
func isBridgeClass(class string) bool {
|
||||
s := strings.ToLower(strings.TrimSpace(class))
|
||||
s = strings.TrimPrefix(s, "0x")
|
||||
return len(s) >= 2 && s[:2] == "06"
|
||||
}
|
||||
|
||||
// filterBridgeMembers returns members with bridge-class devices removed.
|
||||
func filterBridgeMembers(members []IOMMUGroupMember) []IOMMUGroupMember {
|
||||
out := make([]IOMMUGroupMember, 0, len(members))
|
||||
for _, m := range members {
|
||||
if !isBridgeClass(m.Class) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -70,3 +70,38 @@ func TestGroupMembers_MissingGroup(t *testing.T) {
|
||||
t.Error("want error for non-existent IOMMU group, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBridgeClass(t *testing.T) {
|
||||
cases := []struct {
|
||||
class string
|
||||
want bool
|
||||
}{
|
||||
{"0x060400", true}, // PCI-to-PCI bridge
|
||||
{"0x060000", true}, // host bridge
|
||||
{"0x030200", false}, // 3D controller (GPU)
|
||||
{"0x040300", false}, // audio device
|
||||
{"", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isBridgeClass(c.class); got != c.want {
|
||||
t.Errorf("isBridgeClass(%q) = %v, want %v", c.class, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterBridgeMembers(t *testing.T) {
|
||||
members := []IOMMUGroupMember{
|
||||
{PCIAddress: "0000:00:1c.0", Class: "0x060400"},
|
||||
{PCIAddress: "0000:03:00.0", Class: "0x030200"},
|
||||
{PCIAddress: "0000:03:00.1", Class: "0x040300"},
|
||||
}
|
||||
got := filterBridgeMembers(members)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 members after filtering bridge, got %d", len(got))
|
||||
}
|
||||
for _, m := range got {
|
||||
if m.PCIAddress == "0000:00:1c.0" {
|
||||
t.Error("bridge member 0000:00:1c.0 should have been filtered out")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,6 +167,7 @@ func (m *Manager) Claim(instanceID, profileName string) (*GPUDevice, *MIGInstanc
|
||||
m.mu.Unlock()
|
||||
return nil, nil, fmt.Errorf("enumerate IOMMU group %d: %w", entry.Device.IOMMUGroup, err)
|
||||
}
|
||||
members = filterBridgeMembers(members)
|
||||
|
||||
drivers := make(map[string]string, len(members))
|
||||
var bound []string
|
||||
@@ -471,6 +472,7 @@ func (m *Manager) ReclaimByAddress(addr, instanceID string) error {
|
||||
"gpu", addr, "group", entry.Device.IOMMUGroup, "err", err)
|
||||
members = []IOMMUGroupMember{{PCIAddress: addr}}
|
||||
}
|
||||
members = filterBridgeMembers(members)
|
||||
|
||||
// Use the stored pre-passthrough driver for the primary GPU so Release
|
||||
// can rebind correctly. Companion devices are left unbound on release
|
||||
|
||||
@@ -66,6 +66,39 @@ func TestManagerClaim_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestManagerClaim_SkipsBridgeGroupMember reproduces a host topology where a
|
||||
// GPU shares its IOMMU group with an upstream PCIe bridge (no ACS isolation
|
||||
// between them). vfio-pci refuses to bind bridge-class devices, so Claim must
|
||||
// skip the bridge entirely rather than fail the whole claim, and Release must
|
||||
// leave the bridge on its original driver afterward.
|
||||
func TestManagerClaim_SkipsBridgeGroupMember(t *testing.T) {
|
||||
root, gpu := buildManagerSysfs(t)
|
||||
// Upstream PCIe bridge in the same IOMMU group as the GPU (0x0604 = PCI-to-PCI bridge).
|
||||
buildSysfsDevice(t, root, "0000:00:1c.0", "0x060400", "0x8086", "0x0db0", "pcieport", 7)
|
||||
makeSysfsDriverDir(t, root, "pcieport")
|
||||
|
||||
m := newManagerForTest([]GPUDevice{gpu}, root)
|
||||
|
||||
if _, _, err := m.Claim("i-001", ""); err != nil {
|
||||
t.Fatalf("Claim: %v", err)
|
||||
}
|
||||
|
||||
bridgeDriver, err := os.Readlink(filepath.Join(root, "bus/pci/devices/0000:00:1c.0/driver"))
|
||||
if err != nil {
|
||||
t.Fatalf("read bridge driver link: %v", err)
|
||||
}
|
||||
if filepath.Base(bridgeDriver) != "pcieport" {
|
||||
t.Errorf("bridge driver = %q, want pcieport (must not be rebound to vfio-pci)", filepath.Base(bridgeDriver))
|
||||
}
|
||||
|
||||
if err := m.Release("i-001"); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
if m.Available() != 1 {
|
||||
t.Errorf("want Available=1 after release, got %d", m.Available())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerClaim_NoGPUAvailable(t *testing.T) {
|
||||
root, gpu := buildManagerSysfs(t)
|
||||
m := newManagerForTest([]GPUDevice{gpu}, root)
|
||||
|
||||
@@ -449,8 +449,11 @@ var AvailableImages = map[string]Images{
|
||||
URL: "https://iso.mulgadc.com/system-ami/spinifex-eks-node-gpu-x86_64.qcow2",
|
||||
Checksum: "https://iso.mulgadc.com/system-ami/spinifex-eks-node-gpu-x86_64.qcow2.sha256",
|
||||
ChecksumType: "sha256",
|
||||
BootMode: "bios",
|
||||
Tags: map[string]string{"spinifex:managed-by": "eks", "gpu-vendor": "nvidia"},
|
||||
// Ubuntu 26.04 is built GPT/EFI-only; legacy BIOS finds no bootable MBR
|
||||
// bootloader and hangs at firmware with zero serial output (no OVMF/pflash
|
||||
// is attached unless BootMode is uefi/uefi-preferred, see vm/lifecycle.go).
|
||||
BootMode: "uefi",
|
||||
Tags: map[string]string{"spinifex:managed-by": "eks", "gpu-vendor": "nvidia"},
|
||||
},
|
||||
|
||||
// ECS container-instance system AMI. Resolved by spinifex:managed-by=ecs tag —
|
||||
@@ -483,8 +486,11 @@ var AvailableImages = map[string]Images{
|
||||
URL: "https://iso.mulgadc.com/system-ami/spinifex-ecs-node-gpu-x86_64.qcow2",
|
||||
Checksum: "https://iso.mulgadc.com/system-ami/spinifex-ecs-node-gpu-x86_64.qcow2.sha256",
|
||||
ChecksumType: "sha256",
|
||||
BootMode: "bios",
|
||||
Tags: map[string]string{"spinifex:managed-by": "ecs", "gpu-vendor": "nvidia"},
|
||||
// Ubuntu 26.04 is built GPT/EFI-only; legacy BIOS finds no bootable MBR
|
||||
// bootloader and hangs at firmware with zero serial output (no OVMF/pflash
|
||||
// is attached unless BootMode is uefi/uefi-preferred, see vm/lifecycle.go).
|
||||
BootMode: "uefi",
|
||||
Tags: map[string]string{"spinifex:managed-by": "ecs", "gpu-vendor": "nvidia"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user