PCIe NUMA topology: support cross-NUMA device placement

On NVIDIA GB200 Grace Blackwell systems, GPUs are on separate NUMA
nodes from all CPUs by design. A 2-superchip GB200 exposes:

  NUMA 0:  Grace CPU kubevirt#1 (CPUs 0-71, ~490 GB LPDDR5X)
  NUMA 1:  Grace CPU kubevirt#2 (CPUs 72-143, ~490 GB LPDDR5X)
  NUMA 2:  Blackwell GPU kubevirt#1 (no CPUs, ~188 GB HBM)
  NUMA 10: Blackwell GPU kubevirt#2 (no CPUs, ~188 GB HBM)

The existing PCIe NUMA-aware topology code discovers the host NUMA node
for each passthrough device but discards it when no vCPUs are pinned to
that node, causing the device to fall back to the default pci.0 bus
without NUMA affinity information.

Extend LookupDevicesNumaNodes to return both aligned devices (host NUMA
node has pinned vCPUs) and unaligned devices (host NUMA node has no
pinned vCPUs but is a valid NUMA node). When the expander bus assigner
encounters an unaligned device, it creates a CPU-less guest NUMA cell
for the device's host NUMA node and places the device under a
pcie-expander-bus targeting that cell.

On a GB200 each GPU naturally gets its own expander bus and SMMUv3 IOMMU
device through the standard per-NUMA topology path, since each GPU is
on its own NUMA node.

Also adds an explicit check for NUMA node -1 in GetDeviceNumaNode to
avoid relying on uint32 overflow for devices without NUMA affinity.

Ref: https://docs.nvidia.com/dccpu/grace-perf-tuning-guide/system.html

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Lee Yarwood <lyarwood@redhat.com>

Author: lyarwood

pkg/util/hardware/hw_utils.go

@@ -197,11 +197,18 @@ func PCIAddressToString(pciBusID *api.Address) string {
 // LookupDevicesNumaNodes looks up the NUMA nodes of multiple devices based on
 // their PCI addresses and the domain spec of the virtual machine.
 //
-// It returns a map of PCI addresses to their corresponding NUMA node IDs.
-func LookupDevicesNumaNodes(pciAddresses []string, domainSpec *api.DomainSpec) map[string]uint32 {
-	results := make(map[string]uint32)
+// It returns two maps:
+//   - aligned: PCI addresses mapped to guest NUMA cell IDs where the device's
+//     host NUMA node has vCPUs pinned to it.
+//   - unaligned: PCI addresses mapped to host NUMA node IDs where the device's
+//     host NUMA node has no pinned vCPUs. The caller can use this to create
+//     CPU-less guest NUMA cells for cross-NUMA device placement (e.g., NVIDIA
+//     GB200 where the GPU is on a separate NUMA node from all CPUs).
+func LookupDevicesNumaNodes(pciAddresses []string, domainSpec *api.DomainSpec) (aligned map[string]uint32, unaligned map[string]uint32) {
+	aligned = make(map[string]uint32)
+	unaligned = make(map[string]uint32)
 	if len(pciAddresses) == 0 || domainSpec == nil || domainSpec.CPU.NUMA == nil || domainSpec.CPUTune == nil {
-		return results
+		return
 	}
 
 	// pcpu -> vcpu mapping
@@ -242,28 +249,46 @@ func LookupDevicesNumaNodes(pciAddresses []string, domainSpec *api.DomainSpec) m
 		if err != nil {
 			continue
 		}
+
+		// Skip devices that report no NUMA affinity (-1 wraps to max uint32)
+		if int32(*deviceNuma) == -1 {
+			continue
+		}
+
 		var pcpus []uint32
 		// if another device is already on this NUMA node, use the same pcpu set
 		if res, exists := pNumaToPCPUSetMap[*deviceNuma]; exists {
 			pcpus = res
 		} else {
 			pcpusNuma, err := GetNumaNodeCPUList(int(*deviceNuma))
 			if err != nil {
-				continue
-			}
-			for _, pcpu := range pcpusNuma {
-				pcpus = append(pcpus, uint32(pcpu))
+				// NUMA node may have no CPUs (e.g., GPU-only NUMA node)
+				pNumaToPCPUSetMap[*deviceNuma] = nil
+				pcpus = nil
+			} else {
+				for _, pcpu := range pcpusNuma {
+					pcpus = append(pcpus, uint32(pcpu))
+				}
+				pNumaToPCPUSetMap[*deviceNuma] = pcpus
 			}
-			pNumaToPCPUSetMap[*deviceNuma] = pcpus
 		}
 
+		found := false
 		for _, pcpu := range pcpus {
 			if vCPU, exist := p2vCPUMap[pcpu]; exist {
 				cellID := vCPUToCellMap[vCPU]
-				results[pciAddress] = cellID
+				aligned[pciAddress] = cellID
+				found = true
 				break
 			}
 		}
+
+		// Device is on a host NUMA node with no pinned vCPUs.
+		// Record the host NUMA node so the caller can create a
+		// CPU-less guest NUMA cell for it.
+		if !found {
+			unaligned[pciAddress] = *deviceNuma
+		}
 	}
-	return results
+	return
 }

pkg/util/hardware/hw_utils_test.go

@@ -34,7 +34,8 @@ import (
 
 var _ = Describe("Hardware utils test", func() {
 	const (
-		testPCIAddress = "0000:00:01.0"
+		testPCIAddress  = "0000:00:01.0"
+		testPCIAddress2 = "0000:00:02.0"
 	)
 
 	var (
@@ -53,7 +54,7 @@ var _ = Describe("Hardware utils test", func() {
 		fakeNodeBasePath, err = os.MkdirTemp("", "numa_nodes")
 		Expect(err).ToNot(HaveOccurred())
 
-		// Create test PCI device with NUMA node
+		// Create test PCI device on NUMA node 0
 		pciDevicePath := filepath.Join(fakePciBasePath, testPCIAddress)
 		err = os.MkdirAll(pciDevicePath, 0o755)
 		Expect(err).ToNot(HaveOccurred())
@@ -62,6 +63,15 @@ var _ = Describe("Hardware utils test", func() {
 		err = os.WriteFile(numaNodeFile, []byte("0\n"), 0o644)
 		Expect(err).ToNot(HaveOccurred())
 
+		// Create test PCI device on NUMA node 1
+		pciDevicePath2 := filepath.Join(fakePciBasePath, testPCIAddress2)
+		err = os.MkdirAll(pciDevicePath2, 0o755)
+		Expect(err).ToNot(HaveOccurred())
+
+		numaNodeFile2 := filepath.Join(pciDevicePath2, "numa_node")
+		err = os.WriteFile(numaNodeFile2, []byte("1\n"), 0o644)
+		Expect(err).ToNot(HaveOccurred())
+
 		// Create NUMA node 0 with cpulist
 		numaNode0Path := filepath.Join(fakeNodeBasePath, "node0")
 		err = os.MkdirAll(numaNode0Path, 0o755)
@@ -277,26 +287,29 @@ var _ = Describe("Hardware utils test", func() {
 	})
 
 	Context("devices NUMA affinity", func() {
-		It("should return an empty result for no PCI addresses", func() {
+		It("should return empty results for no PCI addresses", func() {
 			domainSpec := &api.DomainSpec{}
-			devicesNumaNodes := LookupDevicesNumaNodes([]string{}, domainSpec)
-			Expect(devicesNumaNodes).To(BeEmpty())
+			aligned, unaligned := LookupDevicesNumaNodes([]string{}, domainSpec)
+			Expect(aligned).To(BeEmpty())
+			Expect(unaligned).To(BeEmpty())
 		})
 
-		It("should return an empty result for nil domain spec", func() {
-			devicesNumaNodes := LookupDevicesNumaNodes([]string{testPCIAddress}, nil)
-			Expect(devicesNumaNodes).To(BeEmpty())
+		It("should return empty results for nil domain spec", func() {
+			aligned, unaligned := LookupDevicesNumaNodes([]string{testPCIAddress}, nil)
+			Expect(aligned).To(BeEmpty())
+			Expect(unaligned).To(BeEmpty())
 		})
 
-		It("should return an empty result when domain spec has no NUMA info", func() {
+		It("should return empty results when domain spec has no NUMA info", func() {
 			domainSpec := &api.DomainSpec{
 				CPU: api.CPU{},
 			}
-			devicesNumaNodes := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
-			Expect(devicesNumaNodes).To(BeEmpty())
+			aligned, unaligned := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
+			Expect(aligned).To(BeEmpty())
+			Expect(unaligned).To(BeEmpty())
 		})
 
-		It("should handle domain spec with NUMA cells but no vCPU affinity", func() {
+		It("should return device as unaligned when NUMA cells exist but no vCPU affinity", func() {
 			domainSpec := &api.DomainSpec{
 				CPU: api.CPU{
 					NUMA: &api.NUMA{
@@ -311,8 +324,12 @@ var _ = Describe("Hardware utils test", func() {
 				},
 			}
 
-			devicesNumaNodes := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
-			Expect(devicesNumaNodes).To(BeEmpty())
+			// Device is on NUMA 0 but no vCPUs are pinned anywhere,
+			// so it should be reported as unaligned with its host NUMA node
+			aligned, unaligned := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
+			Expect(aligned).To(BeEmpty())
+			Expect(unaligned).To(HaveKey(testPCIAddress))
+			Expect(unaligned[testPCIAddress]).To(Equal(uint32(0)))
 		})
 
 		It("should return devices vCPU NUMA nodes for their aligned vCPUs", func() {
@@ -334,10 +351,40 @@ var _ = Describe("Hardware utils test", func() {
 			}
 
 			// Device is on host NUMA node 0, and vCPU 0 is on guest NUMA cell 0
-			devicesNumaNodes := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
-			Expect(devicesNumaNodes).ToNot(BeEmpty())
-			Expect(devicesNumaNodes).To(HaveKey(testPCIAddress))
-			Expect(devicesNumaNodes[testPCIAddress]).To(Equal(uint32(0)))
+			aligned, unaligned := LookupDevicesNumaNodes([]string{testPCIAddress}, domainSpec)
+			Expect(aligned).ToNot(BeEmpty())
+			Expect(aligned).To(HaveKey(testPCIAddress))
+			Expect(aligned[testPCIAddress]).To(Equal(uint32(0)))
+			Expect(unaligned).To(BeEmpty())
+		})
+
+		It("should return unaligned devices when device is on NUMA node without vCPUs", func() {
+			// Simulate GB200: vCPUs on NUMA 0, device on NUMA 1
+			domainSpec := &api.DomainSpec{
+				CPU: api.CPU{
+					NUMA: &api.NUMA{
+						Cells: []api.NUMACell{
+							{ID: "0", CPUs: "0-1", Memory: pointer.P(uint64(2048)), Unit: "MiB"},
+						},
+					},
+				},
+				CPUTune: &api.CPUTune{
+					VCPUPin: []api.CPUTuneVCPUPin{
+						{VCPU: 0, CPUSet: "0"},
+						{VCPU: 1, CPUSet: "1"},
+					},
+				},
+			}
+
+			// testPCIAddress (0000:00:01.0) is on NUMA node 0,
+			// use a device on NUMA node 1
+			aligned, unaligned := LookupDevicesNumaNodes([]string{testPCIAddress, testPCIAddress2}, domainSpec)
+			// testPCIAddress is on NUMA 0 with vCPUs -> aligned
+			Expect(aligned).To(HaveKey(testPCIAddress))
+			Expect(aligned[testPCIAddress]).To(Equal(uint32(0)))
+			// testPCIAddress2 is on NUMA 1 with no vCPUs -> unaligned
+			Expect(unaligned).To(HaveKey(testPCIAddress2))
+			Expect(unaligned[testPCIAddress2]).To(Equal(uint32(1)))
 		})
 	})
 })

pkg/virt-launcher/virtwrap/converter/pci-placement.go

@@ -191,6 +191,11 @@ type expanderBusAssigner struct {
 	devices          map[string]*api.HostDevice
 	devicesNUMANodes map[string]uint32
 
+	// hostNUMAToCellMap tracks CPU-less guest NUMA cells created for host
+	// NUMA nodes that have passthrough devices but no pinned vCPUs (e.g.,
+	// NVIDIA GB200 where the GPU is on a separate NUMA node from all CPUs).
+	hostNUMAToCellMap map[uint32]uint32
+
 	// lastAssignedBusNr tracks the last assigned bus number for expander buses.
 	// It starts from maxExpanderBusNr and decreases as expander buses are assigned
 	// to ensure controller indices don't conflict with expander bus number space.
@@ -222,6 +227,7 @@ func newExpanderBusAssigner(domainSpec *api.DomainSpec, iommupci *iommupci.Iommu
 		topologyMap:       make(map[uint32]*numaAwareTopology),
 		devices:           make(map[string]*api.HostDevice),
 		devicesNUMANodes:  make(map[string]uint32),
+		hostNUMAToCellMap: make(map[uint32]uint32),
 		controllerIndex:   currentControllerIndex,
 		controllerCount:   0,
 		lastAssignedBusNr: maxExpanderBusNr,
@@ -286,18 +292,67 @@ func (a *expanderBusAssigner) addDevices(devices []api.HostDevice) {
 		devicesByAddress[address] = &devices[i]
 	}
 
-	numaNodes := hardware.LookupDevicesNumaNodes(pciAddresses, a.domainSpec)
+	aligned, unaligned := hardware.LookupDevicesNumaNodes(pciAddresses, a.domainSpec)
 
 	for address, device := range devicesByAddress {
-		if numaNode, exists := numaNodes[address]; exists {
+		if numaNode, exists := aligned[address]; exists {
 			a.devices[address] = device
 			a.devicesNUMANodes[address] = numaNode
+		} else if hostNUMANode, exists := unaligned[address]; exists {
+			// Device is on a host NUMA node with no pinned vCPUs (e.g.,
+			// NVIDIA GB200 where the GPU is on a separate NUMA node from
+			// all CPUs). Create a CPU-less guest NUMA cell for this host
+			// NUMA node so the device can be placed under an expander bus
+			// targeting it.
+			guestCellID := a.getOrCreateCPULessNUMACell(hostNUMANode)
+			a.devices[address] = device
+			a.devicesNUMANodes[address] = guestCellID
 		} else {
 			log.Log.Infof("device %s has no NUMA affinity information, skipping for pcie-expander-bus assignment", address)
 		}
 	}
 }
 
+// getOrCreateCPULessNUMACell returns the guest NUMA cell ID for a host NUMA
+// node that has no pinned vCPUs. If a CPU-less cell has already been created
+// for this host NUMA node, it returns the existing cell ID. Otherwise, it
+// creates a new CPU-less NUMA cell in the domain spec.
+func (a *expanderBusAssigner) getOrCreateCPULessNUMACell(hostNUMANode uint32) uint32 {
+	// Check if we already created a cell for this host NUMA node
+	if cellID, exists := a.hostNUMAToCellMap[hostNUMANode]; exists {
+		return cellID
+	}
+
+	// Find the next available cell ID
+	nextCellID := uint32(0)
+	if a.domainSpec.CPU.NUMA != nil {
+		for _, cell := range a.domainSpec.CPU.NUMA.Cells {
+			id, err := strconv.Atoi(cell.ID)
+			if err != nil {
+				continue
+			}
+			if uint32(id) >= nextCellID {
+				nextCellID = uint32(id) + 1
+			}
+		}
+	}
+
+	newCell := api.NUMACell{
+		ID:   strconv.Itoa(int(nextCellID)),
+		CPUs: "",
+	}
+
+	if a.domainSpec.CPU.NUMA == nil {
+		a.domainSpec.CPU.NUMA = &api.NUMA{}
+	}
+	a.domainSpec.CPU.NUMA.Cells = append(a.domainSpec.CPU.NUMA.Cells, newCell)
+	a.hostNUMAToCellMap[hostNUMANode] = nextCellID
+
+	log.Log.Infof("created CPU-less guest NUMA cell %d for host NUMA node %d", nextCellID, hostNUMANode)
+
+	return nextCellID
+}
+
 // numaDeviceGroups represents a mapping of NUMA nodes to host devices.
 type numaDeviceGroups map[uint32][]*api.HostDevice