Merge pull request #155 from rajatchopra/iommufd

cdi spec file should print devices with multiple name keys

Author: zvonkok

go.mod

@@ -5,7 +5,7 @@ go 1.24.0
 toolchain go1.24.9
 
 require (
-	github.com/NVIDIA/go-nvlib v0.8.1
+	github.com/NVIDIA/go-nvlib v0.9.0
 	github.com/NVIDIA/nvidia-container-toolkit v1.18.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/pelletier/go-toml v1.9.5

go.sum

@@ -1,7 +1,9 @@
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/NVIDIA/go-nvlib v0.8.1 h1:OPEHVvn3zcV5OXB68A7WRpeCnYMRSPl7LdeJH/d3gZI=
-github.com/NVIDIA/go-nvlib v0.8.1/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c=
+github.com/NVIDIA/go-nvlib v0.8.2-0.20251124190111-9a6788d93d8c h1:1w55FLYiTkHL1q1QjKDozZPu5Ca8lfyC3GHV+jMTSz8=
+github.com/NVIDIA/go-nvlib v0.8.2-0.20251124190111-9a6788d93d8c/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c=
+github.com/NVIDIA/go-nvlib v0.9.0 h1:GKLIvLJ0uhCtTLLZp2Q8QIDRxOYH45MM4Y5OO3U5Rho=
+github.com/NVIDIA/go-nvlib v0.9.0/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c=
 github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
 github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
 github.com/NVIDIA/nvidia-container-toolkit v1.18.0 h1:bXoKq9C1WHU5fF6VqXvX3RkMzpp4ihTUgBPrh66vTf0=

internal/cdi/lib-vfio.go

@@ -29,6 +29,12 @@ import (
 
 type vfiolib nvcdilib
 
+const (
+	VfioPath        = "/dev/vfio"
+	VfioDevicesPath = "/dev/vfio/devices"
+	VfioDriver      = "vfio-pci"
+)
+
 var _ nvcdi.Interface = (*vfiolib)(nil)
 
 // GetSpec returns the complete CDI spec
@@ -60,20 +66,49 @@ func (l *vfiolib) GetAllDeviceSpecs() ([]specs.Device, error) {
 		return nil, fmt.Errorf("failed getting NVIDIA GPUs: %w", err)
 	}
 
+	path := VfioPath
+	devName := ""
+
 	for idx, dev := range devices {
-		if dev.Driver == "vfio-pci" {
+		if dev.Driver == VfioDriver {
 			klog.Infof("Found NVIDIA device: address=%s, driver=%s, iommu_group=%d, deviceId=%x",
 				dev.Address, dev.Driver, dev.IommuGroup, dev.Device)
-			deviceSpecs = append(deviceSpecs, specs.Device{
-				Name: fmt.Sprintf("%d", idx),
-				ContainerEdits: specs.ContainerEdits{
-					DeviceNodes: []*specs.DeviceNode{
-						{
-							Path: fmt.Sprintf("/dev/vfio/%d", dev.IommuGroup),
-						},
+
+			if dev.IommuFD != "" {
+				path = VfioDevicesPath
+				devName = dev.IommuFD
+			} else {
+				devName = fmt.Sprintf("%d", dev.IommuGroup)
+			}
+			cedits := specs.ContainerEdits{
+				DeviceNodes: []*specs.DeviceNode{
+					{
+						Path: fmt.Sprintf("%s/%s", path, devName),
 					},
 				},
+			}
+			// Add the same device multiple times with keys for meant for
+			// various use cases:
+			// key=idx: use case where cdi annotations are manually put
+			//   on pod spec e.g. 0,1,2 etc
+			// key=IommuGroup e.g. 65 for /dev/vfio/65 in non-iommufd setup
+			//   and legacy device plugin case
+			// key=IommuFD e.g. vfio0 for /dev/vfio/devices/vfio0 for
+			//   iommufd support
+			deviceSpecs = append(deviceSpecs, specs.Device{
+				Name:           fmt.Sprintf("%d", idx),
+				ContainerEdits: cedits,
+			})
+			deviceSpecs = append(deviceSpecs, specs.Device{
+				Name:           fmt.Sprintf("%d", dev.IommuGroup),
+				ContainerEdits: cedits,
 			})
+			if dev.IommuFD != "" {
+				deviceSpecs = append(deviceSpecs, specs.Device{
+					Name:           fmt.Sprintf("%s", dev.IommuFD),
+					ContainerEdits: cedits,
+				})
+			}
 		}
 	}
 

internal/cdi/lib-vfio_test.go

@@ -61,6 +61,57 @@ func TestGetAllDeviceSpecs(t *testing.T) {
 			},
 			expectedDeviceSpecs: nil,
 		},
+		{
+			description: "one IommuFD NVIDIA device, bound to vfio-pci",
+			lib: &vfiolib{
+				nvpcilib: &nvpciInterfaceMock{
+					GetGPUsFunc: func() ([]*nvpci.NvidiaPCIDevice, error) {
+						devices := []*nvpci.NvidiaPCIDevice{
+							{
+								Address:    "000:3B:00.0",
+								Device:     0x2331,
+								IommuGroup: 60,
+								IommuFD:    "vfio0",
+								Driver:     "vfio-pci",
+							},
+						}
+						return devices, nil
+					},
+				},
+			},
+			expectedDeviceSpecs: []specs.Device{
+				{
+					Name: "0",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/devices/vfio0",
+							},
+						},
+					},
+				},
+				{
+					Name: "60",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/devices/vfio0",
+							},
+						},
+					},
+				},
+				{
+					Name: "vfio0",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/devices/vfio0",
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			description: "one NVIDIA device, bound to vfio-pci",
 			lib: &vfiolib{
@@ -89,6 +140,16 @@ func TestGetAllDeviceSpecs(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name: "60",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/60",
+							},
+						},
+					},
+				},
 			},
 		},
 		{
@@ -125,6 +186,16 @@ func TestGetAllDeviceSpecs(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name: "60",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/60",
+							},
+						},
+					},
+				},
 			},
 		},
 		{
@@ -161,6 +232,16 @@ func TestGetAllDeviceSpecs(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name: "60",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/60",
+							},
+						},
+					},
+				},
 				{
 					Name: "1",
 					ContainerEdits: specs.ContainerEdits{
@@ -171,6 +252,16 @@ func TestGetAllDeviceSpecs(t *testing.T) {
 						},
 					},
 				},
+				{
+					Name: "90",
+					ContainerEdits: specs.ContainerEdits{
+						DeviceNodes: []*specs.DeviceNode{
+							{
+								Path: "/dev/vfio/90",
+							},
+						},
+					},
+				},
 			},
 		},
 	}

vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/device.go

@@ -2,7 +2,7 @@
 ## explicit; go 1.18
 github.com/BurntSushi/toml
 github.com/BurntSushi/toml/internal
-# github.com/NVIDIA/go-nvlib v0.8.1
+# github.com/NVIDIA/go-nvlib v0.9.0
 ## explicit; go 1.20
 github.com/NVIDIA/go-nvlib/pkg/nvlib/device
 github.com/NVIDIA/go-nvlib/pkg/nvlib/info