Skip to content

Commit fce45db

Browse files
karthikvetrivelkarthikvetrivel
authored
Merge pull request #90 from karthikvetrivel/nvpci-subsystem-ids
Add SubsystemVendor and SubsystemDevice fields to NvidiaPCIDevice
2 parents fc8796e + e0f2e8c commit fce45db

3 files changed

Lines changed: 174 additions & 28 deletions

File tree

‎pkg/nvpci/mock.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,24 @@ func createNVIDIAgpuFiles(deviceDir string) error {
167167
return err
168168
}
169169

170+
subsystemVendor, err := os.Create(filepath.Join(deviceDir, "subsystem_vendor"))
171+
if err != nil {
172+
return err
173+
}
174+
_, err = fmt.Fprintf(subsystemVendor, "0x%x", PCINvidiaVendorID)
175+
if err != nil {
176+
return err
177+
}
178+
179+
subsystemDevice, err := os.Create(filepath.Join(deviceDir, "subsystem_device"))
180+
if err != nil {
181+
return err
182+
}
183+
_, err = subsystemDevice.WriteString("0x16c0")
184+
if err != nil {
185+
return err
186+
}
187+
170188
_, err = os.Create(filepath.Join(deviceDir, "nvidia"))
171189
if err != nil {
172190
return err

‎pkg/nvpci/nvpci.go‎

Lines changed: 62 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -107,20 +107,22 @@ func (s *SriovInfo) IsVF() bool {
107107

108108
// NvidiaPCIDevice represents a PCI device for an NVIDIA product.
109109
type NvidiaPCIDevice struct {
110-
Path string
111-
Address string
112-
Vendor uint16
113-
Class uint32
114-
ClassName string
115-
Device uint16
116-
DeviceName string
117-
Driver string
118-
IommuGroup int
119-
IommuFD string
120-
NumaNode int
121-
Config *ConfigSpace
122-
Resources MemoryResources
123-
SriovInfo SriovInfo
110+
Path string
111+
Address string
112+
Vendor uint16
113+
Class uint32
114+
ClassName string
115+
Device uint16
116+
SubsystemVendor uint16
117+
SubsystemDevice uint16
118+
DeviceName string
119+
Driver string
120+
IommuGroup int
121+
IommuFD string
122+
NumaNode int
123+
Config *ConfigSpace
124+
Resources MemoryResources
125+
SriovInfo SriovInfo
124126
}
125127

126128
// IsVGAController if class == 0x300.
@@ -298,6 +300,36 @@ func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*Nvid
298300
return nil, fmt.Errorf("unable to convert device string to uint16: %v", deviceStr)
299301
}
300302

303+
var subsystemVendorID uint64
304+
subsystemVendor, err := os.ReadFile(path.Join(devicePath, "subsystem_vendor"))
305+
switch {
306+
case err == nil:
307+
subsystemVendorStr := strings.TrimSpace(string(subsystemVendor))
308+
subsystemVendorID, err = strconv.ParseUint(subsystemVendorStr, 0, 16)
309+
if err != nil {
310+
return nil, fmt.Errorf("unable to convert subsystem vendor string to uint16: %v", subsystemVendorStr)
311+
}
312+
case os.IsNotExist(err):
313+
p.logger.Warningf("subsystem_vendor file not found for %s", address)
314+
default:
315+
return nil, fmt.Errorf("unable to read PCI subsystem vendor id for %s: %v", address, err)
316+
}
317+
318+
var subsystemDeviceID uint64
319+
subsystemDevice, err := os.ReadFile(path.Join(devicePath, "subsystem_device"))
320+
switch {
321+
case err == nil:
322+
subsystemDeviceStr := strings.TrimSpace(string(subsystemDevice))
323+
subsystemDeviceID, err = strconv.ParseUint(subsystemDeviceStr, 0, 16)
324+
if err != nil {
325+
return nil, fmt.Errorf("unable to convert subsystem device string to uint16: %v", subsystemDeviceStr)
326+
}
327+
case os.IsNotExist(err):
328+
p.logger.Warningf("subsystem_device file not found for %s", address)
329+
default:
330+
return nil, fmt.Errorf("unable to read PCI subsystem device id for %s: %v", address, err)
331+
}
332+
301333
driver, err := getDriver(devicePath)
302334
if err != nil {
303335
return nil, fmt.Errorf("unable to detect driver for %s: %w", address, err)
@@ -391,20 +423,22 @@ func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*Nvid
391423
}
392424

393425
nvdevice := &NvidiaPCIDevice{
394-
Path: devicePath,
395-
Address: address,
396-
Vendor: uint16(vendorID),
397-
Class: uint32(classID),
398-
Device: uint16(deviceID),
399-
Driver: driver,
400-
IommuGroup: int(iommuGroup),
401-
IommuFD: iommuFD,
402-
NumaNode: int(numaNode),
403-
Config: config,
404-
Resources: resources,
405-
DeviceName: deviceName,
406-
ClassName: className,
407-
SriovInfo: sriovInfo,
426+
Path: devicePath,
427+
Address: address,
428+
Vendor: uint16(vendorID),
429+
Class: uint32(classID),
430+
Device: uint16(deviceID),
431+
SubsystemVendor: uint16(subsystemVendorID),
432+
SubsystemDevice: uint16(subsystemDeviceID),
433+
Driver: driver,
434+
IommuGroup: int(iommuGroup),
435+
IommuFD: iommuFD,
436+
NumaNode: int(numaNode),
437+
Config: config,
438+
Resources: resources,
439+
DeviceName: deviceName,
440+
ClassName: className,
441+
SriovInfo: sriovInfo,
408442
}
409443

410444
// Cache physical functions only as VF can't be a root device.

‎pkg/nvpci/nvpci_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package nvpci
1818

1919
import (
20+
"os"
21+
"path/filepath"
2022
"testing"
2123

2224
"github.com/stretchr/testify/require"
@@ -74,6 +76,98 @@ func TestNvpci(t *testing.T) {
7476
_, err = nvpci.GetGPUByIndex(1)
7577
require.Error(t, err, "No error returned when getting GPU at invalid index")
7678
}
79+
80+
func TestNvpciSubsystem(t *testing.T) {
81+
nvpci, err := NewMockNvpci()
82+
require.Nil(t, err, "Error creating NewMockNvpci")
83+
defer nvpci.Cleanup()
84+
85+
err = nvpci.AddMockA100("0000:80:05.1", 0, nil)
86+
require.Nil(t, err, "Error adding Mock A100 device to MockNvpci")
87+
88+
devices, err := nvpci.GetGPUs()
89+
require.Nil(t, err, "Error getting GPUs")
90+
require.Equal(t, 1, len(devices), "Wrong number of GPU devices")
91+
require.Equal(t, uint16(0x10de), devices[0].SubsystemVendor, "Wrong SubsystemVendor for device")
92+
require.Equal(t, uint16(0x16c0), devices[0].SubsystemDevice, "Wrong SubsystemDevice for device")
93+
}
94+
95+
func TestNvpciSubsystemMissing(t *testing.T) {
96+
nvpci, err := NewMockNvpci()
97+
require.Nil(t, err, "Error creating NewMockNvpci")
98+
defer nvpci.Cleanup()
99+
100+
err = nvpci.AddMockA100("0000:80:05.1", 0, nil)
101+
require.Nil(t, err, "Error adding Mock A100 device to MockNvpci")
102+
103+
deviceDir := filepath.Join(nvpci.pciDevicesRoot, "0000:80:05.1")
104+
require.NoError(t, os.Remove(filepath.Join(deviceDir, "subsystem_vendor")))
105+
require.NoError(t, os.Remove(filepath.Join(deviceDir, "subsystem_device")))
106+
107+
devices, err := nvpci.GetGPUs()
108+
require.Nil(t, err, "Error getting GPUs")
109+
require.Equal(t, 1, len(devices), "Wrong number of GPU devices")
110+
require.Equal(t, uint16(0), devices[0].SubsystemVendor, "SubsystemVendor should default to 0 when sysfs file is absent")
111+
require.Equal(t, uint16(0), devices[0].SubsystemDevice, "SubsystemDevice should default to 0 when sysfs file is absent")
112+
}
113+
114+
func TestNvpciSubsystemPartial(t *testing.T) {
115+
testCases := []struct {
116+
Description string
117+
RemoveFile string
118+
SubsystemVendor uint16
119+
SubsystemDevice uint16
120+
}{
121+
{
122+
Description: "subsystem_vendor missing",
123+
RemoveFile: "subsystem_vendor",
124+
SubsystemVendor: 0,
125+
SubsystemDevice: 0x16c0,
126+
},
127+
{
128+
Description: "subsystem_device missing",
129+
RemoveFile: "subsystem_device",
130+
SubsystemVendor: 0x10de,
131+
SubsystemDevice: 0,
132+
},
133+
}
134+
135+
for _, tc := range testCases {
136+
t.Run(tc.Description, func(t *testing.T) {
137+
nvpci, err := NewMockNvpci()
138+
require.Nil(t, err, "Error creating NewMockNvpci")
139+
defer nvpci.Cleanup()
140+
141+
err = nvpci.AddMockA100("0000:80:05.1", 0, nil)
142+
require.Nil(t, err, "Error adding Mock A100 device to MockNvpci")
143+
144+
deviceDir := filepath.Join(nvpci.pciDevicesRoot, "0000:80:05.1")
145+
require.NoError(t, os.Remove(filepath.Join(deviceDir, tc.RemoveFile)))
146+
147+
devices, err := nvpci.GetGPUs()
148+
require.Nil(t, err, "Error getting GPUs")
149+
require.Equal(t, 1, len(devices), "Wrong number of GPU devices")
150+
require.Equal(t, tc.SubsystemVendor, devices[0].SubsystemVendor, "Wrong SubsystemVendor for device")
151+
require.Equal(t, tc.SubsystemDevice, devices[0].SubsystemDevice, "Wrong SubsystemDevice for device")
152+
})
153+
}
154+
}
155+
156+
func TestNvpciSubsystemMalformed(t *testing.T) {
157+
nvpci, err := NewMockNvpci()
158+
require.Nil(t, err, "Error creating NewMockNvpci")
159+
defer nvpci.Cleanup()
160+
161+
err = nvpci.AddMockA100("0000:80:05.1", 0, nil)
162+
require.Nil(t, err, "Error adding Mock A100 device to MockNvpci")
163+
164+
deviceDir := filepath.Join(nvpci.pciDevicesRoot, "0000:80:05.1")
165+
require.NoError(t, os.WriteFile(filepath.Join(deviceDir, "subsystem_vendor"), []byte("notanid"), 0644))
166+
167+
_, err = nvpci.GetGPUs()
168+
require.Error(t, err, "Expected error when subsystem_vendor contents are malformed")
169+
}
170+
77171
func TestNvpciIOMMUFD(t *testing.T) {
78172
testCases := []struct {
79173
Description string

0 commit comments

Comments
 (0)