pciutil.go

/**
# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package vgpu

import (
	"fmt"
	"os"
	"path"
	"strings"
)

// NvidiaPCI interface allows us to get a list of all NVIDIA PCI devices
type NvidiaPCI interface {
	Devices() ([]*PCIDevice, error)
}

// PCIDevice represents a single PCI device
type PCIDevice struct {
	Path    string
	Address string
	Class   string
	Vendor  string
	Config  []byte
}

const (
	// PciDevicesRoot represents base path for all pci devices under sysfs
	PciDevicesRoot = "/sys/bus/pci/devices"
	// PciStatusByte indicates status byte
	PciStatusByte = 0x06
	// PciStatusCapabilityList indicates if capability list is supported
	PciStatusCapabilityList = 0x10
	// PciCapabilityList indicates offset of first capability list entry
	PciCapabilityList = 0x34
	// PciCapabilityListID indicates offset for capability id
	PciCapabilityListID = 0
	// PciCapabilityListNext indicates offset for next capability in the list
	PciCapabilityListNext = 1
	// PciCapabilityLength indicates offset for capability length
	PciCapabilityLength = 2
	// PciCapabilityVendorSpecificID indicates PCI vendor specific capability id
	PciCapabilityVendorSpecificID = 0x09
	// PciNvidiaVendorID represents PCI vendor id for Nvidia
	PciNvidiaVendorID = "0x10de"
)

// NvidiaPCILib implements the NvidiaPCI interface
type NvidiaPCILib struct{}

// NewNvidiaPCILib returns an instance of NvidiaPCILib implementing the NvidiaPCI interface
func NewNvidiaPCILib() NvidiaPCI {
	return &NvidiaPCILib{}
}

// Devices returns all PCI devices on the system
func (p *NvidiaPCILib) Devices() ([]*PCIDevice, error) {
	deviceDirs, err := os.ReadDir(PciDevicesRoot)
	if err != nil {
		return nil, fmt.Errorf("unable to read PCI bus devices: %v", err)
	}

	var devices []*PCIDevice
	for _, deviceDir := range deviceDirs {
		devicePath := path.Join(PciDevicesRoot, deviceDir.Name())
		address := deviceDir.Name()

		vendor, err := os.ReadFile(path.Join(devicePath, "vendor"))
		if err != nil {
			return nil, fmt.Errorf("unable to read PCI device vendor id for %s: %v", address, err)
		}

		if strings.TrimSpace(string(vendor)) != PciNvidiaVendorID {
			continue
		}

		class, err := os.ReadFile(path.Join(devicePath, "class"))
		if err != nil {
			return nil, fmt.Errorf("unable to read PCI device class for %s: %v", address, err)
		}

		config, err := os.ReadFile(path.Join(devicePath, "config"))
		if err != nil {
			return nil, fmt.Errorf("unable to read PCI configuration space for %s: %v", address, err)
		}

		device := &PCIDevice{
			Path:    devicePath,
			Address: address,
			Vendor:  strings.TrimSpace(string(vendor)),
			Class:   string(class)[0:4],
			Config:  config,
		}

		devices = append(devices, device)
	}

	return devices, nil
}

// GetVendorSpecificCapability returns the vendor specific capability from configuration space
func (d *PCIDevice) GetVendorSpecificCapability() ([]byte, error) {
	if len(d.Config) < 256 {
		return nil, fmt.Errorf("entire PCI configuration is not read for device %s. Please run GFD with privileged mode to read complete PCI configuration data", d.Address)
	}

	if d.Config[PciStatusByte]&PciStatusCapabilityList == 0 {
		return nil, nil
	}

	var visited [256]byte
	pos := GetByte(d.Config, PciCapabilityList)
	for pos != 0 {
		id := GetByte(d.Config, pos+PciCapabilityListID)
		next := GetByte(d.Config, pos+PciCapabilityListNext)
		length := GetByte(d.Config, pos+PciCapabilityLength)

		if visited[pos] != 0 {
			// chain looped
			break
		}
		if id == 0xff {
			// chain broken
			break
		}
		if id == PciCapabilityVendorSpecificID {
			start := int(pos) + PciCapabilityListID
			if length == 0 || start+int(length) > len(d.Config) {
				// Malformed capability record (e.g. length read from PCI
				// config space exceeds the buffer). Skip it instead of
				// panicking so a single bad device can't crash GFD.
				return nil, nil
			}
			capability := d.Config[start : start+int(length)]
			return capability, nil
		}

		visited[pos]++
		pos = next
	}

	return nil, nil
}

// GetByte returns a single byte of data at specified position
func GetByte(buffer []byte, pos uint8) uint8 {
	return buffer[pos]
}

// GetWord returns 2 bytes of data from specified position
func GetWord(buffer []byte, pos int) uint16 {
	return uint16(buffer[pos]) | (uint16(buffer[pos+1]) << 8)
}

// GetLong returns 4 bytes of data from specified position
func GetLong(buffer []byte, pos int) uint32 {
	return uint32(buffer[pos]) |
		uint32(buffer[pos+1])<<8 |
		uint32(buffer[pos+2])<<16 |
		uint32(buffer[pos+3])<<24
}

// MockNvidiaPCI represents mock of NvidiaPCI interface
type MockNvidiaPCI struct {
	devices []*PCIDevice
}

// Devices returns PCI devices with mocked data
func (p *MockNvidiaPCI) Devices() ([]*PCIDevice, error) {
	return p.devices, nil
}

// NewMockNvidiaPCI initializes and returns mock PCI interface type
func NewMockNvidiaPCI() NvidiaPCI {
	var (
		gpuPassThroughConfig = []byte{0xde, 0x10, 0x8a, 0x11, 0x07, 0x04, 0x10, 0x00, 0xa1, 0x00, 0x00, 0x03, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, 0x0c, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x10, 0x14, 0x10, 0x00, 0x00, 0x00, 0xee, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, 0xde, 0x10, 0x14, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xce, 0xd6, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x68, 0x03, 0x00, 0x08, 0x00, 0x00, 0x00, 0x05, 0x78, 0x81, 0x00, 0x00, 0x70, 0xe6, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x10, 0xb4, 0x02, 0x00, 0xe1, 0x8d, 0x64, 0x00, 0x10, 0x29, 0x00, 0x00, 0x03, 0x3d, 0x45, 0x10, 0x00, 0x00, 0x01, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x03, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
		vgpuConfig           = []byte{0xde, 0x10, 0xb8, 0x1e, 0x02, 0x05, 0xff, 0x06, 0xa1, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x0c, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x10, 0x0f, 0x13, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xce, 0xd6, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x81, 0x00, 0x00, 0x00, 0xe0, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x68, 0x1b, 0x56, 0x46, 0x00, 0x16, 0x34, 0x36, 0x30, 0x2e, 0x31, 0x36, 0x00, 0x00, 0x00, 0x00, 0x72, 0x34, 0x36, 0x30, 0x5f, 0x30, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	)

	return &MockNvidiaPCI{
		devices: []*PCIDevice{
			{
				Path:    "",
				Address: "passthrough",
				Vendor:  "0x10de",
				Class:   "300",
				Config:  gpuPassThroughConfig,
			},
			{
				Path:    "",
				Address: "vgpu",
				Vendor:  "0x10de",
				Class:   "300",
				Config:  vgpuConfig,
			},
		},
	}
}