feat: Add PF VFIO support

This change enables ib-sriov-cni to support Physical Function (PF)
passthrough in addition to Virtual Functions (VF), particularly for
KubeVirt GPU workloads requiring InfiniBand RDMA.

Closes #159

Signed-off-by: Zhen(Winson) Wang <zhewang@nvidia.com>

Author: winsopc

README.md

@@ -207,7 +207,9 @@ echo 8 > /sys/class/net/ib0/device/sriov_numvfs
 * `rdmaIsolation` (boolean, optional): Enable RDMA network namespace isolation for RDMA workloads. More information
 about the system requirements to support this mode of operation can be found [here](https://github.com/Mellanox/rdma-cni)
 * `ibKubernetesEnabled` (bool, optional): Enforces ib-sriov-cni to work with [ib-kubernetes](https://www.github.com/Mellanox/ib-kubernetes).
-* `vfioPciMode` (boolean, optional): Enable VFIO mode for VF devices bound to vfio-pci driver. When enabled, the CNI skips network interface configuration as VFIO devices are used for direct device assignment (e.g., for kubevirt/VM workloads). Defaults to false. If not explicitly set, the mode is auto-detected based on the VF's driver binding.
+* `vfioPciMode` (boolean, optional): Enable VFIO mode for devices (VF or PF) bound to vfio-pci driver. When enabled, the CNI skips network interface configuration as VFIO devices are used for direct device assignment (e.g., for kubevirt/VM workloads). Defaults to false. If not explicitly set, the mode is auto-detected based on the device's driver binding.
+
+> *__Note__*: PF passthrough is only supported in VFIO mode. When using a PF device, it must be bound to the vfio-pci driver and `vfioPciMode` must be enabled (or auto-detected). Moving a PF's InfiniBand interface into a pod network namespace is not supported.
 
 > *__Note__*: If `rdmaIsolation` is set to _true_, [`rdma-cni`](https://github.com/Mellanox/rdma-cni) should not be used.
 

cmd/ib-sriov-cni/main.go

@@ -96,10 +96,6 @@ func unlockCNIExecution(lock *flock.Flock) {
 }
 
 func handleVfioPciDetection(netConf *localtypes.NetConf) error {
-	if netConf.DeviceID == "" {
-		return fmt.Errorf("device ID is required for VFIO PCI detection")
-	}
-
 	isVfioPci, err := utils.IsVfioPciDevice(netConf.DeviceID)
 	if err != nil {
 		return fmt.Errorf("failed to check vfio-pci driver binding for device %s: %v", netConf.DeviceID, err)
@@ -132,29 +128,45 @@ func getNetConfNetns(args *skel.CmdArgs) (*localtypes.NetConf, ns.NetNS, error)
 			infiniBandAnnotation, configuredInfiniBand)
 	}
 
-	netConf.GUID = getGUIDFromConf(netConf)
-
-	// Ensure GUID was provided if ib-kubernetes integration is enabled
-	if netConf.IBKubernetesEnabled && netConf.GUID == "" {
-		return nil, nil, fmt.Errorf(
-			"infiniband SRIOV-CNI failed, Unexpected error. GUID must be provided by ib-kubernetes")
-	}
-
 	if netConf.RdmaIsolation {
 		err = utils.EnsureRdmaSystemMode()
 		if err != nil {
 			return nil, nil, err
 		}
 	}
 
+	// Validate deviceID is provided
+	if netConf.DeviceID == "" {
+		return nil, nil, fmt.Errorf("deviceID is required")
+	}
+
 	// Handle vfio-pci detection
 	if err := handleVfioPciDetection(netConf); err != nil {
 		return nil, nil, err
 	}
 
-	err = config.LoadDeviceInfo(netConf)
+	// Check if device is PF or VF to load appropriate device info
+	isVF, err := utils.IsVirtualFunction(netConf.DeviceID)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to get device specific information. %v", err)
+		return nil, nil, fmt.Errorf("failed to determine if device %s is VF or PF: %v", netConf.DeviceID, err)
+	}
+	netConf.IsVFDevice = isVF
+
+	netConf.GUID = getGUIDFromConf(netConf)
+
+	// Ensure GUID was provided if ib-kubernetes integration is enabled
+	// Note: PF devices already have their own GUID, so only check for VF devices
+	if netConf.IBKubernetesEnabled && netConf.IsVFDevice && netConf.GUID == "" {
+		return nil, nil, fmt.Errorf(
+			"infiniband SRIOV-CNI failed, Unexpected error. GUID must be provided by ib-kubernetes")
+	}
+
+	// Only load VF device info for VF devices (PF device that is bound to vfio dont need this)
+	if netConf.IsVFDevice {
+		err = config.LoadDeviceInfo(netConf)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get VF device information: %v", err)
+		}
 	}
 
 	netns, err := ns.GetNS(args.Netns)
@@ -249,23 +261,11 @@ func runIPAMPlugin(stdinData []byte, netConf *localtypes.NetConf) (_ *current.Re
 	return newResult, nil
 }
 
-func cmdAdd(args *skel.CmdArgs) (retErr error) {
-	netConf, netns, err := getNetConfNetns(args)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = netns.Close() }()
-
+// handleVFAdd handles VF device configuration in cmdAdd
+func handleVFAdd(args *skel.CmdArgs, netConf *localtypes.NetConf, netns ns.NetNS, result *current.Result) (retErr error) {
 	sm := sriov.NewSriovManager()
 
-	// Lock CNI operation to serialize the operation
-	lock, err := lockCNIExecution()
-	if err != nil {
-		return err
-	}
-	defer unlockCNIExecution(lock)
-
-	err = doVFConfig(sm, netConf, netns, args)
+	err := doVFConfig(sm, netConf, netns, args)
 	if err != nil {
 		return err
 	}
@@ -284,12 +284,6 @@ func cmdAdd(args *skel.CmdArgs) (retErr error) {
 		}
 	}()
 
-	result := &current.Result{}
-	result.Interfaces = []*current.Interface{{
-		Name:    args.IfName,
-		Sandbox: netns.Path(),
-	}}
-
 	// VFIO devices don't have network interfaces, skip IPAM configuration
 	if netConf.IPAM.Type != "" && !netConf.VfioPciMode {
 		var newResult *current.Result
@@ -318,12 +312,54 @@ func cmdAdd(args *skel.CmdArgs) (retErr error) {
 			return err
 		}
 
-		result = newResult
+		// Update result pointer to point to the new result
+		*result = *newResult
 	}
 
 	// Cache NetConf for CmdDel
 	if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
-		return fmt.Errorf("error saving NetConf %q", err)
+		return fmt.Errorf("error saving NetConf: %v", err)
+	}
+
+	return nil
+}
+
+func cmdAdd(args *skel.CmdArgs) (retErr error) {
+	netConf, netns, err := getNetConfNetns(args)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = netns.Close() }()
+
+	// Lock CNI operation to serialize the operation
+	lock, err := lockCNIExecution()
+	if err != nil {
+		return err
+	}
+	defer unlockCNIExecution(lock)
+
+	result := &current.Result{}
+	result.Interfaces = []*current.Interface{{
+		Name:    args.IfName,
+		Sandbox: netns.Path(),
+	}}
+
+	// Check if device is PF (Physical Function) - flag was set in getNetConfNetns
+	// PF passthrough devices don't need VF configuration
+	if !netConf.IsVFDevice {
+		if !netConf.VfioPciMode {
+			return fmt.Errorf("PF device %s requires vfioPciMode to be enabled", netConf.DeviceID)
+		}
+		// PF device - just cache config and return success
+		if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
+			return fmt.Errorf("error saving NetConf: %v", err)
+		}
+	} else {
+		// VF device - continue with normal VF configuration
+		err = handleVFAdd(args, netConf, netns, result)
+		if err != nil {
+			return err
+		}
 	}
 
 	return types.PrintResult(result, netConf.CNIVersion)
@@ -340,6 +376,38 @@ func handleIPAMCleanup(netConf *localtypes.NetConf, stdinData []byte) error {
 	return ipam.ExecDel(netConf.IPAM.Type, stdinData)
 }
 
+// handleVFCleanup performs VF-specific cleanup operations
+func handleVFCleanup(sm localtypes.Manager, netConf *localtypes.NetConf, args *skel.CmdArgs, netns ns.NetNS) error {
+	// VFIO devices don't have network interfaces to release
+	if !netConf.VfioPciMode {
+		err := sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Move RDMA device to default namespace
+	// Note(adrianc): Due to some un-intuitive kernel behavior (which i hope will change), moving an RDMA device
+	// to namespace causes all of its associated ULP devices (IPoIB) to be recreated in the default namespace.
+	// we strategically place this here to allow:
+	//   1. netedv cleanup during ReleaseVF.
+	//   2. rdma dev netns cleanup as ResetVFConfig will rebind the VF.
+	// Doing anything would have yielded the same results however ResetVFConfig will eventually not trigger VF rebind.
+	if netConf.RdmaIsolation {
+		err := utils.MoveRdmaDevFromNs(netConf.RdmaNetState.ContainerRdmaDevName, netns)
+		if err != nil {
+			return fmt.Errorf(
+				"failed to restore RDMA device %s to default namespace. %v",
+				netConf.RdmaNetState.ContainerRdmaDevName, err)
+		}
+	}
+
+	if err := sm.ResetVFConfig(netConf); err != nil {
+		return fmt.Errorf("cmdDel() error resetting VF: %v", err)
+	}
+	return nil
+}
+
 func cmdDel(args *skel.CmdArgs) (retErr error) {
 	// https://github.com/kubernetes/kubernetes/pull/35240
 	if args.Netns == "" {
@@ -388,41 +456,25 @@ func cmdDel(args *skel.CmdArgs) (retErr error) {
 	}
 	defer func() { _ = netns.Close() }()
 
-	// Lock CNI operation to serialize the operation
-	lock, err := lockCNIExecution()
+	// Detect if device is VF or PF at runtime during Del
+	isVF, err := utils.IsVirtualFunction(netConf.DeviceID)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to determine if device %s is VF or PF: %v", netConf.DeviceID, err)
 	}
-	defer unlockCNIExecution(lock)
 
-	// VFIO devices don't have network interfaces to release
-	if !netConf.VfioPciMode {
-		err = sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns)
-		if err != nil {
-			return err
-		}
+	// PF devices don't need VF cleanup
+	if !isVF {
+		return nil
 	}
 
-	// Move RDMA device to default namespace
-	// Note(adrianc): Due to some un-intuitive kernel behavior (which i hope will change), moving an RDMA device
-	// to namespace causes all of its associated ULP devices (IPoIB) to be recreated in the default namespace.
-	// we strategically place this here to allow:
-	//   1. netedv cleanup during ReleaseVF.
-	//   2. rdma dev netns cleanup as ResetVFConfig will rebind the VF.
-	// Doing anything would have yielded the same results however ResetVFConfig will eventually not trigger VF rebind.
-	if netConf.RdmaIsolation {
-		err = utils.MoveRdmaDevFromNs(netConf.RdmaNetState.ContainerRdmaDevName, netns)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to restore RDMA device %s to default namespace. %v",
-				netConf.RdmaNetState.ContainerRdmaDevName, err)
-		}
+	// Lock CNI operation to serialize the operation
+	lock, err := lockCNIExecution()
+	if err != nil {
+		return err
 	}
+	defer unlockCNIExecution(lock)
 
-	if err = sm.ResetVFConfig(netConf); err != nil {
-		return fmt.Errorf("cmdDel() error reseting VF: %q", err)
-	}
-	return nil
+	return handleVFCleanup(sm, netConf, args, netns)
 }
 
 func cmdCheck(args *skel.CmdArgs) error {

pkg/types/types.go

@@ -31,6 +31,7 @@ type IbSriovNetConf struct {
 	RdmaIsolation       bool   `json:"rdmaIsolation,omitempty"`
 	IBKubernetesEnabled bool   `json:"ibKubernetesEnabled,omitempty"`
 	VfioPciMode         bool   `json:"vfioPciMode,omitempty"` // Skip SR-IOV network setup, default false
+	IsVFDevice          bool   `json:"-"`                     // Runtime flag: true if device is VF, false if PF
 	RdmaNetState        rdmatypes.RdmaNetState
 	RuntimeConfig       RuntimeConf `json:"runtimeConfig,omitempty"`
 	Args                struct {

pkg/utils/utils.go

@@ -262,3 +262,22 @@ func IsVfioPciDevice(pciAddr string) (bool, error) {
 	driverName := filepath.Base(linkTarget)
 	return driverName == VfioPciDriverName, nil
 }
+
+// IsVirtualFunction checks if a PCI device is a VF by checking for physfn symlink
+func IsVirtualFunction(pciAddr string) (bool, error) {
+	physfnPath := filepath.Join(SysBusPci, pciAddr, "physfn")
+
+	// Check if physfn symlink exists
+	_, err := os.Lstat(physfnPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// physfn doesn't exist, so this is not a VF (likely a PF)
+			return false, nil
+		}
+		// Other error occurred
+		return false, err
+	}
+
+	// physfn exists, so this is a VF
+	return true, nil
+}

pkg/utils/utils_test.go

@@ -81,6 +81,32 @@ var _ = Describe("Utils", func() {
 			Expect(guid).To(Equal(""))
 		})
 	})
+	Context("Checking IsVirtualFunction function", func() {
+		It("Assuming VF device (has physfn)", func() {
+			// This test assumes 0000:af:06.0 is a VF with physfn symlink
+			result, err := IsVirtualFunction("0000:af:06.0")
+			Expect(err).NotTo(HaveOccurred(), "Should not return error for valid PCI address")
+			Expect(result).To(Equal(true), "VF device should return true")
+		})
+		It("Assuming PF device (no physfn)", func() {
+			// Test with the actual PF device 0000:af:00.1 (ib0) from fixture
+			result, err := IsVirtualFunction("0000:af:00.1")
+			Expect(err).NotTo(HaveOccurred(), "Should not return error for valid PCI address")
+			Expect(result).To(Equal(false), "PF device should return false")
+		})
+		It("Assuming VFIO VF device (has physfn)", func() {
+			// Test with VFIO VF 0000:af:06.1 - should still be detected as VF
+			result, err := IsVirtualFunction("0000:af:06.1")
+			Expect(err).NotTo(HaveOccurred(), "Should not return error for valid PCI address")
+			Expect(result).To(Equal(true), "VFIO VF device should still return true")
+		})
+		It("Assuming non-existing device", func() {
+			// This should return false and no error for non-existing device
+			result, err := IsVirtualFunction("0000:ff:ff.f")
+			Expect(err).NotTo(HaveOccurred(), "Should not return error for non-existing device")
+			Expect(result).To(Equal(false), "Non-existing device should return false")
+		})
+	})
 	Context("Checking IsVfioPciDevice function", func() {
 		It("Assuming device bound to vfio-pci driver", func() {
 			// Test with VF (0000:af:06.1) that is bound to vfio-pci in the mock