bug: rdma exlusive handling

In case a RDMA device in exclusive mode is in use
by a Pod, the DP was not reporting it as a resource
after DP restart.

Following changes are introduced in RdmaSpec:

- isRdma: in case of no rdma resources,
  check if netlink "enable_rdma" is available.
- GetRdmaDeviceSpec: the device specs are retrieved
  dynamically and not on discovery stage as before.

Dynamic RDMA specs computatiopn vs on discovery, comes
to solve following scenario for exlusive mode:
- Discover RDMA device
- Allocate to Pod (resources are hidden on host)
- Restart DP pod
- Deallocate
- Reallocate

Fixes #565

Signed-off-by: Fred Rolland <[email protected]>

Author: rollandf

pkg/devices/rdma.go

@@ -18,22 +18,55 @@
 package devices
 
 import (
+	"github.com/golang/glog"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
 )
 
 type rdmaSpec struct {
-	isSupportRdma bool
-	deviceSpec    []*pluginapi.DeviceSpec
+	deviceID   string
+	deviceType types.DeviceType
 }
 
-func newRdmaSpec(rdmaResources []string) types.RdmaSpec {
+// NewRdmaSpec returns the RdmaSpec
+func NewRdmaSpec(dt types.DeviceType, id string) types.RdmaSpec {
+	if dt == types.AcceleratorType {
+		return nil
+	}
+	return &rdmaSpec{deviceID: id, deviceType: dt}
+}
+
+func (r *rdmaSpec) IsRdma() bool {
+	if len(r.getRdmaResources()) > 0 {
+		return true
+	}
+	// Checking for netlink param for exclusive RDMA use case
+	rdma, err := utils.HasRdmaParam(r.deviceID)
+	if err != nil {
+		glog.Infof("HasRdmaParam(): unable to get Netlink RDMA param for device %s : %q", r.deviceID, err)
+		return false
+	}
+	return rdma
+}
+
+func (r *rdmaSpec) getRdmaResources() []string {
+	//nolint: exhaustive
+	switch r.deviceType {
+	case types.NetDeviceType:
+		return utils.GetRdmaProvider().GetRdmaDevicesForPcidev(r.deviceID)
+	case types.AuxNetDeviceType:
+		return utils.GetRdmaProvider().GetRdmaDevicesForAuxdev(r.deviceID)
+	default:
+		return make([]string, 0)
+	}
+}
+
+func (r *rdmaSpec) GetRdmaDeviceSpec() []*pluginapi.DeviceSpec {
+	rdmaResources := r.getRdmaResources()
 	deviceSpec := make([]*pluginapi.DeviceSpec, 0)
-	isSupportRdma := false
 	if len(rdmaResources) > 0 {
-		isSupportRdma = true
 		for _, res := range rdmaResources {
 			resRdmaDevices := utils.GetRdmaProvider().GetRdmaCharDevices(res)
 			for _, rdmaDevice := range resRdmaDevices {
@@ -45,26 +78,5 @@ func newRdmaSpec(rdmaResources []string) types.RdmaSpec {
 			}
 		}
 	}
-
-	return &rdmaSpec{isSupportRdma: isSupportRdma, deviceSpec: deviceSpec}
-}
-
-// NewRdmaSpec returns the RdmaSpec for PCI address
-func NewRdmaSpec(pciAddr string) types.RdmaSpec {
-	rdmaResources := utils.GetRdmaProvider().GetRdmaDevicesForPcidev(pciAddr)
-	return newRdmaSpec(rdmaResources)
-}
-
-// NewAuxRdmaSpec returns the RdmaSpec for auxiliary device ID
-func NewAuxRdmaSpec(deviceID string) types.RdmaSpec {
-	rdmaResources := utils.GetRdmaProvider().GetRdmaDevicesForAuxdev(deviceID)
-	return newRdmaSpec(rdmaResources)
-}
-
-func (r *rdmaSpec) IsRdma() bool {
-	return r.isSupportRdma
-}
-
-func (r *rdmaSpec) GetRdmaDeviceSpec() []*pluginapi.DeviceSpec {
-	return r.deviceSpec
+	return deviceSpec
 }

pkg/devices/rdma_test.go

@@ -23,6 +23,7 @@ import (
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/devices"
+	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks"
 )
@@ -35,7 +36,7 @@ var _ = Describe("RdmaSpec", func() {
 				fakeRdmaProvider := mocks.RdmaProvider{}
 				fakeRdmaProvider.On("GetRdmaDevicesForPcidev", "0000:00:00.0").Return([]string{})
 				utils.SetRdmaProviderInst(&fakeRdmaProvider)
-				spec := devices.NewRdmaSpec("0000:00:00.0")
+				spec := devices.NewRdmaSpec(types.NetDeviceType, "0000:00:00.0")
 
 				Expect(spec.IsRdma()).To(BeFalse())
 				Expect(spec.GetRdmaDeviceSpec()).To(HaveLen(0))
@@ -50,7 +51,7 @@ var _ = Describe("RdmaSpec", func() {
 					"/dev/infiniband/uverbs0", "/dev/infiniband/rdma_cm",
 				}).On("GetRdmaCharDevices", "fake_1").Return([]string{"/dev/infiniband/rdma_cm"})
 				utils.SetRdmaProviderInst(&fakeRdmaProvider)
-				spec := devices.NewRdmaSpec("0000:00:00.0")
+				spec := devices.NewRdmaSpec(types.NetDeviceType, "0000:00:00.0")
 
 				Expect(spec.IsRdma()).To(BeTrue())
 				Expect(spec.GetRdmaDeviceSpec()).To(Equal([]*pluginapi.DeviceSpec{

pkg/factory/factory.go

@@ -163,15 +163,7 @@ func (rf *resourceFactory) GetResourcePool(rc *types.ResourceConfig, filteredDev
 }
 
 func (rf *resourceFactory) GetRdmaSpec(dt types.DeviceType, deviceID string) types.RdmaSpec {
-	//nolint: exhaustive
-	switch dt {
-	case types.NetDeviceType:
-		return devices.NewRdmaSpec(deviceID)
-	case types.AuxNetDeviceType:
-		return devices.NewAuxRdmaSpec(deviceID)
-	default:
-		return nil
-	}
+	return devices.NewRdmaSpec(dt, deviceID)
 }
 
 func (rf *resourceFactory) GetVdpaDevice(pciAddr string) types.VdpaDevice {

pkg/factory/factory_test.go

@@ -25,10 +25,12 @@ import (
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types/mocks"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
+	utilmocks "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/mock"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
@@ -606,6 +608,9 @@ var _ = Describe("Factory", func() {
 	)
 	Describe("getting rdma spec", func() {
 		Context("check c rdma spec", func() {
+			mockProvider := &utilmocks.NetlinkProvider{}
+			mockProvider.On("HasRdmaParam", mock.AnythingOfType("string")).Return(false, nil)
+			utils.SetNetlinkProviderInst(mockProvider)
 			f := factory.NewResourceFactory("fake", "fake", true, false)
 			rs1 := f.GetRdmaSpec(types.NetDeviceType, "0000:00:00.1")
 			rs2 := f.GetRdmaSpec(types.AcceleratorType, "0000:00:00.2")

pkg/utils/mocks/NetlinkProvider.go

@@ -31,6 +31,8 @@ type NetlinkProvider interface {
 	GetIPv4RouteList(ifName string) ([]nl.Route, error)
 	// DevlinkGetDeviceInfoByNameAsMap returns devlink info for selected device as a map
 	GetDevlinkGetDeviceInfoByNameAsMap(bus, device string) (map[string]string, error)
+	// HasRdmaParam returns true if PCI device has "enable_rdma" param
+	HasRdmaParam(pciAddr string) (bool, error)
 }
 
 type defaultNetlinkProvider struct {
@@ -48,6 +50,19 @@ func GetNetlinkProvider() NetlinkProvider {
 	return netlinkProvider
 }
 
+// HasRdmaParam returns true if PCI device has "enable_rdma" param
+// equivalent to "devlink dev param show pci/0000:d8:01.1 name enable_rdma"
+func (defaultNetlinkProvider) HasRdmaParam(pciAddr string) (bool, error) {
+	param, err := nl.DevlinkGetDeviceParamByName("pci", pciAddr, "enable_rdma")
+	if err != nil {
+		return false, fmt.Errorf("error getting enable_rdma attribute for pci device %s %v", pciAddr, err)
+	}
+	if len(param.Values) == 0 || param.Values[0].Data == nil {
+		return false, nil
+	}
+	return true, nil
+}
+
 // GetLinkAttrs returns a net device's link attributes.
 func (defaultNetlinkProvider) GetLinkAttrs(ifName string) (*nl.LinkAttrs, error) {
 	link, err := nl.LinkByName(ifName)

pkg/utils/netlink_provider.go

@@ -474,6 +474,15 @@ func GetPfEswitchMode(pciAddr string) (string, error) {
 	return devLinkDeviceAttrs.Mode, nil
 }
 
+// HasRdmaParam returns true if PCI device has "enable_rdma" param
+func HasRdmaParam(pciAddr string) (bool, error) {
+	rdma, err := GetNetlinkProvider().HasRdmaParam(pciAddr)
+	if err != nil {
+		return false, err
+	}
+	return rdma, nil
+}
+
 // HasDefaultRoute returns true if PCI network device is default route interface
 func HasDefaultRoute(pciAddr string) (bool, error) {
 	// Get net interface name