Skip to content

Commit 9bd805e

Browse files
rollandfrollandf
committed
bug: rdma exlusive handling
In case a RDMA device in exclusive mode is in use by a Pod, the DP was not reporting it as a resource after DP restart. Following changes are introduced in RdmaSpec: - isRdma: in case of no rdma resources, check if netlink "enable_rdma" is available. - GetRdmaDeviceSpec: the device specs are retrieved dynamically and not on discovery stage as before. Dynamic RDMA specs computation vs on discovery, comes to solve following scenario for exlusive mode: - Discover RDMA device - Allocate to Pod (resources are hidden on host) - Restart DP pod - Deallocate - Reallocate Fixes #565 Signed-off-by: Fred Rolland <[email protected]>
1 parent 39a434e commit 9bd805e

File tree

7 files changed

+165
-41
lines changed

7 files changed

+165
-41
lines changed

‎pkg/devices/rdma.go‎

Lines changed: 49 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,65 @@
1818
package devices
1919

2020
import (
21+
"github.com/golang/glog"
2122
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
2223

2324
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
2425
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
2526
)
2627

2728
type rdmaSpec struct {
28-
isSupportRdma bool
29-
deviceSpec []*pluginapi.DeviceSpec
29+
deviceID string
30+
deviceType types.DeviceType
3031
}
3132

32-
func newRdmaSpec(rdmaResources []string) types.RdmaSpec {
33+
// NewRdmaSpec returns the RdmaSpec
34+
func NewRdmaSpec(dt types.DeviceType, id string) types.RdmaSpec {
35+
if dt == types.AcceleratorType {
36+
return nil
37+
}
38+
return &rdmaSpec{deviceID: id, deviceType: dt}
39+
}
40+
41+
func (r *rdmaSpec) IsRdma() bool {
42+
if len(r.getRdmaResources()) > 0 {
43+
return true
44+
}
45+
var bus string
46+
//nolint: exhaustive
47+
switch r.deviceType {
48+
case types.NetDeviceType:
49+
bus = "pci"
50+
case types.AuxNetDeviceType:
51+
bus = "auxiliary"
52+
default:
53+
return false
54+
}
55+
// Checking for netlink param for exclusive RDMA use case
56+
rdma, err := utils.HasRdmaParam(bus, r.deviceID)
57+
if err != nil {
58+
glog.Infof("HasRdmaParam(): unable to get Netlink RDMA param for device %s : %q", r.deviceID, err)
59+
return false
60+
}
61+
return rdma
62+
}
63+
64+
func (r *rdmaSpec) getRdmaResources() []string {
65+
//nolint: exhaustive
66+
switch r.deviceType {
67+
case types.NetDeviceType:
68+
return utils.GetRdmaProvider().GetRdmaDevicesForPcidev(r.deviceID)
69+
case types.AuxNetDeviceType:
70+
return utils.GetRdmaProvider().GetRdmaDevicesForAuxdev(r.deviceID)
71+
default:
72+
return make([]string, 0)
73+
}
74+
}
75+
76+
func (r *rdmaSpec) GetRdmaDeviceSpec() []*pluginapi.DeviceSpec {
77+
rdmaResources := r.getRdmaResources()
3378
deviceSpec := make([]*pluginapi.DeviceSpec, 0)
34-
isSupportRdma := false
3579
if len(rdmaResources) > 0 {
36-
isSupportRdma = true
3780
for _, res := range rdmaResources {
3881
resRdmaDevices := utils.GetRdmaProvider().GetRdmaCharDevices(res)
3982
for _, rdmaDevice := range resRdmaDevices {
@@ -45,26 +88,5 @@ func newRdmaSpec(rdmaResources []string) types.RdmaSpec {
4588
}
4689
}
4790
}
48-
49-
return &rdmaSpec{isSupportRdma: isSupportRdma, deviceSpec: deviceSpec}
50-
}
51-
52-
// NewRdmaSpec returns the RdmaSpec for PCI address
53-
func NewRdmaSpec(pciAddr string) types.RdmaSpec {
54-
rdmaResources := utils.GetRdmaProvider().GetRdmaDevicesForPcidev(pciAddr)
55-
return newRdmaSpec(rdmaResources)
56-
}
57-
58-
// NewAuxRdmaSpec returns the RdmaSpec for auxiliary device ID
59-
func NewAuxRdmaSpec(deviceID string) types.RdmaSpec {
60-
rdmaResources := utils.GetRdmaProvider().GetRdmaDevicesForAuxdev(deviceID)
61-
return newRdmaSpec(rdmaResources)
62-
}
63-
64-
func (r *rdmaSpec) IsRdma() bool {
65-
return r.isSupportRdma
66-
}
67-
68-
func (r *rdmaSpec) GetRdmaDeviceSpec() []*pluginapi.DeviceSpec {
69-
return r.deviceSpec
91+
return deviceSpec
7092
}

‎pkg/devices/rdma_test.go‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
2424

2525
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/devices"
26+
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
2627
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
2728
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks"
2829
)
@@ -31,16 +32,58 @@ var _ = Describe("RdmaSpec", func() {
3132
Describe("creating new RdmaSpec", func() {
3233
t := GinkgoT()
3334
Context("successfully", func() {
34-
It("without device specs", func() {
35+
It("without device specs, without netlink enable_rdma param", func() {
36+
mockProvider := &mocks.NetlinkProvider{}
37+
mockProvider.On("HasRdmaParam", "pci", "0000:00:00.0").Return(false, nil)
38+
utils.SetNetlinkProviderInst(mockProvider)
3539
fakeRdmaProvider := mocks.RdmaProvider{}
3640
fakeRdmaProvider.On("GetRdmaDevicesForPcidev", "0000:00:00.0").Return([]string{})
3741
utils.SetRdmaProviderInst(&fakeRdmaProvider)
38-
spec := devices.NewRdmaSpec("0000:00:00.0")
42+
spec := devices.NewRdmaSpec(types.NetDeviceType, "0000:00:00.0")
3943

4044
Expect(spec.IsRdma()).To(BeFalse())
4145
Expect(spec.GetRdmaDeviceSpec()).To(HaveLen(0))
4246
fakeRdmaProvider.AssertExpectations(t)
4347
})
48+
It("without device specs, with netlink enable_rdma param", func() {
49+
mockProvider := &mocks.NetlinkProvider{}
50+
mockProvider.On("HasRdmaParam", "pci", "0000:00:00.0").Return(true, nil)
51+
utils.SetNetlinkProviderInst(mockProvider)
52+
fakeRdmaProvider := mocks.RdmaProvider{}
53+
fakeRdmaProvider.On("GetRdmaDevicesForPcidev", "0000:00:00.0").Return([]string{})
54+
utils.SetRdmaProviderInst(&fakeRdmaProvider)
55+
spec := devices.NewRdmaSpec(types.NetDeviceType, "0000:00:00.0")
56+
57+
Expect(spec.IsRdma()).To(BeTrue())
58+
Expect(spec.GetRdmaDeviceSpec()).To(HaveLen(0))
59+
fakeRdmaProvider.AssertExpectations(t)
60+
})
61+
It("aux without device specs, without netlink enable_rdma param", func() {
62+
mockProvider := &mocks.NetlinkProvider{}
63+
mockProvider.On("HasRdmaParam", "auxiliary", "mlx5_core.sf.4").Return(false, nil)
64+
utils.SetNetlinkProviderInst(mockProvider)
65+
fakeRdmaProvider := mocks.RdmaProvider{}
66+
fakeRdmaProvider.On("GetRdmaDevicesForAuxdev", "mlx5_core.sf.4").Return([]string{})
67+
utils.SetRdmaProviderInst(&fakeRdmaProvider)
68+
spec := devices.NewRdmaSpec(types.AuxNetDeviceType, "mlx5_core.sf.4")
69+
70+
Expect(spec.IsRdma()).To(BeFalse())
71+
Expect(spec.GetRdmaDeviceSpec()).To(HaveLen(0))
72+
fakeRdmaProvider.AssertExpectations(t)
73+
})
74+
It("aux without device specs, with netlink enable_rdma param", func() {
75+
mockProvider := &mocks.NetlinkProvider{}
76+
mockProvider.On("HasRdmaParam", "auxiliary", "mlx5_core.sf.4").Return(true, nil)
77+
utils.SetNetlinkProviderInst(mockProvider)
78+
fakeRdmaProvider := mocks.RdmaProvider{}
79+
fakeRdmaProvider.On("GetRdmaDevicesForAuxdev", "mlx5_core.sf.4").Return([]string{})
80+
utils.SetRdmaProviderInst(&fakeRdmaProvider)
81+
spec := devices.NewRdmaSpec(types.AuxNetDeviceType, "mlx5_core.sf.4")
82+
83+
Expect(spec.IsRdma()).To(BeTrue())
84+
Expect(spec.GetRdmaDeviceSpec()).To(HaveLen(0))
85+
fakeRdmaProvider.AssertExpectations(t)
86+
})
4487
It("with device specs", func() {
4588
fakeRdmaProvider := mocks.RdmaProvider{}
4689
fakeRdmaProvider.On("GetRdmaDevicesForPcidev", "0000:00:00.0").
@@ -50,7 +93,7 @@ var _ = Describe("RdmaSpec", func() {
5093
"/dev/infiniband/uverbs0", "/dev/infiniband/rdma_cm",
5194
}).On("GetRdmaCharDevices", "fake_1").Return([]string{"/dev/infiniband/rdma_cm"})
5295
utils.SetRdmaProviderInst(&fakeRdmaProvider)
53-
spec := devices.NewRdmaSpec("0000:00:00.0")
96+
spec := devices.NewRdmaSpec(types.NetDeviceType, "0000:00:00.0")
5497

5598
Expect(spec.IsRdma()).To(BeTrue())
5699
Expect(spec.GetRdmaDeviceSpec()).To(Equal([]*pluginapi.DeviceSpec{

‎pkg/factory/factory.go‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -163,15 +163,7 @@ func (rf *resourceFactory) GetResourcePool(rc *types.ResourceConfig, filteredDev
163163
}
164164

165165
func (rf *resourceFactory) GetRdmaSpec(dt types.DeviceType, deviceID string) types.RdmaSpec {
166-
//nolint: exhaustive
167-
switch dt {
168-
case types.NetDeviceType:
169-
return devices.NewRdmaSpec(deviceID)
170-
case types.AuxNetDeviceType:
171-
return devices.NewAuxRdmaSpec(deviceID)
172-
default:
173-
return nil
174-
}
166+
return devices.NewRdmaSpec(dt, deviceID)
175167
}
176168

177169
func (rf *resourceFactory) GetVdpaDevice(pciAddr string) types.VdpaDevice {

‎pkg/factory/factory_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@ import (
2525
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
2626
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types/mocks"
2727
"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
28+
utilmocks "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks"
2829

2930
. "github.com/onsi/ginkgo"
3031
. "github.com/onsi/ginkgo/extensions/table"
3132
. "github.com/onsi/gomega"
33+
"github.com/stretchr/testify/mock"
3234
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
3335
)
3436

@@ -606,6 +608,10 @@ var _ = Describe("Factory", func() {
606608
)
607609
Describe("getting rdma spec", func() {
608610
Context("check c rdma spec", func() {
611+
mockProvider := &utilmocks.NetlinkProvider{}
612+
mockProvider.On("HasRdmaParam", mock.AnythingOfType("string"),
613+
mock.AnythingOfType("string")).Return(false, nil)
614+
utils.SetNetlinkProviderInst(mockProvider)
609615
f := factory.NewResourceFactory("fake", "fake", true, false)
610616
rs1 := f.GetRdmaSpec(types.NetDeviceType, "0000:00:00.1")
611617
rs2 := f.GetRdmaSpec(types.AcceleratorType, "0000:00:00.2")

‎pkg/utils/mocks/NetlinkProvider.go‎

Lines changed: 30 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/utils/netlink_provider.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ type NetlinkProvider interface {
3131
GetIPv4RouteList(ifName string) ([]nl.Route, error)
3232
// DevlinkGetDeviceInfoByNameAsMap returns devlink info for selected device as a map
3333
GetDevlinkGetDeviceInfoByNameAsMap(bus, device string) (map[string]string, error)
34+
// HasRdmaParam returns true if device has "enable_rdma" param
35+
HasRdmaParam(bus, pciAddr string) (bool, error)
3436
}
3537

3638
type defaultNetlinkProvider struct {
@@ -48,6 +50,26 @@ func GetNetlinkProvider() NetlinkProvider {
4850
return netlinkProvider
4951
}
5052

53+
// HasRdmaParam returns true if device has "enable_rdma" param
54+
// equivalent to "devlink dev param show pci/0000:d8:01.1 name enable_rdma"
55+
// or "devlink dev param show auxiliary/mlx5_core.sf.4 name enable_rdma"
56+
func (defaultNetlinkProvider) HasRdmaParam(bus, deviceID string) (bool, error) {
57+
param, err := nl.DevlinkGetDeviceParamByName(bus, deviceID, "enable_rdma")
58+
if err != nil {
59+
return false, fmt.Errorf("error getting enable_rdma attribute for device %s on bus %s %v",
60+
deviceID, bus, err)
61+
}
62+
if len(param.Values) == 0 || param.Values[0].Data == nil {
63+
return false, nil
64+
}
65+
var boolValue bool
66+
boolValue, ok := param.Values[0].Data.(bool)
67+
if !ok {
68+
return false, fmt.Errorf("value is not a bool")
69+
}
70+
return boolValue, nil
71+
}
72+
5173
// GetLinkAttrs returns a net device's link attributes.
5274
func (defaultNetlinkProvider) GetLinkAttrs(ifName string) (*nl.LinkAttrs, error) {
5375
link, err := nl.LinkByName(ifName)

‎pkg/utils/utils.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,17 @@ func GetPfEswitchMode(pciAddr string) (string, error) {
474474
return devLinkDeviceAttrs.Mode, nil
475475
}
476476

477+
// HasRdmaParam returns true if deviceID has "enable_rdma" param
478+
// for example: pci 0000:d8:01.1
479+
// or auxiliary mlx5_core.sf.4
480+
func HasRdmaParam(bus, deviceID string) (bool, error) {
481+
rdma, err := GetNetlinkProvider().HasRdmaParam(bus, deviceID)
482+
if err != nil {
483+
return false, err
484+
}
485+
return rdma, nil
486+
}
487+
477488
// HasDefaultRoute returns true if PCI network device is default route interface
478489
func HasDefaultRoute(pciAddr string) (bool, error) {
479490
// Get net interface name

0 commit comments

Comments
 (0)