Skip to content

Commit d828dd8

Browse files
nvidia-ci-cdnvidia-ci-cd
authored
Merge branch 'k8snetworkplumbingwg:master' into master
2 parents 7bf8eea + 023a4b3 commit d828dd8

File tree

9 files changed

+313
-43
lines changed

9 files changed

+313
-43
lines changed

‎pkg/helper/host.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@ type hostHelpers struct {
4242

4343
func NewDefaultHostHelpers() (HostHelpersInterface, error) {
4444
utilsHelper := utils.New()
45-
mlxHelper := mlx.New(utilsHelper)
4645
hostManager, err := host.NewHostManager(utilsHelper)
4746
if err != nil {
4847
log.Log.Error(err, "failed to create host manager")
4948
return nil, err
5049
}
50+
mlxHelper := mlx.New(utilsHelper, hostManager)
5151
storeManager, err := store.NewManager()
5252
if err != nil {
5353
log.Log.Error(err, "failed to create store manager")

‎pkg/helper/mock/mock_helper.go‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/host/internal/network/network_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ import (
2828
"go.uber.org/mock/gomock"
2929

3030
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
31-
hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock"
3231
dputilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils/mock"
3332
ethtoolMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool/mock"
3433
netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock"
3534
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
35+
utilsMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils/mock"
3636
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem"
3737
"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers"
3838
)
@@ -52,7 +52,7 @@ var _ = Describe("Network", func() {
5252
netlinkLibMock *netlinkMockPkg.MockNetlinkLib
5353
ethtoolLibMock *ethtoolMockPkg.MockEthtoolLib
5454
dputilsLibMock *dputilsMockPkg.MockDPUtilsLib
55-
hostMock *hostMockPkg.MockHostHelpersInterface
55+
utilsMock *utilsMockPkg.MockCmdInterface
5656

5757
testCtrl *gomock.Controller
5858
testErr = fmt.Errorf("test")
@@ -62,9 +62,9 @@ var _ = Describe("Network", func() {
6262
netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl)
6363
ethtoolLibMock = ethtoolMockPkg.NewMockEthtoolLib(testCtrl)
6464
dputilsLibMock = dputilsMockPkg.NewMockDPUtilsLib(testCtrl)
65-
hostMock = hostMockPkg.NewMockHostHelpersInterface(testCtrl)
65+
utilsMock = utilsMockPkg.NewMockCmdInterface(testCtrl)
6666

67-
n = New(hostMock, dputilsLibMock, netlinkLibMock, ethtoolLibMock)
67+
n = New(utilsMock, dputilsLibMock, netlinkLibMock, ethtoolLibMock)
6868
})
6969

7070
AfterEach(func() {

‎pkg/plugins/mellanox/mellanox_plugin.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ func (p *MellanoxPlugin) Apply() error {
222222
return err
223223
}
224224
if vars.FeatureGate.IsEnabled(consts.MellanoxFirmwareResetFeatureGate) {
225-
return p.helpers.MlxResetFW(pciAddressesToReset)
225+
return p.helpers.MlxResetFW(pciAddressesToReset, mellanoxNicsStatus)
226226
}
227227
return nil
228228
}

‎pkg/plugins/mellanox/mellanox_plugin_test.go‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,9 +366,85 @@ var _ = Describe("SRIOV", Ordered, func() {
366366
vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true})
367367
h.EXPECT().IsKernelLockdownMode().Return(false)
368368
h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil)
369-
h.EXPECT().MlxResetFW(gomock.Any()).Return(nil)
369+
h.EXPECT().MlxResetFW(gomock.Any(), gomock.Any()).Return(nil)
370370
err := m.Apply()
371371
Expect(err).ToNot(HaveOccurred())
372372
})
373+
374+
It("should reset VFs on both ports for dual-port NIC when firmware reset is required", func() {
375+
vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true})
376+
377+
// Setup dual-port NIC status
378+
mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{
379+
"0000:d8:00.": {
380+
"0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"},
381+
"0000:d8:00.1": {PciAddress: "0000:d8:00.1", Vendor: "15b3"},
382+
},
383+
}
384+
385+
// Setup dual-port NIC spec
386+
mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{
387+
"0000:d8:00.0": {PciAddress: "0000:d8:00.0", NumVfs: 10},
388+
"0000:d8:00.1": {PciAddress: "0000:d8:00.1", NumVfs: 8},
389+
}
390+
391+
// Only one port needs firmware reset
392+
pciAddressesToReset = []string{"0000:d8:00.0"}
393+
394+
h.EXPECT().IsKernelLockdownMode().Return(false)
395+
h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil)
396+
397+
h.EXPECT().MlxResetFW([]string{"0000:d8:00.0"}, gomock.Any()).Return(nil)
398+
399+
err := m.Apply()
400+
Expect(err).ToNot(HaveOccurred())
401+
})
402+
403+
It("should reset VFs on single port NIC when firmware reset is required", func() {
404+
vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true})
405+
406+
// Setup single-port NIC status
407+
mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{
408+
"0000:d8:00.": {
409+
"0000:d8:00.0": {PciAddress: "0000:d8:00.0", Vendor: "15b3"},
410+
},
411+
}
412+
413+
// Setup single-port NIC spec
414+
mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{
415+
"0000:d8:00.0": {PciAddress: "0000:d8:00.0", NumVfs: 10},
416+
}
417+
418+
pciAddressesToReset = []string{"0000:d8:00.0"}
419+
420+
h.EXPECT().IsKernelLockdownMode().Return(false)
421+
h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil)
422+
423+
h.EXPECT().MlxResetFW([]string{"0000:d8:00.0"}, gomock.Any()).Return(nil)
424+
425+
err := m.Apply()
426+
Expect(err).ToNot(HaveOccurred())
427+
})
428+
429+
It("should call MlxResetFW with correct parameters for multiple NICs", func() {
430+
vars.FeatureGate.Init(map[string]bool{consts.MellanoxFirmwareResetFeatureGate: true})
431+
432+
pciAddressesToReset = []string{"0000:d8:00.0", "0000:d9:00.0"}
433+
434+
h.EXPECT().IsKernelLockdownMode().Return(false)
435+
h.EXPECT().MlxConfigFW(gomock.Any()).Return(nil)
436+
437+
h.EXPECT().MlxResetFW([]string{"0000:d8:00.0", "0000:d9:00.0"}, gomock.Any()).Return(nil)
438+
439+
err := m.Apply()
440+
Expect(err).ToNot(HaveOccurred())
441+
})
442+
443+
BeforeEach(func() {
444+
// Reset global state before each test
445+
pciAddressesToReset = []string{}
446+
mellanoxNicsStatus = map[string]map[string]sriovnetworkv1.InterfaceExt{}
447+
mellanoxNicsSpec = map[string]sriovnetworkv1.Interface{}
448+
})
373449
})
374450
})

‎pkg/vendors/mellanox/mellanox.go‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
3030
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
31+
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
3132
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
3233
)
3334

@@ -78,16 +79,18 @@ type MellanoxInterface interface {
7879
GetMlxNicFwData(pciAddress string) (current, next *MlxNic, err error)
7980

8081
MlxConfigFW(attributesToChange map[string]MlxNic) error
81-
MlxResetFW(pciAddresses []string) error
82+
MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) error
8283
}
8384

8485
type mellanoxHelper struct {
85-
utils utils.CmdInterface
86+
utils utils.CmdInterface
87+
hostHelper host.HostManagerInterface
8688
}
8789

88-
func New(utilsHelper utils.CmdInterface) MellanoxInterface {
90+
func New(utilsHelper utils.CmdInterface, hostHelper host.HostManagerInterface) MellanoxInterface {
8991
return &mellanoxHelper{
90-
utils: utilsHelper,
92+
utils: utilsHelper,
93+
hostHelper: hostHelper,
9194
}
9295
}
9396

@@ -160,10 +163,25 @@ func (m *mellanoxHelper) GetMellanoxBlueFieldMode(PciAddress string) (BlueFieldM
160163
return -1, fmt.Errorf("MellanoxBlueFieldMode(): unknown device status for %s", PciAddress)
161164
}
162165

163-
func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error {
166+
func (m *mellanoxHelper) MlxResetFW(pciAddresses []string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) error {
164167
log.Log.Info("mellanox-plugin resetFW()")
165168
var errs []error
166169
for _, pciAddress := range pciAddresses {
170+
err := m.hostHelper.SetSriovNumVfs(pciAddress, 0)
171+
if err != nil {
172+
log.Log.Error(err, "failed to set SR-IOV number of VFs to 0 before firmware reset", "pciAddress", pciAddress)
173+
return err
174+
}
175+
176+
if IsDualPort(pciAddress, mellanoxNicsStatus) {
177+
otherPortPCIAddress := getOtherPortPCIAddress(pciAddress)
178+
err := m.hostHelper.SetSriovNumVfs(otherPortPCIAddress, 0)
179+
if err != nil {
180+
log.Log.Error(err, "failed to set SR-IOV number of VFs to 0 before firmware reset", "pciAddress", otherPortPCIAddress)
181+
return err
182+
}
183+
}
184+
167185
cmdArgs := []string{"-d", pciAddress, "--skip_driver", "-l", "3", "-y", "reset"}
168186
log.Log.Info("mellanox-plugin: resetFW()", "cmd-args", cmdArgs)
169187
// We have to ensure that pciutils is installed into the container image Dockerfile.sriov-network-config-daemon
@@ -173,6 +191,7 @@ func (m *mellanoxHelper) MlxResetFW(pciAddresses []string) error {
173191
errs = append(errs, err)
174192
}
175193
}
194+
176195
return kerrors.NewAggregate(errs)
177196
}
178197

@@ -274,8 +293,9 @@ func HandleTotalVfs(fwCurrent, fwNext, attrs *MlxNic, ifaceSpec sriovnetworkv1.I
274293
totalVfs = ifaceSpec.NumVfs
275294
// Check if the other port is changing the number of VF
276295
if isDualPort {
277-
otherIfaceSpec := getOtherPortSpec(ifaceSpec.PciAddress, mellanoxNicsSpec)
278-
if otherIfaceSpec != nil {
296+
otherPortPCIAddress := getOtherPortPCIAddress(ifaceSpec.PciAddress)
297+
otherIfaceSpec, ok := mellanoxNicsSpec[otherPortPCIAddress]
298+
if ok {
279299
if otherIfaceSpec.NumVfs > totalVfs {
280300
totalVfs = otherIfaceSpec.NumVfs
281301
}
@@ -428,18 +448,16 @@ func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovne
428448
return false, nil
429449
}
430450

431-
func getOtherPortSpec(pciAddress string, mellanoxNicsSpec map[string]sriovnetworkv1.Interface) *sriovnetworkv1.Interface {
451+
func getOtherPortPCIAddress(pciAddress string) string {
432452
log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress)
433453
pciAddrPrefix := GetPciAddressPrefix(pciAddress)
434454
pciAddrSuffix := pciAddress[len(pciAddrPrefix):]
435455

436456
if pciAddrSuffix == "0" {
437-
iface := mellanoxNicsSpec[pciAddrPrefix+"1"]
438-
return &iface
457+
return pciAddrPrefix + "1"
439458
}
440459

441-
iface := mellanoxNicsSpec[pciAddrPrefix+"0"]
442-
return &iface
460+
return pciAddrPrefix + "0"
443461
}
444462

445463
func getIfaceStatus(pciAddress string, mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) sriovnetworkv1.InterfaceExt {

0 commit comments

Comments
 (0)