Skip to content

Commit f68d624

Browse files
nvidia-ci-cdnvidia-ci-cd
authored
Merge branch 'k8snetworkplumbingwg:master' into master
2 parents 50f51a6 + edc0ff6 commit f68d624

File tree

2 files changed

+191
-88
lines changed

2 files changed

+191
-88
lines changed

‎pkg/daemon/daemon.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -660,6 +660,10 @@ func (dn *NodeReconciler) prepareNMUdevRule() error {
660660

661661
// isDrainCompleted returns true if the current-state annotation is drain completed
662662
func (dn *NodeReconciler) isDrainCompleted(reqDrain bool, desiredNodeState *sriovnetworkv1.SriovNetworkNodeState) bool {
663+
if vars.DisableDrain {
664+
return true
665+
}
666+
663667
// if we need to drain check the drain status
664668
if reqDrain {
665669
return utils.ObjectHasAnnotation(desiredNodeState, consts.NodeStateDrainAnnotationCurrent, consts.DrainComplete)

‎pkg/daemon/daemon_test.go‎

Lines changed: 187 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"os"
2222
"sync"
23+
"sync/atomic"
2324
"time"
2425

2526
. "github.com/onsi/ginkgo/v2"
@@ -48,8 +49,6 @@ import (
4849
)
4950

5051
var (
51-
cancel context.CancelFunc
52-
ctx context.Context
5352
k8sManager manager.Manager
5453
kubeclient *kubernetes.Clientset
5554
eventRecorder *daemon.EventRecorder
@@ -60,6 +59,11 @@ var (
6059
mockCtrl *gomock.Controller
6160
hostHelper *mock_helper.MockHostHelpersInterface
6261
platformHelper *mock_platforms.MockInterface
62+
63+
discoverSriovReturn atomic.Pointer[[]sriovnetworkv1.InterfaceExt]
64+
nodeState *sriovnetworkv1.SriovNetworkNodeState
65+
66+
daemonReconciler *daemon.NodeReconciler
6367
)
6468

6569
const (
@@ -69,8 +73,12 @@ const (
6973

7074
var _ = Describe("Daemon Controller", Ordered, func() {
7175
BeforeAll(func() {
72-
ctx, cancel = context.WithCancel(context.Background())
7376
wg = sync.WaitGroup{}
77+
DeferCleanup(wg.Wait)
78+
79+
ctx, cancel := context.WithCancel(context.Background())
80+
DeferCleanup(cancel)
81+
7482
startDaemon = func(dc *daemon.NodeReconciler) {
7583
By("start controller manager")
7684
wg.Add(1)
@@ -108,9 +116,8 @@ var _ = Describe("Daemon Controller", Ordered, func() {
108116
} else {
109117
vars.ClusterType = constants.ClusterTypeKubernetes
110118
}
111-
})
112119

113-
BeforeEach(func() {
120+
By("Init mock functions")
114121
t = GinkgoT()
115122
mockCtrl = gomock.NewController(t)
116123
hostHelper = mock_helper.NewMockHostHelpersInterface(mockCtrl)
@@ -129,95 +136,71 @@ var _ = Describe("Daemon Controller", Ordered, func() {
129136
hostHelper.EXPECT().Chroot(gomock.Any()).Return(func() error { return nil }, nil).AnyTimes()
130137
hostHelper.EXPECT().RunCommand("/bin/sh", gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", nil).AnyTimes()
131138

132-
})
139+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{})
133140

134-
AfterEach(func() {
135-
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
141+
hostHelper.EXPECT().DiscoverSriovDevices(hostHelper).DoAndReturn(func(helpersInterface helper.HostHelpersInterface) ([]sriovnetworkv1.InterfaceExt, error) {
142+
return *discoverSriovReturn.Load(), nil
143+
}).AnyTimes()
144+
145+
hostHelper.EXPECT().LoadPfsStatus("0000:16:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes()
146+
147+
hostHelper.EXPECT().ClearPCIAddressFolder().Return(nil).AnyTimes()
148+
hostHelper.EXPECT().DiscoverRDMASubsystem().Return("shared", nil).AnyTimes()
149+
hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes()
150+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgPciRealloc).Return(true).AnyTimes()
151+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIntelIommu).Return(true).AnyTimes()
152+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIommuPt).Return(true).AnyTimes()
153+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaExclusive).Return(false).AnyTimes()
154+
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaShared).Return(false).AnyTimes()
155+
hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes()
136156

137-
By("Shutdown controller manager")
138-
cancel()
139-
wg.Wait()
157+
hostHelper.EXPECT().ConfigSriovInterfaces(gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).AnyTimes()
158+
159+
// k8s plugin for k8s cluster type
160+
if vars.ClusterType == constants.ClusterTypeKubernetes {
161+
hostHelper.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
162+
hostHelper.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
163+
}
164+
165+
featureGates := featuregate.New()
166+
featureGates.Init(map[string]bool{})
167+
daemonReconciler = createDaemon(hostHelper, platformHelper, featureGates, []string{})
168+
startDaemon(daemonReconciler)
169+
170+
_, nodeState = createNode("node1")
140171
})
141172

142173
AfterAll(func() {
174+
Expect(k8sClient.DeleteAllOf(context.Background(), &corev1.Node{})).ToNot(HaveOccurred())
175+
176+
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovNetworkNodeState{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
143177
Expect(k8sClient.DeleteAllOf(context.Background(), &sriovnetworkv1.SriovOperatorConfig{}, client.InNamespace(testNamespace))).ToNot(HaveOccurred())
144178
})
145179

146180
Context("Config Daemon generic flow", func() {
147-
BeforeEach(func() {
148-
// k8s plugin for k8s cluster type
149-
if vars.ClusterType == constants.ClusterTypeKubernetes {
150-
hostHelper.EXPECT().ReadServiceManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
151-
hostHelper.EXPECT().ReadServiceInjectionManifestFile(gomock.Any()).Return(&hostTypes.Service{Name: "test"}, nil).AnyTimes()
152-
}
153-
})
154181

155-
It("Should expose nodeState Status section", func() {
156-
By("Init mock functions")
157-
afterConfig := false
158-
hostHelper.EXPECT().DiscoverSriovDevices(hostHelper).DoAndReturn(func(helpersInterface helper.HostHelpersInterface) ([]sriovnetworkv1.InterfaceExt, error) {
159-
interfaceExtList := []sriovnetworkv1.InterfaceExt{
160-
{
161-
Name: "eno1",
162-
Driver: "ice",
163-
PciAddress: "0000:16:00.0",
164-
DeviceID: "1593",
165-
Vendor: "8086",
166-
EswitchMode: "legacy",
167-
LinkAdminState: "up",
168-
LinkSpeed: "10000 Mb/s",
169-
LinkType: "ETH",
170-
Mac: "aa:bb:cc:dd:ee:ff",
171-
Mtu: 1500,
172-
TotalVfs: 2,
173-
NumVfs: 0,
174-
},
175-
}
176-
177-
if afterConfig {
178-
interfaceExtList[0].NumVfs = 2
179-
interfaceExtList[0].VFs = []sriovnetworkv1.VirtualFunction{
180-
{
181-
Name: "eno1f0",
182-
PciAddress: "0000:16:00.1",
183-
VfID: 0,
184-
},
185-
{
186-
Name: "eno1f1",
187-
PciAddress: "0000:16:00.2",
188-
VfID: 1,
189-
}}
190-
}
191-
return interfaceExtList, nil
192-
}).AnyTimes()
193-
194-
hostHelper.EXPECT().LoadPfsStatus("0000:16:00.0").Return(&sriovnetworkv1.Interface{ExternallyManaged: false}, true, nil).AnyTimes()
195-
196-
hostHelper.EXPECT().ClearPCIAddressFolder().Return(nil).AnyTimes()
197-
hostHelper.EXPECT().DiscoverRDMASubsystem().Return("shared", nil).AnyTimes()
198-
hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes()
199-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgPciRealloc).Return(true).AnyTimes()
200-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIntelIommu).Return(true).AnyTimes()
201-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgIommuPt).Return(true).AnyTimes()
202-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaExclusive).Return(false).AnyTimes()
203-
hostHelper.EXPECT().IsKernelArgsSet("", constants.KernelArgRdmaShared).Return(false).AnyTimes()
204-
hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes()
205-
206-
hostHelper.EXPECT().ConfigSriovInterfaces(gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).AnyTimes()
207-
208-
featureGates := featuregate.New()
209-
featureGates.Init(map[string]bool{})
210-
dc := createDaemon(hostHelper, platformHelper, featureGates, []string{})
211-
startDaemon(dc)
212-
213-
_, nodeState := createNode("node1")
214-
By("waiting for state to be succeeded")
215-
EventuallyWithOffset(1, func(g Gomega) {
216-
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
217-
ToNot(HaveOccurred())
182+
It("Should expose nodeState Status section", func(ctx context.Context) {
183+
184+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
185+
{
186+
Name: "eno1",
187+
Driver: "ice",
188+
PciAddress: "0000:16:00.0",
189+
DeviceID: "1593",
190+
Vendor: "8086",
191+
EswitchMode: "legacy",
192+
LinkAdminState: "up",
193+
LinkSpeed: "10000 Mb/s",
194+
LinkType: "ETH",
195+
Mac: "aa:bb:cc:dd:ee:ff",
196+
Mtu: 1500,
197+
TotalVfs: 2,
198+
NumVfs: 0,
199+
},
200+
})
218201

219-
g.Expect(nodeState.Status.SyncStatus).To(Equal(constants.SyncStatusSucceeded))
220-
}, waitTime, retryTime).Should(Succeed())
202+
By("waiting for state to be succeeded")
203+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
221204

222205
By("add spec to node state")
223206
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
@@ -235,14 +218,44 @@ var _ = Describe("Daemon Controller", Ordered, func() {
235218
VfRange: "eno1#0-1"},
236219
}},
237220
}
238-
afterConfig = true
221+
222+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
223+
{
224+
Name: "eno1",
225+
Driver: "ice",
226+
PciAddress: "0000:16:00.0",
227+
DeviceID: "1593",
228+
Vendor: "8086",
229+
EswitchMode: "legacy",
230+
LinkAdminState: "up",
231+
LinkSpeed: "10000 Mb/s",
232+
LinkType: "ETH",
233+
Mac: "aa:bb:cc:dd:ee:ff",
234+
Mtu: 1500,
235+
TotalVfs: 2,
236+
NumVfs: 2,
237+
VFs: []sriovnetworkv1.VirtualFunction{
238+
{
239+
Name: "eno1f0",
240+
PciAddress: "0000:16:00.1",
241+
VfID: 0,
242+
},
243+
{
244+
Name: "eno1f1",
245+
PciAddress: "0000:16:00.2",
246+
VfID: 1,
247+
}},
248+
},
249+
})
250+
239251
err = k8sClient.Update(ctx, nodeState)
240252
Expect(err).ToNot(HaveOccurred())
253+
241254
By("waiting to require drain")
242255
EventuallyWithOffset(1, func(g Gomega) {
243256
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
244257
ToNot(HaveOccurred())
245-
g.Expect(dc.GetLastAppliedGeneration()).To(Equal(int64(2)))
258+
g.Expect(daemonReconciler.GetLastAppliedGeneration()).To(Equal(int64(2)))
246259
}, waitTime, retryTime).Should(Succeed())
247260

248261
err = k8sClient.Get(ctx, types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)
@@ -278,13 +291,72 @@ var _ = Describe("Daemon Controller", Ordered, func() {
278291

279292
Expect(nodeState.Status.LastSyncError).To(Equal(""))
280293
})
294+
295+
It("Should apply the reset configuration when disableDrain is true", func(ctx context.Context) {
296+
DeferCleanup(func(x bool) { vars.DisableDrain = x }, vars.DisableDrain)
297+
vars.DisableDrain = true
298+
299+
discoverSriovReturn.Store(&[]sriovnetworkv1.InterfaceExt{
300+
{
301+
Name: "eno1",
302+
Driver: "ice",
303+
PciAddress: "0000:16:00.0",
304+
DeviceID: "1593",
305+
Vendor: "8086",
306+
EswitchMode: "legacy",
307+
LinkAdminState: "up",
308+
LinkSpeed: "10000 Mb/s",
309+
LinkType: "ETH",
310+
Mac: "aa:bb:cc:dd:ee:ff",
311+
Mtu: 1500,
312+
TotalVfs: 2,
313+
NumVfs: 2,
314+
VFs: []sriovnetworkv1.VirtualFunction{
315+
{
316+
Name: "eno1f0",
317+
PciAddress: "0000:16:00.1",
318+
VfID: 0,
319+
},
320+
{
321+
Name: "eno1f1",
322+
PciAddress: "0000:16:00.2",
323+
VfID: 1,
324+
}},
325+
},
326+
})
327+
328+
nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{
329+
{Name: "eno1",
330+
PciAddress: "0000:16:00.0",
331+
LinkType: "eth",
332+
NumVfs: 2,
333+
VfGroups: []sriovnetworkv1.VfGroup{
334+
{ResourceName: "test",
335+
DeviceType: "netdevice",
336+
PolicyName: "test-policy",
337+
VfRange: "eno1#0-1"},
338+
}},
339+
}
340+
err := k8sClient.Update(ctx, nodeState)
341+
Expect(err).ToNot(HaveOccurred())
342+
343+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
344+
345+
By("Simulate node policy removal")
346+
nodeState.Spec.Interfaces = []sriovnetworkv1.Interface{}
347+
err = k8sClient.Update(ctx, nodeState)
348+
Expect(err).ToNot(HaveOccurred())
349+
350+
eventuallySyncStatusEqual(nodeState, constants.SyncStatusSucceeded)
351+
assertLastStatusTransitionsContains(nodeState, 2, constants.SyncStatusInProgress)
352+
})
281353
})
282354
})
283355

284356
func patchAnnotation(nodeState *sriovnetworkv1.SriovNetworkNodeState, key, value string) {
285357
originalNodeState := nodeState.DeepCopy()
286358
nodeState.Annotations[key] = value
287-
err := k8sClient.Patch(ctx, nodeState, client.MergeFrom(originalNodeState))
359+
err := k8sClient.Patch(context.Background(), nodeState, client.MergeFrom(originalNodeState))
288360
Expect(err).ToNot(HaveOccurred())
289361
}
290362

@@ -313,8 +385,8 @@ func createNode(nodeName string) (*corev1.Node, *sriovnetworkv1.SriovNetworkNode
313385
},
314386
}
315387

316-
Expect(k8sClient.Create(ctx, &node)).ToNot(HaveOccurred())
317-
Expect(k8sClient.Create(ctx, &nodeState)).ToNot(HaveOccurred())
388+
Expect(k8sClient.Create(context.Background(), &node)).ToNot(HaveOccurred())
389+
Expect(k8sClient.Create(context.Background(), &nodeState)).ToNot(HaveOccurred())
318390
vars.NodeName = nodeName
319391

320392
return &node, &nodeState
@@ -345,3 +417,30 @@ func createDaemon(
345417

346418
return configController
347419
}
420+
421+
func eventuallySyncStatusEqual(nodeState *sriovnetworkv1.SriovNetworkNodeState, expectedState string) {
422+
EventuallyWithOffset(1, func(g Gomega) {
423+
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Namespace: nodeState.Namespace, Name: nodeState.Name}, nodeState)).
424+
ToNot(HaveOccurred())
425+
g.Expect(nodeState.Status.SyncStatus).To(Equal(expectedState))
426+
}, 10*time.Second, 100*time.Millisecond).Should(Succeed())
427+
}
428+
429+
func assertLastStatusTransitionsContains(nodeState *sriovnetworkv1.SriovNetworkNodeState, numberOfTransitions int, status string) {
430+
events := &corev1.EventList{}
431+
err := k8sClient.List(
432+
context.Background(),
433+
events,
434+
client.MatchingFields{
435+
"involvedObject.name": nodeState.Name,
436+
"reason": "SyncStatusChanged",
437+
},
438+
client.Limit(numberOfTransitions),
439+
)
440+
Expect(err).ToNot(HaveOccurred())
441+
442+
// Status transition events are in the form
443+
// `Status changed from: Succeed to: InProgress`
444+
Expect(events.Items).To(ContainElement(
445+
HaveField("Message", ContainSubstring("to: "+status))))
446+
}

0 commit comments

Comments
 (0)