Skip to content

Commit c7d29a9

Browse files
almaslennikovalmaslennikov
authored
Merge pull request #202 from almaslennikov/multiplane
feat: support SPC-X RA v2.1 with multiplane configurations
2 parents 90b8fde + 28ce40f commit c7d29a9

File tree

15 files changed

+1952
-70
lines changed

15 files changed

+1952
-70
lines changed

‎api/v1alpha1/nicconfigurationtemplate_types.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,14 @@ type GpuDirectOptimizedSpec struct {
7373
}
7474

7575
// SpectrumXOptimizedSpec enables Spectrum-X specific optimizations
76+
// +kubebuilder:validation:XValidation:rule="!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode != 'none' || self.numberOfPlanes == 1",message="when MultiplaneMode is none, numberOfPlanes must be 1"
77+
// +kubebuilder:validation:XValidation:rule="!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode == 'none' || self.numberOfPlanes != 1",message="when MultiplaneMode is not none, numberOfPlanes must not be 1"
78+
// +kubebuilder:validation:XValidation:rule="!has(self.version) || !has(self.multiplaneMode) || !has(self.numberOfPlanes) || !(self.version == 'RA1.3' || self.version == 'RA2.0') || (self.multiplaneMode == 'none' && self.numberOfPlanes == 1)",message="when Version is RA1.3 or RA2.0, MultiplaneMode must be none and numberOfPlanes must be 1"
7679
type SpectrumXOptimizedSpec struct {
7780
// Optimize Spectrum X
7881
Enabled bool `json:"enabled"`
7982
// Version of the Spectrum-X architecture to optimize for
80-
// +kubebuilder:validation:Enum=RA1.3;RA2.0
83+
// +kubebuilder:validation:Enum=RA1.3;RA2.0;RA2.1
8184
// +required
8285
Version string `json:"version"`
8386
// Overlay mode to be configured
@@ -86,6 +89,17 @@ type SpectrumXOptimizedSpec struct {
8689
// +kubebuilder:default:=none
8790
// +optional
8891
Overlay string `json:"overlay,omitempty"`
92+
// Multiplane mode to be configured
93+
// Can be "none", "swplb", "hwplb", or "uniplane"
94+
// +kubebuilder:validation:Enum=none;swplb;hwplb;uniplane
95+
// +kubebuilder:default:=none
96+
// +optional
97+
MultiplaneMode string `json:"multiplaneMode,omitempty"`
98+
// Number of planes to be configured
99+
// +kubebuilder:validation:Enum=1;2;4
100+
// +kubebuilder:default:=1
101+
// +optional
102+
NumberOfPlanes int `json:"numberOfPlanes,omitempty"`
89103
}
90104

91105
type NvConfigParam struct {
@@ -121,6 +135,7 @@ type ConfigurationTemplateSpec struct {
121135

122136
// NicConfigurationTemplateSpec defines the desired state of NicConfigurationTemplate
123137
// +kubebuilder:validation:XValidation:rule="!(has(self.template.spectrumXOptimized) && self.template.spectrumXOptimized.enabled) || (self.nicSelector.nicType == '1023' || self.nicSelector.nicType == 'a2dc')",message="spectrumXOptimized can be enabled only for ConnectX-8 or BlueField-3 SuperNICs"
138+
// +kubebuilder:validation:XValidation:rule="!has(self.template.spectrumXOptimized) || !has(self.template.spectrumXOptimized.multiplaneMode) || self.template.spectrumXOptimized.multiplaneMode != 'hwplb' || self.nicSelector.nicType == '1023'",message="hwplb MultiplaneMode can only be enabled for ConnectX-8 (NicType 1023)"
124139
type NicConfigurationTemplateSpec struct {
125140
// NodeSelector contains labels required on the node. When empty, the template will be applied to matching devices on all nodes.
126141
NodeSelector map[string]string `json:"nodeSelector,omitempty"`

‎api/v1alpha1/validation_test.go‎

Lines changed: 281 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@ import (
2626
"sigs.k8s.io/controller-runtime/pkg/client"
2727
)
2828

29+
const (
30+
nicTypeConnectX8 = "1023"
31+
nicTypeBlueField3 = "a2dc"
32+
unsupportedNicType = "a2d0"
33+
)
34+
2935
var _ = Describe("NicConfigurationTemplate CEL validation", func() {
3036
var (
3137
k8sClient client.Client
@@ -57,7 +63,7 @@ var _ = Describe("NicConfigurationTemplate CEL validation", func() {
5763
Namespace: metav1.NamespaceDefault,
5864
},
5965
Spec: NicConfigurationTemplateSpec{
60-
NicSelector: &NicSelectorSpec{NicType: "a2dc"},
66+
NicSelector: &NicSelectorSpec{NicType: nicTypeBlueField3},
6167
Template: &ConfigurationTemplateSpec{
6268
NumVfs: numVfs,
6369
LinkType: linkType,
@@ -140,29 +146,299 @@ var _ = Describe("NicConfigurationTemplate CEL validation", func() {
140146

141147
It("allows SpectrumXOptimized enabled for ConnectX-8 (NicType 1023)", func() {
142148
obj := newNicConfigurationTemplate("spcx-allow-1023", "Ethernet", 1, &SpectrumXOptimizedSpec{Enabled: true, Version: "RA2.0"})
143-
obj.Spec.NicSelector.NicType = "1023"
149+
obj.Spec.NicSelector.NicType = nicTypeConnectX8
144150
err := k8sClient.Create(ctx, obj)
145151
Expect(err).NotTo(HaveOccurred())
146152
})
147153

148154
It("allows SpectrumXOptimized enabled for BlueField-3 ConnectX-7 (NicType a2dc)", func() {
149155
obj := newNicConfigurationTemplate("spcx-allow-a2dc", "Ethernet", 1, &SpectrumXOptimizedSpec{Enabled: true, Version: "RA2.0"})
150-
obj.Spec.NicSelector.NicType = "a2dc"
156+
obj.Spec.NicSelector.NicType = nicTypeBlueField3
151157
err := k8sClient.Create(ctx, obj)
152158
Expect(err).NotTo(HaveOccurred())
153159
})
154160

155161
It("rejects SpectrumXOptimized enabled for unsupported NicType", func() {
156162
obj := newNicConfigurationTemplate("spcx-reject-unsupported-nictype", "Ethernet", 1, &SpectrumXOptimizedSpec{Enabled: true, Version: "RA2.0"})
157-
obj.Spec.NicSelector.NicType = "a2d0"
163+
obj.Spec.NicSelector.NicType = unsupportedNicType
158164
err := k8sClient.Create(ctx, obj)
159165
Expect(err).To(HaveOccurred())
160166
})
161167

162168
It("allow configuration template without spectrumXOptimized and arbitrary nicType", func() {
163169
obj := newNicConfigurationTemplate("spcx-disabled-arbitrary-nictype", "Ethernet", 1, &SpectrumXOptimizedSpec{Enabled: false, Version: "RA2.0"})
164-
obj.Spec.NicSelector.NicType = "a2d0"
170+
obj.Spec.NicSelector.NicType = unsupportedNicType
165171
err := k8sClient.Create(ctx, obj)
166172
Expect(err).NotTo(HaveOccurred())
167173
})
174+
175+
// Tests for MultiplaneMode and NumberOfPlanes validation rules
176+
Context("MultiplaneMode and NumberOfPlanes validation", func() {
177+
It("allows MultiplaneMode=none with numberOfPlanes=1", func() {
178+
obj := newNicConfigurationTemplate("spcx-none-1plane", "Ethernet", 1, &SpectrumXOptimizedSpec{
179+
Enabled: true,
180+
Version: "RA2.1",
181+
MultiplaneMode: "none",
182+
NumberOfPlanes: 1,
183+
})
184+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
185+
})
186+
187+
It("rejects MultiplaneMode=none with numberOfPlanes=2", func() {
188+
obj := newNicConfigurationTemplate("spcx-none-2planes", "Ethernet", 1, &SpectrumXOptimizedSpec{
189+
Enabled: true,
190+
Version: "RA2.1",
191+
MultiplaneMode: "none",
192+
NumberOfPlanes: 2,
193+
})
194+
err := k8sClient.Create(ctx, obj)
195+
Expect(err).To(HaveOccurred())
196+
Expect(err.Error()).To(ContainSubstring("when MultiplaneMode is none, numberOfPlanes must be 1"))
197+
})
198+
199+
It("rejects MultiplaneMode=none with numberOfPlanes=4", func() {
200+
obj := newNicConfigurationTemplate("spcx-none-4planes", "Ethernet", 1, &SpectrumXOptimizedSpec{
201+
Enabled: true,
202+
Version: "RA2.1",
203+
MultiplaneMode: "none",
204+
NumberOfPlanes: 4,
205+
})
206+
err := k8sClient.Create(ctx, obj)
207+
Expect(err).To(HaveOccurred())
208+
Expect(err.Error()).To(ContainSubstring("when MultiplaneMode is none, numberOfPlanes must be 1"))
209+
})
210+
211+
It("allows MultiplaneMode=swplb with numberOfPlanes=2", func() {
212+
obj := newNicConfigurationTemplate("spcx-swplb-2planes", "Ethernet", 1, &SpectrumXOptimizedSpec{
213+
Enabled: true,
214+
Version: "RA2.1",
215+
MultiplaneMode: "swplb",
216+
NumberOfPlanes: 2,
217+
})
218+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
219+
})
220+
221+
It("allows MultiplaneMode=hwplb with numberOfPlanes=4", func() {
222+
obj := newNicConfigurationTemplate("spcx-hwplb-4planes", "Ethernet", 1, &SpectrumXOptimizedSpec{
223+
Enabled: true,
224+
Version: "RA2.1",
225+
MultiplaneMode: "hwplb",
226+
NumberOfPlanes: 4,
227+
})
228+
obj.Spec.NicSelector.NicType = nicTypeConnectX8
229+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
230+
})
231+
232+
It("rejects MultiplaneMode=swplb with numberOfPlanes=1", func() {
233+
obj := newNicConfigurationTemplate("spcx-swplb-1plane", "Ethernet", 1, &SpectrumXOptimizedSpec{
234+
Enabled: true,
235+
Version: "RA2.1",
236+
MultiplaneMode: "swplb",
237+
NumberOfPlanes: 1,
238+
})
239+
err := k8sClient.Create(ctx, obj)
240+
Expect(err).To(HaveOccurred())
241+
Expect(err.Error()).To(ContainSubstring("when MultiplaneMode is not none, numberOfPlanes must not be 1"))
242+
})
243+
244+
It("rejects MultiplaneMode=hwplb with numberOfPlanes=1", func() {
245+
obj := newNicConfigurationTemplate("spcx-hwplb-1plane", "Ethernet", 1, &SpectrumXOptimizedSpec{
246+
Enabled: true,
247+
Version: "RA2.1",
248+
MultiplaneMode: "hwplb",
249+
NumberOfPlanes: 1,
250+
})
251+
err := k8sClient.Create(ctx, obj)
252+
Expect(err).To(HaveOccurred())
253+
Expect(err.Error()).To(ContainSubstring("when MultiplaneMode is not none, numberOfPlanes must not be 1"))
254+
})
255+
256+
It("rejects MultiplaneMode=uniplane with numberOfPlanes=1", func() {
257+
obj := newNicConfigurationTemplate("spcx-uniplane-1plane", "Ethernet", 1, &SpectrumXOptimizedSpec{
258+
Enabled: true,
259+
Version: "RA2.1",
260+
MultiplaneMode: "uniplane",
261+
NumberOfPlanes: 1,
262+
})
263+
err := k8sClient.Create(ctx, obj)
264+
Expect(err).To(HaveOccurred())
265+
Expect(err.Error()).To(ContainSubstring("when MultiplaneMode is not none, numberOfPlanes must not be 1"))
266+
})
267+
})
268+
269+
// Tests for Version-specific MultiplaneMode restrictions
270+
Context("Version-specific MultiplaneMode validation", func() {
271+
It("allows Version=RA1.3 with MultiplaneMode=none and numberOfPlanes=1", func() {
272+
obj := newNicConfigurationTemplate("spcx-ra13-none-1", "Ethernet", 1, &SpectrumXOptimizedSpec{
273+
Enabled: true,
274+
Version: "RA1.3",
275+
MultiplaneMode: "none",
276+
NumberOfPlanes: 1,
277+
})
278+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
279+
})
280+
281+
It("rejects Version=RA1.3 with MultiplaneMode=swplb", func() {
282+
obj := newNicConfigurationTemplate("spcx-ra13-swplb", "Ethernet", 1, &SpectrumXOptimizedSpec{
283+
Enabled: true,
284+
Version: "RA1.3",
285+
MultiplaneMode: "swplb",
286+
NumberOfPlanes: 2,
287+
})
288+
err := k8sClient.Create(ctx, obj)
289+
Expect(err).To(HaveOccurred())
290+
Expect(err.Error()).To(ContainSubstring("when Version is RA1.3 or RA2.0, MultiplaneMode must be none and numberOfPlanes must be 1"))
291+
})
292+
293+
It("rejects Version=RA1.3 with MultiplaneMode=none but numberOfPlanes=2", func() {
294+
obj := newNicConfigurationTemplate("spcx-ra13-none-2", "Ethernet", 1, &SpectrumXOptimizedSpec{
295+
Enabled: true,
296+
Version: "RA1.3",
297+
MultiplaneMode: "none",
298+
NumberOfPlanes: 2,
299+
})
300+
err := k8sClient.Create(ctx, obj)
301+
Expect(err).To(HaveOccurred())
302+
Expect(err.Error()).To(ContainSubstring("when Version is RA1.3 or RA2.0, MultiplaneMode must be none and numberOfPlanes must be 1"))
303+
})
304+
305+
It("allows Version=RA2.0 with MultiplaneMode=none and numberOfPlanes=1", func() {
306+
obj := newNicConfigurationTemplate("spcx-ra20-none-1", "Ethernet", 1, &SpectrumXOptimizedSpec{
307+
Enabled: true,
308+
Version: "RA2.0",
309+
MultiplaneMode: "none",
310+
NumberOfPlanes: 1,
311+
})
312+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
313+
})
314+
315+
It("rejects Version=RA2.0 with MultiplaneMode=hwplb", func() {
316+
obj := newNicConfigurationTemplate("spcx-ra20-hwplb", "Ethernet", 1, &SpectrumXOptimizedSpec{
317+
Enabled: true,
318+
Version: "RA2.0",
319+
MultiplaneMode: "hwplb",
320+
NumberOfPlanes: 4,
321+
})
322+
err := k8sClient.Create(ctx, obj)
323+
Expect(err).To(HaveOccurred())
324+
Expect(err.Error()).To(ContainSubstring("when Version is RA1.3 or RA2.0, MultiplaneMode must be none and numberOfPlanes must be 1"))
325+
})
326+
327+
It("rejects Version=RA2.0 with MultiplaneMode=uniplane", func() {
328+
obj := newNicConfigurationTemplate("spcx-ra20-uniplane", "Ethernet", 1, &SpectrumXOptimizedSpec{
329+
Enabled: true,
330+
Version: "RA2.0",
331+
MultiplaneMode: "uniplane",
332+
NumberOfPlanes: 2,
333+
})
334+
err := k8sClient.Create(ctx, obj)
335+
Expect(err).To(HaveOccurred())
336+
Expect(err.Error()).To(ContainSubstring("when Version is RA1.3 or RA2.0, MultiplaneMode must be none and numberOfPlanes must be 1"))
337+
})
338+
339+
It("allows Version=RA2.1 with MultiplaneMode=swplb and numberOfPlanes=2", func() {
340+
obj := newNicConfigurationTemplate("spcx-ra21-swplb-2", "Ethernet", 1, &SpectrumXOptimizedSpec{
341+
Enabled: true,
342+
Version: "RA2.1",
343+
MultiplaneMode: "swplb",
344+
NumberOfPlanes: 2,
345+
})
346+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
347+
})
348+
349+
It("allows Version=RA2.1 with MultiplaneMode=hwplb and numberOfPlanes=4", func() {
350+
obj := newNicConfigurationTemplate("spcx-ra21-hwplb-4", "Ethernet", 1, &SpectrumXOptimizedSpec{
351+
Enabled: true,
352+
Version: "RA2.1",
353+
MultiplaneMode: "hwplb",
354+
NumberOfPlanes: 4,
355+
})
356+
obj.Spec.NicSelector.NicType = nicTypeConnectX8
357+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
358+
})
359+
360+
It("allows Version=RA2.1 with MultiplaneMode=none and numberOfPlanes=1", func() {
361+
obj := newNicConfigurationTemplate("spcx-ra21-none-1", "Ethernet", 1, &SpectrumXOptimizedSpec{
362+
Enabled: true,
363+
Version: "RA2.1",
364+
MultiplaneMode: "none",
365+
NumberOfPlanes: 1,
366+
})
367+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
368+
})
369+
})
370+
371+
// Tests for hwplb NicType restriction
372+
Context("hwplb NicType validation", func() {
373+
It("allows hwplb with NicType 1023 (ConnectX-8)", func() {
374+
obj := newNicConfigurationTemplate("hwplb-cx8-valid", "Ethernet", 1, &SpectrumXOptimizedSpec{
375+
Enabled: true,
376+
Version: "RA2.1",
377+
MultiplaneMode: "hwplb",
378+
NumberOfPlanes: 4,
379+
})
380+
obj.Spec.NicSelector.NicType = nicTypeConnectX8
381+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
382+
})
383+
384+
It("rejects hwplb with NicType a2dc (BlueField-3)", func() {
385+
obj := newNicConfigurationTemplate("hwplb-bf3-invalid", "Ethernet", 1, &SpectrumXOptimizedSpec{
386+
Enabled: true,
387+
Version: "RA2.1",
388+
MultiplaneMode: "hwplb",
389+
NumberOfPlanes: 4,
390+
})
391+
obj.Spec.NicSelector.NicType = nicTypeBlueField3
392+
err := k8sClient.Create(ctx, obj)
393+
Expect(err).To(HaveOccurred())
394+
Expect(err.Error()).To(ContainSubstring("hwplb MultiplaneMode can only be enabled for ConnectX-8 (NicType 1023)"))
395+
})
396+
397+
It("allows swplb with NicType a2dc (BlueField-3)", func() {
398+
obj := newNicConfigurationTemplate("swplb-bf3-valid", "Ethernet", 1, &SpectrumXOptimizedSpec{
399+
Enabled: true,
400+
Version: "RA2.1",
401+
MultiplaneMode: "swplb",
402+
NumberOfPlanes: 2,
403+
})
404+
obj.Spec.NicSelector.NicType = nicTypeBlueField3
405+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
406+
})
407+
408+
It("allows uniplane with NicType a2dc (BlueField-3)", func() {
409+
obj := newNicConfigurationTemplate("uniplane-bf3-valid", "Ethernet", 1, &SpectrumXOptimizedSpec{
410+
Enabled: true,
411+
Version: "RA2.1",
412+
MultiplaneMode: "uniplane",
413+
NumberOfPlanes: 2,
414+
})
415+
obj.Spec.NicSelector.NicType = nicTypeBlueField3
416+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
417+
})
418+
419+
It("allows none with NicType a2dc (BlueField-3)", func() {
420+
obj := newNicConfigurationTemplate("none-bf3-valid", "Ethernet", 1, &SpectrumXOptimizedSpec{
421+
Enabled: true,
422+
Version: "RA2.1",
423+
MultiplaneMode: "none",
424+
NumberOfPlanes: 1,
425+
})
426+
obj.Spec.NicSelector.NicType = nicTypeBlueField3
427+
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
428+
})
429+
430+
It("rejects hwplb with unsupported NicType", func() {
431+
obj := newNicConfigurationTemplate("hwplb-unsupported-nictype", "Ethernet", 1, &SpectrumXOptimizedSpec{
432+
Enabled: true,
433+
Version: "RA2.1",
434+
MultiplaneMode: "hwplb",
435+
NumberOfPlanes: 4,
436+
})
437+
obj.Spec.NicSelector.NicType = unsupportedNicType
438+
err := k8sClient.Create(ctx, obj)
439+
Expect(err).To(HaveOccurred())
440+
// Should fail on the spectrumXOptimized device type check first
441+
Expect(err.Error()).To(ContainSubstring("spectrumXOptimized can be enabled only for ConnectX-8 or BlueField-3 SuperNICs"))
442+
})
443+
})
168444
})

0 commit comments

Comments
 (0)