Skip to content

Commit 79013d7

Browse files
e0neroot
e0ne
authored and
root
committed
Implement RDMA subsystem mode change
Now it's possible to configure RDMA subsystem mode using SR-IOV Network Operator in systemd mode We can't configure RDMA subsystem in a daemon mode because it should be done on host before any network namespace is created.
1 parent 225a839 commit 79013d7

File tree

15 files changed

+265
-102
lines changed

15 files changed

+265
-102
lines changed

‎api/v1/sriovnetworkpoolconfig_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
2121
// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
2222
// even if maxUnavailable is greater than one.
2323
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
24+
25+
// +kubebuilder:validation:Enum=shared;exclusive
26+
// RDMA subsystem. Allowed value "shared", "exclusive".
27+
RdmaMode string `json:"rdmaMode,omitempty"`
2428
}
2529

2630
type OvsHardwareOffloadConfig struct {

‎cmd/sriov-network-config-daemon/service.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,19 @@ func phasePre(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helpe
152152
hostHelpers.TryEnableTun()
153153
hostHelpers.TryEnableVhostNet()
154154

155+
if conf.RdmaMode != "" {
156+
rdmaSubsystem, err := hostHelpers.GetRDMASubsystem()
157+
if err != nil {
158+
setupLog.Error(err, "failed to get RDMA subsystem mode")
159+
}
160+
if rdmaSubsystem != conf.RdmaMode {
161+
err = hostHelpers.SetRDMASubsystem(conf.RdmaMode)
162+
if err != nil {
163+
setupLog.Error(err, "failed to set RDMA subsystem mode")
164+
}
165+
}
166+
}
167+
155168
return callPlugin(setupLog, PhasePre, conf, hostHelpers)
156169
}
157170

‎cmd/sriov-network-config-daemon/service_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ var _ = Describe("Service", func() {
161161
hostHelpers.EXPECT().TryEnableRdma().Return(true, nil)
162162
hostHelpers.EXPECT().TryEnableTun().Return()
163163
hostHelpers.EXPECT().TryEnableVhostNet().Return()
164+
hostHelpers.EXPECT().GetRDMASubsystem().Return("", nil)
164165
hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{
165166
Name: "enp216s0f0np0",
166167
}}, nil)
@@ -186,6 +187,7 @@ var _ = Describe("Service", func() {
186187
hostHelpers.EXPECT().TryEnableRdma().Return(true, nil)
187188
hostHelpers.EXPECT().TryEnableTun().Return()
188189
hostHelpers.EXPECT().TryEnableVhostNet().Return()
190+
hostHelpers.EXPECT().GetRDMASubsystem().Return("", nil)
189191

190192
platformHelper.EXPECT().CreateOpenstackDevicesInfo().Return(nil)
191193
platformHelper.EXPECT().DiscoverSriovDevicesVirtual().Return([]sriovnetworkv1.InterfaceExt{{
@@ -214,6 +216,7 @@ var _ = Describe("Service", func() {
214216
hostHelpers.EXPECT().TryEnableRdma().Return(true, nil)
215217
hostHelpers.EXPECT().TryEnableTun().Return()
216218
hostHelpers.EXPECT().TryEnableVhostNet().Return()
219+
hostHelpers.EXPECT().GetRDMASubsystem().Return("", nil)
217220
hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{
218221
Name: "enp216s0f0np0",
219222
}}, nil)

‎config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ spec:
103103
offload'
104104
type: string
105105
type: object
106+
rdmaMode:
107+
description: RDMA subsystem. Allowed value "shared", "exclusive".
108+
enum:
109+
- shared
110+
- exclusive
111+
type: string
106112
type: object
107113
status:
108114
description: SriovNetworkPoolConfigStatus defines the observed state of

‎controllers/drain_controller.go‎

Lines changed: 1 addition & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,8 @@ import (
2424

2525
corev1 "k8s.io/api/core/v1"
2626
"k8s.io/apimachinery/pkg/api/errors"
27-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28-
"k8s.io/apimachinery/pkg/labels"
2927
"k8s.io/apimachinery/pkg/runtime"
3028
"k8s.io/apimachinery/pkg/types"
31-
"k8s.io/apimachinery/pkg/util/intstr"
3229
"k8s.io/client-go/tools/record"
3330
"k8s.io/client-go/util/workqueue"
3431
ctrl "sigs.k8s.io/controller-runtime"
@@ -48,13 +45,6 @@ import (
4845
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
4946
)
5047

51-
var (
52-
oneNode = intstr.FromInt32(1)
53-
defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
54-
MaxUnavailable: &oneNode,
55-
NodeSelector: &metav1.LabelSelector{}}}
56-
)
57-
5848
type DrainReconcile struct {
5949
client.Client
6050
Scheme *runtime.Scheme
@@ -345,94 +335,7 @@ func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (
345335
}
346336

347337
func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
348-
logger := log.FromContext(ctx)
349-
logger.Info("findNodePoolConfig():")
350-
// get all the sriov network pool configs
351-
npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
352-
err := dr.List(ctx, npcl)
353-
if err != nil {
354-
logger.Error(err, "failed to list sriovNetworkPoolConfig")
355-
return nil, nil, err
356-
}
357-
358-
selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
359-
nodesInPools := map[string]interface{}{}
360-
361-
for _, npc := range npcl.Items {
362-
// we skip hw offload objects
363-
if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
364-
continue
365-
}
366-
367-
if npc.Spec.NodeSelector == nil {
368-
npc.Spec.NodeSelector = &metav1.LabelSelector{}
369-
}
370-
371-
selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
372-
if err != nil {
373-
logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
374-
return nil, nil, err
375-
}
376-
377-
if selector.Matches(labels.Set(node.Labels)) {
378-
selectedNpcl = append(selectedNpcl, npc.DeepCopy())
379-
}
380-
381-
nodeList := &corev1.NodeList{}
382-
err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
383-
if err != nil {
384-
logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
385-
"machineConfigPoolName", npc,
386-
"nodeSelector", npc.Spec.NodeSelector)
387-
return nil, nil, err
388-
}
389-
390-
for _, nodeName := range nodeList.Items {
391-
nodesInPools[nodeName.Name] = nil
392-
}
393-
}
394-
395-
if len(selectedNpcl) > 1 {
396-
// don't allow the node to be part of multiple pools
397-
err = fmt.Errorf("node is part of more then one pool")
398-
logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
399-
return nil, nil, err
400-
} else if len(selectedNpcl) == 1 {
401-
// found one pool for our node
402-
logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
403-
selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
404-
if err != nil {
405-
logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
406-
return nil, nil, err
407-
}
408-
409-
// list all the nodes that are also part of this pool and return them
410-
nodeList := &corev1.NodeList{}
411-
err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
412-
if err != nil {
413-
logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
414-
return nil, nil, err
415-
}
416-
417-
return selectedNpcl[0], nodeList.Items, nil
418-
} else {
419-
// in this case we get all the nodes and remove the ones that already part of any pool
420-
logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl)
421-
nodeList := &corev1.NodeList{}
422-
err = dr.List(ctx, nodeList)
423-
if err != nil {
424-
logger.Error(err, "failed to list all the nodes")
425-
return nil, nil, err
426-
}
427-
428-
defaultNodeLists := []corev1.Node{}
429-
for _, nodeObj := range nodeList.Items {
430-
if _, exist := nodesInPools[nodeObj.Name]; !exist {
431-
defaultNodeLists = append(defaultNodeLists, nodeObj)
432-
}
433-
}
434-
return defaultNpcl, defaultNodeLists, nil
435-
}
338+
return utils.FindNodePoolConfig(ctx, node, dr.Client)
436339
}
437340

438341
// SetupWithManager sets up the controller with the Manager.

‎controllers/sriovnetworkpoolconfig_controller.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,19 @@ func (r *SriovNetworkPoolConfigReconciler) Reconcile(ctx context.Context, req ct
7373
return reconcile.Result{}, err
7474
}
7575

76+
// RdmaMode could be set in systemd mode only
77+
if instance.Spec.RdmaMode != "" {
78+
operatorConfig := &sriovnetworkv1.SriovOperatorConfig{}
79+
err := r.Get(ctx, types.NamespacedName{Namespace: vars.Namespace, Name: constants.DefaultConfigName}, operatorConfig)
80+
if err != nil {
81+
logger.Error(err, "failed to list SriovOperatorConfig")
82+
return reconcile.Result{}, err
83+
}
84+
if operatorConfig.Spec.ConfigurationMode == sriovnetworkv1.DaemonConfigurationMode {
85+
logger.Info("rdmaSpec is ignored in 'daemon' configuration mode")
86+
}
87+
}
88+
7689
// we don't need a finalizer for pools that doesn't use the ovs hardware offload feature
7790
if instance.Spec.OvsHardwareOffloadConfig.Name == "" {
7891
return ctrl.Result{}, nil

‎deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ spec:
103103
offload'
104104
type: string
105105
type: object
106+
rdmaMode:
107+
description: RDMA subsystem. Allowed value "shared", "exclusive".
108+
enum:
109+
- shared
110+
- exclusive
111+
type: string
106112
type: object
107113
status:
108114
description: SriovNetworkPoolConfigStatus defines the observed state of

‎pkg/consts/constants.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ const (
4747
VdpaTypeVirtio = "virtio"
4848
VdpaTypeVhost = "vhost"
4949

50+
RdmaSubsystemModeShared = "shared"
51+
RdmaSubsystemModeExclusive = "exclusive"
52+
5053
ClusterTypeOpenshift = "openshift"
5154
ClusterTypeKubernetes = "kubernetes"
5255

‎pkg/daemon/daemon.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"sync"
1414
"time"
1515

16+
corev1 "k8s.io/api/core/v1"
17+
1618
"golang.org/x/time/rate"
1719
"k8s.io/apimachinery/pkg/api/errors"
1820
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -453,7 +455,19 @@ func (dn *Daemon) nodeStateSyncHandler() error {
453455
// When using systemd configuration we write the file
454456
if vars.UsingSystemdMode {
455457
log.Log.V(0).Info("nodeStateSyncHandler(): writing systemd config file to host")
456-
systemdConfModified, err := systemd.WriteConfFile(dn.desiredNodeState)
458+
// get node object
459+
node := &corev1.Node{}
460+
err := dn.client.Get(context.TODO(), client.ObjectKey{Name: vars.NodeName}, node)
461+
if err != nil {
462+
log.Log.Error(err, "nodeStateSyncHandler(): failed to get node object")
463+
return err
464+
}
465+
netPoolConfig, _, err := utils.FindNodePoolConfig(context.Background(), node, dn.client)
466+
if err != nil {
467+
log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node")
468+
}
469+
470+
systemdConfModified, err := systemd.WriteConfFile(dn.desiredNodeState, netPoolConfig)
457471
if err != nil {
458472
log.Log.Error(err, "nodeStateSyncHandler(): failed to write configuration file for systemd mode")
459473
return err

‎pkg/helper/mock/mock_helper.go‎

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)