diff --git a/api/v1/helper.go b/api/v1/helper.go index 363006dc5..22619605a 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -359,6 +359,7 @@ func (p *SriovNetworkNodePolicy) Apply(state *SriovNetworkNodeState, equalPriori Name: iface.Name, LinkType: p.Spec.LinkType, EswitchMode: p.Spec.EswitchMode, + ESwitchParams: p.Spec.ESwitchParams, NumVfs: p.Spec.NumVfs, ExternallyManaged: p.Spec.ExternallyManaged, } diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go index bbcba03a2..2e62c9f42 100644 --- a/api/v1/sriovnetworknodepolicy_types.go +++ b/api/v1/sriovnetworknodepolicy_types.go @@ -65,6 +65,9 @@ type SriovNetworkNodePolicySpec struct { // contains bridge configuration for matching PFs, // valid only for eSwitchMode==switchdev Bridge Bridge `json:"bridge,omitempty"` + // contains bridge configuration eSwitch, + // valid only for eSwitchMode==switchdev + ESwitchParams ESwitchParams `json:"eSwitchParams,omitempty"` } type SriovNetworkNicSelector struct { @@ -91,6 +94,11 @@ func (b *Bridge) IsEmpty() bool { return b.OVS == nil } +type ESwitchParams struct { + // enabled multiport eswitch mode + Multiport bool `json:"multiport,omitempty"` +} + // OVSConfig optional configuration for OVS bridge and uplink Interface type OVSConfig struct { // contains bridge level settings diff --git a/api/v1/sriovnetworknodestate_types.go b/api/v1/sriovnetworknodestate_types.go index e5f59d71c..371a7adb6 100644 --- a/api/v1/sriovnetworknodestate_types.go +++ b/api/v1/sriovnetworknodestate_types.go @@ -33,14 +33,15 @@ type SriovNetworkNodeStateSpec struct { type Interfaces []Interface type Interface struct { - PciAddress string `json:"pciAddress"` - NumVfs int `json:"numVfs,omitempty"` - Mtu int `json:"mtu,omitempty"` - Name string `json:"name,omitempty"` - LinkType string `json:"linkType,omitempty"` - EswitchMode string `json:"eSwitchMode,omitempty"` - VfGroups []VfGroup `json:"vfGroups,omitempty"` - ExternallyManaged bool `json:"externallyManaged,omitempty"` + PciAddress string `json:"pciAddress"` + NumVfs int `json:"numVfs,omitempty"` + Mtu int `json:"mtu,omitempty"` + Name string `json:"name,omitempty"` + LinkType string `json:"linkType,omitempty"` + EswitchMode string `json:"eSwitchMode,omitempty"` + VfGroups []VfGroup `json:"vfGroups,omitempty"` + ExternallyManaged bool `json:"externallyManaged,omitempty"` + ESwitchParams ESwitchParams `json:"eSwitchParams,omitempty"` } type VfGroup struct { @@ -70,6 +71,7 @@ type InterfaceExt struct { ExternallyManaged bool `json:"externallyManaged,omitempty"` TotalVfs int `json:"totalvfs,omitempty"` VFs []VirtualFunction `json:"Vfs,omitempty"` + ESwitchParams ESwitchParams `json:"eSwitchParams,omitempty"` } type InterfaceExts []InterfaceExt diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0d9d3c4cf..3075f5645 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -89,6 +89,21 @@ func (in ByPriority) DeepCopy() ByPriority { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ESwitchParams) DeepCopyInto(out *ESwitchParams) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ESwitchParams. +func (in *ESwitchParams) DeepCopy() *ESwitchParams { + if in == nil { + return nil + } + out := new(ESwitchParams) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Interface) DeepCopyInto(out *Interface) { *out = *in @@ -97,6 +112,7 @@ func (in *Interface) DeepCopyInto(out *Interface) { *out = make([]VfGroup, len(*in)) copy(*out, *in) } + out.ESwitchParams = in.ESwitchParams } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Interface. @@ -691,6 +707,7 @@ func (in *SriovNetworkNodePolicySpec) DeepCopyInto(out *SriovNetworkNodePolicySp } in.NicSelector.DeepCopyInto(&out.NicSelector) in.Bridge.DeepCopyInto(&out.Bridge) + out.ESwitchParams = in.ESwitchParams } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovNetworkNodePolicySpec. diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index b72514e6a..af526b39d 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -126,6 +126,15 @@ spec: - legacy - switchdev type: string + eSwitchParams: + description: |- + contains bridge configuration eSwitch, + valid only for eSwitchMode==switchdev + properties: + multiport: + description: enabled multiport eswitch mode + type: boolean + type: object excludeTopology: description: Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false. diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index 7230ee212..a5b910321 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -150,6 +150,12 @@ spec: properties: eSwitchMode: type: string + eSwitchParams: + properties: + multiport: + description: enabled multiport eswitch mode + type: boolean + type: object externallyManaged: type: boolean linkType: diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index b72514e6a..af526b39d 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -126,6 +126,15 @@ spec: - legacy - switchdev type: string + eSwitchParams: + description: |- + contains bridge configuration eSwitch, + valid only for eSwitchMode==switchdev + properties: + multiport: + description: enabled multiport eswitch mode + type: boolean + type: object excludeTopology: description: Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false. diff --git a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index 7230ee212..a5b910321 100644 --- a/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -150,6 +150,12 @@ spec: properties: eSwitchMode: type: string + eSwitchParams: + properties: + multiport: + description: enabled multiport eswitch mode + type: boolean + type: object externallyManaged: type: boolean linkType: diff --git a/pkg/plugins/mellanox/mellanox_plugin.go b/pkg/plugins/mellanox/mellanox_plugin.go index da7344bfc..de9513b55 100644 --- a/pkg/plugins/mellanox/mellanox_plugin.go +++ b/pkg/plugins/mellanox/mellanox_plugin.go @@ -107,6 +107,9 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS needReboot = totalVfsNeedReboot || sriovEnNeedReboot changeWithoutReboot = totalVfsChangeWithoutReboot || sriovEnChangeWithoutReboot + needESwitchParamsChange := mlx.HandleESwitchParams(pciPrefix, attrs, mellanoxNicsSpec, mellanoxNicsStatus) + needReboot = needReboot || needESwitchParamsChange + needLinkChange, err := mlx.HandleLinkType(pciPrefix, fwCurrent, attrs, mellanoxNicsSpec, mellanoxNicsStatus) if err != nil { return false, false, err diff --git a/pkg/vendors/mellanox/mellanox.go b/pkg/vendors/mellanox/mellanox.go index 0cc60b8b7..81d555d7f 100644 --- a/pkg/vendors/mellanox/mellanox.go +++ b/pkg/vendors/mellanox/mellanox.go @@ -36,16 +36,13 @@ const ( disabled = "DISABLED" enabled = "ENABLED" - VendorMellanox = "15b3" - DeviceBF2 = "a2d6" - DeviceBF3 = "a2dc" - PreconfiguredLinkType = "Preconfigured" UnknownLinkType = "Unknown" TotalVfs = "NUM_OF_VFS" EnableSriov = "SRIOV_EN" LinkTypeP1 = "LINK_TYPE_P1" LinkTypeP2 = "LINK_TYPE_P2" + LagResourceAllocation = "LAG_RESOURCE_ALLOCATION" MellanoxVendorID = "15b3" ) @@ -54,6 +51,7 @@ type MlxNic struct { TotalVfs int LinkTypeP1 string LinkTypeP2 string + Multiport int } //go:generate ../../../bin/mockgen -destination mock/mock_mellanox.go -source mellanox.go @@ -207,6 +205,8 @@ func (m *mellanoxHelper) MlxConfigFW(attributesToChange map[string]MlxNic) error cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%s", LinkTypeP2, fwArgs.LinkTypeP2)) } + cmdArgs = append(cmdArgs, fmt.Sprintf("%s=%d", LagResourceAllocation, fwArgs.Multiport)) + log.Log.V(2).Info("mellanox-plugin: configFW()", "cmd-args", cmdArgs) if len(cmdArgs) <= 4 { continue @@ -376,6 +376,30 @@ func HandleLinkType(pciPrefix string, fwData, attr *MlxNic, return needReboot, nil } +// HandleESwitchParams check if eswitch params should be changes +func HandleESwitchParams(pciPrefix string, attr *MlxNic, + mellanoxNicsSpec map[string]sriovnetworkv1.Interface, + mellanoxNicsStatus map[string]map[string]sriovnetworkv1.InterfaceExt) bool { + needReboot := false + + pciAddress := pciPrefix + "0" + if firstPortSpec, ok := mellanoxNicsSpec[pciAddress]; ok { + ifaceStatus := getIfaceStatus(pciAddress, mellanoxNicsStatus) + needChange := isESwitchParamsRequireChange(firstPortSpec, ifaceStatus) + if needChange { + log.Log.V(2).Info("Changing eswitch params (multiport), needs reboot", + "from", firstPortSpec.ESwitchParams.Multiport, "to", firstPortSpec.ESwitchParams.Multiport) + if firstPortSpec.ESwitchParams.Multiport { + attr.Multiport = 1 + } else { + attr.Multiport = 0 + } + needReboot = true + } + } + return needReboot +} + func mlnxNicFromMap(mstData map[string]string) (*MlxNic, error) { log.Log.Info("mellanox-plugin mlnxNicFromMap()", "data", mstData) fwData := &MlxNic{} @@ -432,6 +456,12 @@ func isLinkTypeRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovne return false, nil } +func isESwitchParamsRequireChange(iface sriovnetworkv1.Interface, ifaceStatus sriovnetworkv1.InterfaceExt) bool { + log.Log.Info("mellanox-plugin isLagResourceAllocationRequireChange()", "device", iface.PciAddress) + + return iface.ESwitchParams.Multiport != ifaceStatus.ESwitchParams.Multiport +} + func getOtherPortPCIAddress(pciAddress string) string { log.Log.Info("mellanox-plugin getOtherPortSpec()", "pciAddress", pciAddress) pciAddrPrefix := GetPciAddressPrefix(pciAddress) diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index 42b504280..c2d4a255d 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -233,6 +233,10 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol if !cr.Spec.Bridge.IsEmpty() && cr.Spec.ExternallyManaged { return false, fmt.Errorf("software bridge management can't be used when the device externally managed") } + // multiport must be enabled only in switchdev mode + if cr.Spec.ESwitchParams.Multiport && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { + return false, fmt.Errorf("multiport requires the device to be configured in switchdev mode") + } return true, nil }