Skip to content

Commit 13dd9f1

Browse files
SchSebaSchSeba
committed
improve virtual_plugin to reset devices
before this change when we remove a policy the daemon didn't revert the driver Signed-off-by: Sebastian Sch <[email protected]>
1 parent 85a7bda commit 13dd9f1

File tree

7 files changed

+762
-85
lines changed

7 files changed

+762
-85
lines changed

‎pkg/helper/mock/mock_helper.go‎

Lines changed: 6 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/host/internal/sriov/sriov.go‎

Lines changed: 98 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ import (
2828
)
2929

3030
type interfaceToConfigure struct {
31-
iface sriovnetworkv1.Interface
32-
ifaceStatus sriovnetworkv1.InterfaceExt
31+
Iface sriovnetworkv1.Interface
32+
IfaceStatus sriovnetworkv1.InterfaceExt
3333
}
3434

3535
type sriov struct {
@@ -715,7 +715,7 @@ func (s *sriov) getConfigureAndReset(storeManager store.ManagerInterface, interf
715715
}
716716
iface := iface
717717
ifaceStatus := ifaceStatus
718-
toBeConfigured = append(toBeConfigured, interfaceToConfigure{iface: iface, ifaceStatus: ifaceStatus})
718+
toBeConfigured = append(toBeConfigured, interfaceToConfigure{Iface: iface, IfaceStatus: ifaceStatus})
719719
}
720720
}
721721

@@ -736,12 +736,12 @@ func (s *sriov) configSriovInterfacesInParallel(storeManager store.ManagerInterf
736736
interfacesToConfigure += 1
737737
go func(iface *interfaceToConfigure) {
738738
var err error
739-
if err = s.configSriovDevice(&iface.iface, skipVFConfiguration); err != nil {
740-
log.Log.Error(err, "configSriovInterfacesInParallel(): fail to configure sriov interface. resetting interface.", "address", iface.iface.PciAddress)
741-
if iface.iface.ExternallyManaged {
739+
if err = s.configSriovDevice(&iface.Iface, skipVFConfiguration); err != nil {
740+
log.Log.Error(err, "configSriovInterfacesInParallel(): fail to configure sriov interface. resetting interface.", "address", iface.Iface.PciAddress)
741+
if iface.Iface.ExternallyManaged {
742742
log.Log.V(2).Info("configSriovInterfacesInParallel(): skipping device reset as the nic is marked as externally created")
743743
} else {
744-
if resetErr := s.ResetSriovDevice(iface.ifaceStatus); resetErr != nil {
744+
if resetErr := s.ResetSriovDevice(iface.IfaceStatus); resetErr != nil {
745745
log.Log.Error(resetErr, "configSriovInterfacesInParallel(): failed to reset on error SR-IOV interface")
746746
err = resetErr
747747
}
@@ -750,7 +750,7 @@ func (s *sriov) configSriovInterfacesInParallel(storeManager store.ManagerInterf
750750
errChannel <- err
751751
}(&interfaces[ifaceIndex])
752752
// Save the PF status to the host
753-
err := storeManager.SaveLastPfAppliedStatus(&iface.iface)
753+
err := storeManager.SaveLastPfAppliedStatus(&iface.Iface)
754754
if err != nil {
755755
log.Log.Error(err, "configSriovInterfacesInParallel(): failed to save PF applied config to host")
756756
return err
@@ -800,20 +800,20 @@ func (s *sriov) resetSriovInterfacesInParallel(storeManager store.ManagerInterfa
800800
func (s *sriov) configSriovInterfaces(storeManager store.ManagerInterface, interfaces []interfaceToConfigure, skipVFConfiguration bool) error {
801801
log.Log.V(2).Info("configSriovInterfaces(): start sriov configuration")
802802
for _, iface := range interfaces {
803-
if err := s.configSriovDevice(&iface.iface, skipVFConfiguration); err != nil {
804-
log.Log.Error(err, "configSriovInterfaces(): fail to configure sriov interface. resetting interface.", "address", iface.iface.PciAddress)
805-
if iface.iface.ExternallyManaged {
803+
if err := s.configSriovDevice(&iface.Iface, skipVFConfiguration); err != nil {
804+
log.Log.Error(err, "configSriovInterfaces(): fail to configure sriov interface. resetting interface.", "address", iface.Iface.PciAddress)
805+
if iface.Iface.ExternallyManaged {
806806
log.Log.V(2).Info("configSriovInterfaces(): skipping device reset as the nic is marked as externally created")
807807
} else {
808-
if resetErr := s.ResetSriovDevice(iface.ifaceStatus); resetErr != nil {
808+
if resetErr := s.ResetSriovDevice(iface.IfaceStatus); resetErr != nil {
809809
log.Log.Error(resetErr, "configSriovInterfaces(): failed to reset on error SR-IOV interface")
810810
}
811811
}
812812
return err
813813
}
814814

815815
// Save the PF status to the host
816-
err := storeManager.SaveLastPfAppliedStatus(&iface.iface)
816+
err := storeManager.SaveLastPfAppliedStatus(&iface.Iface)
817817
if err != nil {
818818
log.Log.Error(err, "configSriovInterfaces(): failed to save PF applied config to host")
819819
return err
@@ -900,49 +900,98 @@ func (s *sriov) checkForConfigAndReset(ifaceStatus sriovnetworkv1.InterfaceExt,
900900
return nil
901901
}
902902

903-
func (s *sriov) ConfigSriovDeviceVirtual(iface *sriovnetworkv1.Interface) error {
904-
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): config interface", "address", iface.PciAddress, "config", iface)
905-
// Config VFs
906-
if iface.NumVfs > 0 {
907-
if iface.NumVfs > 1 {
908-
log.Log.Error(nil, "ConfigSriovDeviceVirtual(): in a virtual environment, only one VF per interface",
909-
"numVfs", iface.NumVfs)
910-
return errors.New("NumVfs > 1")
911-
}
912-
if len(iface.VfGroups) != 1 {
913-
log.Log.Error(nil, "ConfigSriovDeviceVirtual(): missing VFGroup")
914-
return errors.New("NumVfs != 1")
915-
}
916-
addr := iface.PciAddress
917-
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "address", addr)
918-
driver := ""
919-
vfID := 0
920-
for _, group := range iface.VfGroups {
921-
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "group", group)
922-
if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {
923-
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "indexInRange", vfID)
924-
if sriovnetworkv1.StringInArray(group.DeviceType, vars.DpdkDrivers) {
925-
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "driver", group.DeviceType)
926-
driver = group.DeviceType
903+
func (s *sriov) ConfigSriovDevicesVirtual(storeManager store.ManagerInterface, interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt) error {
904+
toBeConfigured, toBeResetted, err := s.getConfigureAndReset(storeManager, interfaces, ifaceStatuses)
905+
if err != nil {
906+
log.Log.Error(err, "ConfigSriovDevicesVirtual(): cannot get a list of interfaces to configure")
907+
return fmt.Errorf("cannot get a list of interfaces to configure")
908+
}
909+
log.Log.V(2).Info("ConfigSriovDevicesVirtual(): configuration to be done", "toBeConfigured", toBeConfigured, "toBeResetted", toBeResetted)
910+
911+
for _, ifaceToConfigure := range toBeConfigured {
912+
if ifaceToConfigure.Iface.NumVfs > 0 {
913+
if ifaceToConfigure.Iface.NumVfs > 1 {
914+
log.Log.Error(nil, "ConfigSriovDeviceVirtual(): in a virtual environment, only one VF per interface",
915+
"numVfs", ifaceToConfigure.Iface.NumVfs)
916+
return errors.New("NumVfs > 1")
917+
}
918+
if len(ifaceToConfigure.Iface.VfGroups) != 1 {
919+
log.Log.Error(nil, "ConfigSriovDeviceVirtual(): missing VFGroup")
920+
return errors.New("NumVfs != 1")
921+
}
922+
addr := ifaceToConfigure.Iface.PciAddress
923+
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "address", addr)
924+
driver := ""
925+
vfID := 0
926+
for _, group := range ifaceToConfigure.Iface.VfGroups {
927+
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "group", group)
928+
if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {
929+
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "indexInRange", vfID)
930+
if sriovnetworkv1.StringInArray(group.DeviceType, vars.DpdkDrivers) {
931+
log.Log.V(2).Info("ConfigSriovDeviceVirtual()", "driver", group.DeviceType)
932+
driver = group.DeviceType
933+
}
934+
break
927935
}
928-
break
929936
}
930-
}
931-
if driver == "" {
932-
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind default")
933-
if err := s.kernelHelper.BindDefaultDriver(addr); err != nil {
934-
log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind default driver", "device", addr)
935-
return err
937+
if driver == "" {
938+
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind default")
939+
if err := s.kernelHelper.BindDefaultDriver(addr); err != nil {
940+
log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind default driver", "device", addr)
941+
return err
942+
}
943+
} else {
944+
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind driver", "driver", driver)
945+
if err := s.kernelHelper.BindDpdkDriver(addr, driver); err != nil {
946+
log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind driver for device",
947+
"driver", driver, "device", addr)
948+
return err
949+
}
936950
}
937-
} else {
938-
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind driver", "driver", driver)
939-
if err := s.kernelHelper.BindDpdkDriver(addr, driver); err != nil {
940-
log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind driver for device",
941-
"driver", driver, "device", addr)
951+
// Save the PF status to the host
952+
err := storeManager.SaveLastPfAppliedStatus(&ifaceToConfigure.Iface)
953+
if err != nil {
954+
log.Log.Error(err, "configSriovInterfaces(): failed to save PF applied config to host")
942955
return err
943956
}
944957
}
945958
}
959+
960+
for _, iface := range toBeResetted {
961+
// check if the vf were created by the sriov operator
962+
pfStatus, exist, err := storeManager.LoadPfsStatus(iface.PciAddress)
963+
if err != nil {
964+
log.Log.Error(err, "ConfigSriovDeviceVirtual(): failed to load info about PF status for device",
965+
"address", iface.PciAddress)
966+
return err
967+
}
968+
if !exist {
969+
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): PF name with pci address has VFs configured but they weren't created by the sriov operator. Skipping the device reset",
970+
"pf-name", iface.Name,
971+
"address", iface.PciAddress)
972+
continue
973+
}
974+
if pfStatus.ExternallyManaged {
975+
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): PF name with pci address was externally created skipping the device reset",
976+
"pf-name", iface.Name,
977+
"address", iface.PciAddress)
978+
continue
979+
}
980+
981+
log.Log.V(2).Info("ConfigSriovDeviceVirtual(): bind default")
982+
if err := s.kernelHelper.BindDefaultDriver(iface.PciAddress); err != nil {
983+
log.Log.Error(err, "ConfigSriovDeviceVirtual(): fail to bind default driver", "device", iface.PciAddress)
984+
return err
985+
}
986+
987+
// Remove the PF status from the host
988+
err = storeManager.RemovePfAppliedStatus(iface.PciAddress)
989+
if err != nil {
990+
log.Log.Error(err, "configSriovInterfaces(): failed to remove PF applied config from host")
991+
return err
992+
}
993+
}
994+
946995
return nil
947996
}
948997

0 commit comments

Comments
 (0)