Implement Probe() function for netResourcePool

The implementation checks:
1. If the physical function PF of the SR-IOV devices is carrier down. This
should be marked unhealthy. Normally, SR-IOV would still function when the
PF is carrier down. But in the case of DPUs/IPUs/SmartNics with an embedded
CPU, the PF being down **can** signal that the embedded CPU is in reset or
shutdown with carrier down.
2. If any of the devices are gone. This could be due to someone changing
the number of virtual functions. Or in the case of DPUs/IPUs/SmartNics with
an embedded CPU, the driver needed to reset. This will cause the virtual
functions to be removed. All devices that are gone should be marked
unhealthy. Normally this won't be the case since the SR-IOV Network Operator
will be managing the SR-IOV devices. However for DPUs/IPUs/SmartNics with
an embedded CPU, would be externally managed with a separate operator.

Both these can be switched on and off using checkHealthOnPf and
checkHealthOnDeviceExist within the resource config.

Signed-off-by: William Zhao <[email protected]>

Author: wizhaoredhat

README.md

@@ -286,6 +286,8 @@ This plugin creates device plugin endpoints based on the configurations given in
 | "resourcePrefix"  | N        | Endpoint resource prefix name override. Should not contain special characters                                                          | string Default : "intel.com"                          | "yourcompany.com"                                                      |
 | "deviceType"      | N        | Device Type for a resource pool.                                                                                                       | string value of supported types. Default: "netDevice" | Currently supported values: "accelerator", "netDevice", "auxNetDevice" |
 | "excludeTopology" | N        | Exclude advertising of device's NUMA topology                                                                                          | bool Default: "false"                                 | "excludeTopology": true                                                |
+| "checkHealthOnPf" | N        | Check the health of a net device by inspecting the link state of the PF                                                                                       | bool Default: "false"                                 | "checkHealthOnPf": true                                                |
+| "checkHealthOnDeviceExist" | N        | Check the health of a net device by periodically checking if the PCI device exists in sysfs                                                                                      | bool Default: "false"                                 | "checkHealthOnDeviceExist": true                                                |
 | "selectors"       | N        | Either a single device selector map or a list of maps. The list syntax is preferred. The "deviceType" value determines the device selector options.                                                  | json list of objects or json object. Default: null                   | Example: "selectors": [{"vendors": ["8086"],"devices": ["154c"]}]        |
 | "additionalInfo" | N | A map of map to add additional information to the pod via environment variables to devices                                             | json object as string Default: null  | Example: "additionalInfo": {"*": {"token": "3e49019f-412f-4f02-824e-4cd195944205"}} |
 

pkg/netdevice/netResourcePool.go

@@ -30,6 +30,7 @@ import (
 type netResourcePool struct {
 	*resources.ResourcePoolImpl
 	nadutils types.NadUtils
+	config   *types.ResourceConfig
 }
 
 var _ types.ResourcePool = &netResourcePool{}
@@ -41,6 +42,7 @@ func NewNetResourcePool(nadutils types.NadUtils, rc *types.ResourceConfig,
 	return &netResourcePool{
 		ResourcePoolImpl: rp,
 		nadutils:         nadutils,
+		config:           rc,
 	}
 }
 
@@ -66,6 +68,82 @@ func (rp *netResourcePool) GetDeviceSpecs(deviceIDs []string) []*pluginapi.Devic
 	return devSpecs
 }
 
+func (rp *netResourcePool) Probe() bool {
+	// 1. If the physical function PF of the SR-IOV devices is carrier down. This should be marked unhealthy. Normally, SR-IOV
+	// would still function when the PF is carrier down. But in the case of DPUs/IPUs/SmartNics with an embedded CPU,
+	// the PF being down **can** signal that the embedded CPU is in reset or shutdown with carrier down.
+	// 2. If any of the devices are gone. This could be due to someone changing the number of virtual functions.
+	// Or in the case of DPUs/IPUs/SmartNics with an embedded CPU, the driver needed to reset. This will cause the
+	// virtual functions to be removed. All devices that are gone should be marked unhealthy. Normally this won't be the case
+	// since the SR-IOV Network Operator will be managing the SR-IOV devices. However for DPUs/IPUs/SmartNics with an embedded CPU,
+	// would be externally managed with a separate operator.
+	changes := false
+	cachedPfLinkStatus := make(map[string]bool)
+	for id, device := range rp.GetDevicePool() {
+		netDev, ok := device.(types.PciNetDevice)
+		if !ok {
+			// Skip devices that are not PCI net devices
+			continue
+		}
+		currentHealth := device.GetHealth()
+		pfName := netDev.GetPfNetName()
+
+		pfIsUp := true
+		var err error
+		pfIsUpLog := ""
+		if rp.config.CheckHealthOnPf {
+			if cachedStatus, exists := cachedPfLinkStatus[pfName]; exists {
+				pfIsUp = cachedStatus
+			} else {
+				pfIsUp, err = netDev.IsPfLinkUp()
+				if err != nil {
+					// If we can't check the link status, assume it's up. It could be that the PF was moved to a different netns.
+					// We want a conservative approach, as we don't want to mark the device as unhealthy if we are unsure.
+					pfIsUp = true
+				}
+				cachedPfLinkStatus[pfName] = pfIsUp
+			}
+			pfIsUpLog = fmt.Sprintf("PF %s", pfName)
+			if pfIsUp {
+				pfIsUpLog = fmt.Sprintf("%s is UP, ", pfIsUpLog)
+			} else {
+				pfIsUpLog = fmt.Sprintf("%s is DOWN, ", pfIsUpLog)
+			}
+		}
+
+		deviceExists := true
+		deviceExistsLog := ""
+		if rp.config.CheckHealthOnDeviceExist {
+			deviceExists = netDev.DeviceExists()
+			deviceExistsLog = fmt.Sprintf("Device %s", netDev.GetPciAddr())
+			if deviceExists {
+				deviceExistsLog = fmt.Sprintf("%s is existing, ", deviceExistsLog)
+			} else {
+				deviceExistsLog = fmt.Sprintf("%s is missing, ", deviceExistsLog)
+			}
+		}
+
+		if pfIsUp && deviceExists && !currentHealth {
+			glog.Infof("%s%sdevice was unhealthy, marking device %s as healthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(true)
+			changes = true
+		} else if !pfIsUp && deviceExists && currentHealth {
+			glog.Infof("%s%sdevice was healthy, marking device %s as unhealthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(false)
+			changes = true
+		} else if pfIsUp && !deviceExists && currentHealth {
+			glog.Infof("%s%sdevice was healthy, marking device %s as unhealthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(false)
+			changes = true
+		} else if !pfIsUp && !deviceExists && currentHealth {
+			glog.Infof("%s%sdevice was healthy, marking device %s as unhealthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(false)
+			changes = true
+		}
+	}
+	return changes
+}
+
 // StoreDeviceInfoFile stores the Device Info files according to the
 // k8snetworkplumbingwg/device-info-spec
 // for the requested deviceIDs

pkg/netdevice/netResourcePool_test.go

@@ -237,4 +237,117 @@ var _ = Describe("NetResourcePool", func() {
 			})
 		})
 	})
+	Describe("Health Checking on Net Devices", func() {
+		Context("PF link status and device existence for VFs with Same PF", func() {
+			var fake1health bool
+			var fake2health bool
+			var fake1 *mocks.PciNetDevice
+			var fake2 *mocks.PciNetDevice
+			var rp types.ResourcePool
+
+			BeforeEach(func() {
+				rf := factory.NewResourceFactory("fake", "fake", true, false)
+				nadutils := rf.GetNadUtils()
+				rc := &types.ResourceConfig{
+					ResourceName:             "fake",
+					ResourcePrefix:           "fake",
+					CheckHealthOnPf:          true,
+					CheckHealthOnDeviceExist: true,
+					SelectorObjs:             []interface{}{&types.NetDeviceSelectors{}},
+				}
+
+				fake1 = &mocks.PciNetDevice{}
+				fake1.On("GetPfNetName").Return("fakepf1")
+				fake1.On("GetPciAddr").Return("0000:01:00.1")
+				fake1.On("SetHealth", Anything).Run(func(args Arguments) {
+					fake1health = args.Bool(0)
+				}).Return()
+				fake1health = true
+
+				fake2 = &mocks.PciNetDevice{}
+				fake2.On("GetPfNetName").Return("fakepf1")
+				fake2.On("GetPciAddr").Return("0000:01:00.2")
+				fake2.On("SetHealth", Anything).Run(func(args Arguments) {
+					fake2health = args.Bool(0)
+				}).Return()
+				fake2health = true
+
+				pcis := map[string]types.HostDevice{"fake1": fake1, "fake2": fake2}
+
+				rp = netdevice.NewNetResourcePool(nadutils, rc, pcis)
+			})
+
+			SetCurrentHealth := func(health bool) {
+				fake1health = health
+				fake2health = health
+				fake1.On("GetHealth").Return(health).Once()
+				fake2.On("GetHealth").Return(health).Once()
+			}
+
+			SetCurrentLinkState := func(up bool) {
+				fake1.On("IsPfLinkUp").Return(up, nil).Once()
+				fake2.On("IsPfLinkUp").Return(up, nil).Once()
+			}
+
+			SetCurrentDeviceExistance := func(exist bool) {
+				fake1.On("DeviceExists").Return(exist, nil).Once()
+				fake2.On("DeviceExists").Return(exist, nil).Once()
+			}
+
+			SetupHealthProbeTest := func(health, link, exist bool) {
+				SetCurrentHealth(health)
+				SetCurrentLinkState(link)
+				SetCurrentDeviceExistance(exist)
+			}
+
+			It("Currently Device Healthy, PF Link Up, Device exists", func() {
+				SetupHealthProbeTest(true, true, true)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeTrue())
+				Expect(fake2health).To(BeTrue())
+			})
+			It("Currently Device Healthy, PF Link Up, Device missing", func() {
+				SetupHealthProbeTest(true, true, false)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Healthy, PF Link Down, Device exists", func() {
+				SetupHealthProbeTest(true, false, true)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Healthy, PF Link Down, Device missing", func() {
+				SetupHealthProbeTest(true, false, false)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Up, Device exists", func() {
+				SetupHealthProbeTest(false, true, true)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeTrue())
+				Expect(fake2health).To(BeTrue())
+			})
+			It("Currently Device Unhealthy, PF Link Up, Device missing", func() {
+				SetupHealthProbeTest(false, true, false)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Down, Device exists", func() {
+				SetupHealthProbeTest(false, false, true)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Down, Device missing", func() {
+				SetupHealthProbeTest(false, false, false)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+		})
+	})
 })

pkg/resources/server.go

@@ -73,7 +73,7 @@ func NewResourceServer(prefix, suffix string, pluginWatch, useCdi bool, rp types
 		termSignal:         make(chan bool, 1),
 		updateSignal:       make(chan bool),
 		stopWatcher:        make(chan bool),
-		checkIntervals:     20, // updates every 20 seconds
+		checkIntervals:     5, // updates every 5 seconds
 		cdi:                cdiPkg.New(),
 	}
 }

pkg/types/types.go

@@ -97,12 +97,14 @@ type ResourceConfig struct {
 	// optional resource prefix that will overwrite	global prefix specified in cli params
 	ResourcePrefix string `json:"resourcePrefix,omitempty"`
 	//nolint:lll
-	ResourceName    string                    `json:"resourceName"` // the resource name will be added with resource prefix in K8s api
-	DeviceType      DeviceType                `json:"deviceType,omitempty"`
-	ExcludeTopology bool                      `json:"excludeTopology,omitempty"`
-	Selectors       *json.RawMessage          `json:"selectors,omitempty"`
-	AdditionalInfo  map[string]AdditionalInfo `json:"additionalInfo,omitempty"`
-	SelectorObjs    []interface{}
+	ResourceName             string                    `json:"resourceName"` // the resource name will be added with resource prefix in K8s api
+	DeviceType               DeviceType                `json:"deviceType,omitempty"`
+	ExcludeTopology          bool                      `json:"excludeTopology,omitempty"`
+	CheckHealthOnPf          bool                      `json:"checkHealthOnPf,omitempty"`
+	CheckHealthOnDeviceExist bool                      `json:"checkHealthOnDeviceExist,omitempty"`
+	Selectors                *json.RawMessage          `json:"selectors,omitempty"`
+	AdditionalInfo           map[string]AdditionalInfo `json:"additionalInfo,omitempty"`
+	SelectorObjs             []interface{}
 }
 
 // DeviceSelectors contains common device selectors fields