Add health check for Net Devices

README.md

@@ -286,6 +286,8 @@ This plugin creates device plugin endpoints based on the configurations given in
 | "resourcePrefix"  | N        | Endpoint resource prefix name override. Should not contain special characters                                                          | string Default : "intel.com"                          | "yourcompany.com"                                                      |
 | "deviceType"      | N        | Device Type for a resource pool.                                                                                                       | string value of supported types. Default: "netDevice" | Currently supported values: "accelerator", "netDevice", "auxNetDevice" |
 | "excludeTopology" | N        | Exclude advertising of device's NUMA topology                                                                                          | bool Default: "false"                                 | "excludeTopology": true                                                |
+| "checkHealthOnPf" | N        | Check the health of a net device by inspecting the link state of the PF                                                                                       | bool Default: "false"                                 | "checkHealthOnPf": true                                                |
+| "checkHealthOnDeviceExist" | N        | Check the health of a net device by periodically checking if the PCI device exists in sysfs                                                                                      | bool Default: "false"                                 | "checkHealthOnDeviceExist": true                                                |
 | "selectors"       | N        | Either a single device selector map or a list of maps. The list syntax is preferred. The "deviceType" value determines the device selector options.                                                  | json list of objects or json object. Default: null                   | Example: "selectors": [{"vendors": ["8086"],"devices": ["154c"]}]        |
 | "additionalInfo" | N | A map of map to add additional information to the pod via environment variables to devices                                             | json object as string Default: null  | Example: "additionalInfo": {"*": {"token": "3e49019f-412f-4f02-824e-4cd195944205"}} |
 

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.10.0
 	github.com/vishvananda/netlink v1.3.0
+	golang.org/x/sys v0.31.0
 	google.golang.org/grpc v1.69.2
 	k8s.io/kubelet v0.32.0
 )
@@ -59,7 +60,6 @@ require (
 	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/term v0.30.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	golang.org/x/time v0.7.0 // indirect

pkg/devices/api.go

@@ -82,3 +82,17 @@ func (ad *APIDeviceImpl) GetMounts() []*pluginapi.Mount {
 func (ad *APIDeviceImpl) GetAPIDevice() *pluginapi.Device {
 	return ad.device
 }
+
+// GetHealth returns device health status
+func (ad *APIDeviceImpl) GetHealth() bool {
+	return ad.device.Health == pluginapi.Healthy
+}
+
+// SetHealth sets device health status
+func (ad *APIDeviceImpl) SetHealth(health bool) {
+	if health {
+		ad.device.Health = pluginapi.Healthy
+	} else {
+		ad.device.Health = pluginapi.Unhealthy
+	}
+}

pkg/devices/gen_net.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"github.com/golang/glog"
+	"golang.org/x/sys/unix"
 
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
@@ -117,6 +118,15 @@ func (nd *GenNetDevice) GetPfPciAddr() string {
 	return nd.pfAddr
 }
 
+// IsPfLinkUp returns whether the link has carrier and is up
+func (nd *GenNetDevice) IsPfLinkUp() (bool, error) {
+	la, err := utils.GetNetlinkProvider().GetLinkAttrs(nd.GetPfNetName())
+	if err != nil {
+		return false, err
+	}
+	return la.RawFlags&(unix.IFF_UP|unix.IFF_RUNNING) == (unix.IFF_UP | unix.IFF_RUNNING), nil
+}
+
 // GetNetName returns name of the network interface
 func (nd *GenNetDevice) GetNetName() string {
 	return nd.ifName

pkg/devices/gen_pci.go

@@ -49,6 +49,11 @@ func (pd *GenPciDevice) GetPciAddr() string {
 	return pd.pciAddr
 }
 
+// DeviceExists returns true if the device exists in the system
+func (pd *GenPciDevice) DeviceExists() bool {
+	return utils.DeviceExist(pd.GetPciAddr()) == nil
+}
+
 // GetAcpiIndex returns ACPI index of the device
 func (pd *GenPciDevice) GetAcpiIndex() string {
 	return pd.acpiIndex

pkg/netdevice/netResourcePool.go

@@ -30,6 +30,7 @@ import (
 type netResourcePool struct {
 	*resources.ResourcePoolImpl
 	nadutils types.NadUtils
+	config   *types.ResourceConfig
 }
 
 var _ types.ResourcePool = &netResourcePool{}
@@ -41,6 +42,7 @@ func NewNetResourcePool(nadutils types.NadUtils, rc *types.ResourceConfig,
 	return &netResourcePool{
 		ResourcePoolImpl: rp,
 		nadutils:         nadutils,
+		config:           rc,
 	}
 }
 
@@ -66,6 +68,66 @@ func (rp *netResourcePool) GetDeviceSpecs(deviceIDs []string) []*pluginapi.Devic
 	return devSpecs
 }
 
+func (rp *netResourcePool) Probe() bool {
+	// 1. If the physical function PF of the SR-IOV devices is carrier down. This should be marked unhealthy. Normally, SR-IOV
+	// would still function when the PF is carrier down. But in the case of DPUs/IPUs/SmartNics with an embedded CPU,
+	// the PF being down **can** signal that the embedded CPU is in reset or shutdown with carrier down.
+	// 2. If any of the devices are gone. This could be due to someone changing the number of virtual functions.
+	// Or in the case of DPUs/IPUs/SmartNics with an embedded CPU, the driver needed to reset. This will cause the
+	// virtual functions to be removed. All devices that are gone should be marked unhealthy. Normally this won't be the case
+	// since the SR-IOV Network Operator will be managing the SR-IOV devices. However for DPUs/IPUs/SmartNics with an embedded CPU,
+	// would be externally managed with a separate operator.
+	changes := false
+	cachedPfLinkStatus := make(map[string]bool)
+	for id, device := range rp.GetDevicePool() {
+		netDev, ok := device.(types.PciNetDevice)
+		if !ok {
+			// Skip devices that are not PCI net devices
+			continue
+		}
+		currentHealth := device.GetHealth()
+		pfName := netDev.GetPfNetName()
+
+		pfIsUp := true
+		var err error
+		pfIsUpLog := ""
+		if rp.config.CheckHealthOnPf {
+			if cachedStatus, exists := cachedPfLinkStatus[pfName]; exists {
+				pfIsUp = cachedStatus
+			} else {
+				pfIsUp, err = netDev.IsPfLinkUp()
+				if err != nil {
+					// If we can't check the link status, assume it's up. It could be that the PF was moved to a different netns.
+					// We want a conservative approach, as we don't want to mark the device as unhealthy if we are unsure.
+					pfIsUp = true
+				}
+				cachedPfLinkStatus[pfName] = pfIsUp
+			}
+			pfIsUpLog = fmt.Sprintf("PF %s is %s,", pfName, map[bool]string{true: "Up", false: "Down"}[pfIsUp])
+		}
+
+		deviceExists := true
+		deviceExistsLog := ""
+		if rp.config.CheckHealthOnDeviceExist {
+			deviceExists = netDev.DeviceExists()
+			deviceExistsLog = fmt.Sprintf("Device %s is %s,", netDev.GetPciAddr(),
+				map[bool]string{true: "existing", false: "missing"}[deviceExists])
+		}
+
+		if pfIsUp && deviceExists && !currentHealth {
+			glog.Infof("%s %s device was unhealthy, marking device %s as healthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(true)
+			changes = true
+		} else if (!pfIsUp || !deviceExists) && currentHealth {
+			// If either the PF is down or the device is missing:
+			glog.Infof("%s %s device was healthy, marking device %s as unhealthy", pfIsUpLog, deviceExistsLog, id)
+			device.SetHealth(false)
+			changes = true
+		}
+	}
+	return changes
+}
+
 // StoreDeviceInfoFile stores the Device Info files according to the
 // k8snetworkplumbingwg/device-info-spec
 // for the requested deviceIDs

pkg/netdevice/netResourcePool_test.go

@@ -237,4 +237,117 @@ var _ = Describe("NetResourcePool", func() {
 			})
 		})
 	})
+	Describe("Health Checking on Net Devices", func() {
+		Context("PF link status and device existence for VFs with Same PF", func() {
+			var fake1health bool
+			var fake2health bool
+			var fake1 *mocks.PciNetDevice
+			var fake2 *mocks.PciNetDevice
+			var rp types.ResourcePool
+
+			BeforeEach(func() {
+				rf := factory.NewResourceFactory("fake", "fake", true, false)
+				nadutils := rf.GetNadUtils()
+				rc := &types.ResourceConfig{
+					ResourceName:             "fake",
+					ResourcePrefix:           "fake",
+					CheckHealthOnPf:          true,
+					CheckHealthOnDeviceExist: true,
+					SelectorObjs:             []interface{}{&types.NetDeviceSelectors{}},
+				}
+
+				fake1 = &mocks.PciNetDevice{}
+				fake1.On("GetPfNetName").Return("fakepf1")
+				fake1.On("GetPciAddr").Return("0000:01:00.1")
+				fake1.On("SetHealth", Anything).Run(func(args Arguments) {
+					fake1health = args.Bool(0)
+				}).Return()
+				fake1health = true
+
+				fake2 = &mocks.PciNetDevice{}
+				fake2.On("GetPfNetName").Return("fakepf1")
+				fake2.On("GetPciAddr").Return("0000:01:00.2")
+				fake2.On("SetHealth", Anything).Run(func(args Arguments) {
+					fake2health = args.Bool(0)
+				}).Return()
+				fake2health = true
+
+				pcis := map[string]types.HostDevice{"fake1": fake1, "fake2": fake2}
+
+				rp = netdevice.NewNetResourcePool(nadutils, rc, pcis)
+			})
+
+			SetCurrentHealth := func(health bool) {
+				fake1health = health
+				fake2health = health
+				fake1.On("GetHealth").Return(health).Once()
+				fake2.On("GetHealth").Return(health).Once()
+			}
+
+			SetCurrentLinkState := func(up bool) {
+				fake1.On("IsPfLinkUp").Return(up, nil).Once()
+				fake2.On("IsPfLinkUp").Return(up, nil).Once()
+			}
+
+			SetCurrentDeviceExistance := func(exist bool) {
+				fake1.On("DeviceExists").Return(exist).Once()
+				fake2.On("DeviceExists").Return(exist).Once()
+			}
+
+			SetupHealthProbeTest := func(health, link, exist bool) {
+				SetCurrentHealth(health)
+				SetCurrentLinkState(link)
+				SetCurrentDeviceExistance(exist)
+			}
+
+			It("Currently Device Healthy, PF Link Up, Device exists", func() {
+				SetupHealthProbeTest(true, true, true)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeTrue())
+				Expect(fake2health).To(BeTrue())
+			})
+			It("Currently Device Healthy, PF Link Up, Device missing", func() {
+				SetupHealthProbeTest(true, true, false)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Healthy, PF Link Down, Device exists", func() {
+				SetupHealthProbeTest(true, false, true)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Healthy, PF Link Down, Device missing", func() {
+				SetupHealthProbeTest(true, false, false)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Up, Device exists", func() {
+				SetupHealthProbeTest(false, true, true)
+				Expect(rp.Probe()).To(BeTrue())
+				Expect(fake1health).To(BeTrue())
+				Expect(fake2health).To(BeTrue())
+			})
+			It("Currently Device Unhealthy, PF Link Up, Device missing", func() {
+				SetupHealthProbeTest(false, true, false)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Down, Device exists", func() {
+				SetupHealthProbeTest(false, false, true)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+			It("Currently Device Unhealthy, PF Link Down, Device missing", func() {
+				SetupHealthProbeTest(false, false, false)
+				Expect(rp.Probe()).To(BeFalse())
+				Expect(fake1health).To(BeFalse())
+				Expect(fake2health).To(BeFalse())
+			})
+		})
+	})
 })

pkg/resources/server.go

@@ -73,7 +73,7 @@ func NewResourceServer(prefix, suffix string, pluginWatch, useCdi bool, rp types
 		termSignal:         make(chan bool, 1),
 		updateSignal:       make(chan bool),
 		stopWatcher:        make(chan bool),
-		checkIntervals:     20, // updates every 20 seconds
+		checkIntervals:     5, // updates every 5 seconds
 		cdi:                cdiPkg.New(),
 	}
 }