Fix "message too long" error for InfiniBand devices with many VFs

SchSeba · cursoragent · SchSeba · commit 313f8b16b283 · 2026-02-18T13:51:52.000Z
When discovering or configuring InfiniBand devices with many VFs,
netlink.LinkByName() can fail with EMSGSIZE (message too long) because
the kernel's netlink response exceeds the message limit (~16KB). This
happens because LinkByName requests VF info which generates large
messages for IB devices with their GUIDs.

This commit adds two new methods to the netlink wrapper:

1. LinkByNameForSetVf: Returns a minimal link object (name and index
   only) by reading from sysfs. Sufficient for VF operations like
   LinkSetVfNodeGUID.

2. LinkByNameWithBasicInfo: Returns a link with basic info (name,
   index, MTU, MAC, EncapType). If netlink fails with EMSGSIZE, it
   falls back to reading these values from sysfs.

The fix is applied to:
- DiscoverSriovDevices: Uses LinkByNameWithBasicInfo for PF discovery
- getVfInfo: Uses LinkByNameWithBasicInfo for VF info discovery
- GetLinkType: Uses LinkByNameWithBasicInfo for link type detection
- configSriovVFDevices: Falls back to LinkByNameForSetVf on EMSGSIZE
- configSriovDevice: Falls back to LinkByNameForSetVf on EMSGSIZE
- GetNetdevMTU: Uses LinkByNameWithBasicInfo
- GetNetDevMac: Uses LinkByNameWithBasicInfo
- SetNetdevMTU: Falls back to LinkByNameForSetVf on EMSGSIZE

Includes path traversal protection for interface names to prevent
security issues when reading from sysfs.

Signed-off-by: User &lt;user@example.com&gt;
Co-authored-by: Cursor &lt;cursoragent@cursor.com&gt;
diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go
diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go
@@ -1,9 +1,16 @@
 package netlink
 
 import (
+	"errors"
+	"fmt"
 	"net"
+	"os"
+	"strconv"
+	"strings"
+	"syscall"
 
 	"github.com/vishvananda/netlink"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 func New() NetlinkLib {
@@ -24,6 +31,14 @@ type NetlinkLib interface {
 	LinkSetVfPortGUID(link Link, vf int, portguid net.HardwareAddr) error
 	// LinkByName finds a link by name and returns a pointer to the object.
 	LinkByName(name string) (Link, error)
+	// LinkByNameForSetVf returns a minimal link object (name and index only) that can be used
+	// for VF operations like LinkSetVfNodeGUID. This reads from sysfs instead of netlink
+	// to avoid the "message too long" error for InfiniBand devices with many VFs.
+	LinkByNameForSetVf(name string) (Link, error)
+	// LinkByNameWithBasicInfo returns a link with basic info (name, index, MTU, MAC, EncapType).
+	// If standard netlink query fails with EMSGSIZE (common for IB devices with many VFs),
+	// it falls back to reading from sysfs.
+	LinkByNameWithBasicInfo(name string) (Link, error)
 	// LinkByIndex finds a link by index and returns a pointer to the object.
 	LinkByIndex(index int) (Link, error)
 	// LinkList gets a list of link devices.
@@ -91,6 +106,128 @@ func (w *libWrapper) LinkByName(name string) (Link, error) {
 	return netlink.LinkByName(name)
 }
 
+// LinkByNameForSetVf returns a minimal link object (name and index only) that can be used
+// for VF operations like LinkSetVfNodeGUID. This reads from sysfs instead of netlink
+// to avoid the "message too long" error for InfiniBand devices with many VFs.
+func (w *libWrapper) LinkByNameForSetVf(name string) (Link, error) {
+	log.Log.V(2).Info("LinkByNameForSetVf(): getting minimal link info from sysfs", "name", name)
+
+	// Sanitize the interface name to prevent path traversal
+	if strings.ContainsAny(name, "/\\") {
+		return nil, fmt.Errorf("invalid interface name, contains path separators: %s", name)
+	}
+
+	// Read interface index from sysfs
+	indexPath := fmt.Sprintf("/sys/class/net/%s/ifindex", name)
+	indexData, err := os.ReadFile(indexPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read interface index from %s: %w", indexPath, err)
+	}
+
+	index, err := strconv.Atoi(strings.TrimSpace(string(indexData)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse interface index: %w", err)
+	}
+
+	// Create a minimal Device link with just name and index
+	// This is sufficient for VF operations like LinkSetVfNodeGUID
+	link := &netlink.Device{}
+	link.Attrs().Name = name
+	link.Attrs().Index = index
+
+	log.Log.V(2).Info("LinkByNameForSetVf(): created minimal link", "name", name, "index", index)
+	return link, nil
+}
+
+// LinkByNameWithBasicInfo returns a link with basic info (name, index, MTU, MAC, EncapType).
+// If standard netlink query fails with EMSGSIZE (common for IB devices with many VFs),
+// it falls back to reading from sysfs.
+func (w *libWrapper) LinkByNameWithBasicInfo(name string) (Link, error) {
+	// First, try standard LinkByName
+	link, err := netlink.LinkByName(name)
+	if err == nil {
+		return link, nil
+	}
+
+	// If it's not EMSGSIZE, return the original error
+	if !errors.Is(err, syscall.EMSGSIZE) {
+		return nil, err
+	}
+
+	log.Log.V(2).Info("LinkByNameWithBasicInfo(): LinkByName failed with EMSGSIZE, using sysfs fallback", "name", name)
+
+	// Sanitize the interface name to prevent path traversal
+	if strings.ContainsAny(name, "/\\") {
+		return nil, fmt.Errorf("invalid interface name, contains path separators: %s", name)
+	}
+
+	basePath := fmt.Sprintf("/sys/class/net/%s", name)
+
+	// Read interface index
+	indexData, err := os.ReadFile(basePath + "/ifindex")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read interface index: %w", err)
+	}
+	index, err := strconv.Atoi(strings.TrimSpace(string(indexData)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse interface index: %w", err)
+	}
+
+	// Read MTU
+	mtuData, err := os.ReadFile(basePath + "/mtu")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MTU: %w", err)
+	}
+	mtu, err := strconv.Atoi(strings.TrimSpace(string(mtuData)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse MTU: %w", err)
+	}
+
+	// Read MAC address
+	macData, err := os.ReadFile(basePath + "/address")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read MAC address: %w", err)
+	}
+	mac, err := net.ParseMAC(strings.TrimSpace(string(macData)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse MAC address: %w", err)
+	}
+
+	// Read interface type to determine encap type
+	typeData, err := os.ReadFile(basePath + "/type")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read interface type: %w", err)
+	}
+	ifType, err := strconv.Atoi(strings.TrimSpace(string(typeData)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse interface type: %w", err)
+	}
+
+	// Map ARPHRD type to encap type string
+	// See linux/if_arp.h for ARPHRD_* constants
+	var encapType string
+	switch ifType {
+	case 1: // ARPHRD_ETHER
+		encapType = "ether"
+	case 32: // ARPHRD_INFINIBAND
+		encapType = "infiniband"
+	default:
+		encapType = fmt.Sprintf("unknown(%d)", ifType)
+	}
+
+	// Create a link with the basic info
+	link = &netlink.Device{}
+	link.Attrs().Name = name
+	link.Attrs().Index = index
+	link.Attrs().MTU = mtu
+	link.Attrs().HardwareAddr = mac
+	link.Attrs().EncapType = encapType
+
+	log.Log.V(2).Info("LinkByNameWithBasicInfo(): created link from sysfs",
+		"name", name, "index", index, "mtu", mtu, "mac", mac.String(), "encapType", encapType)
+	return link, nil
+}
+
 // LinkByIndex finds a link by index and returns a pointer to the object.
 func (w *libWrapper) LinkByIndex(index int) (Link, error) {
 	return netlink.LinkByIndex(index)
diff --git a/pkg/host/internal/network/network.go b/pkg/host/internal/network/network.go
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/cenkalti/backoff"
@@ -161,7 +162,9 @@ func (n *network) GetNetdevMTU(pciAddr string) int {
 		return 0
 	}
 
-	link, err := n.netlinkLib.LinkByName(ifaceName)
+	// Use LinkByNameWithBasicInfo which falls back to sysfs on EMSGSIZE.
+	// This handles InfiniBand devices with many VFs.
+	link, err := n.netlinkLib.LinkByNameWithBasicInfo(ifaceName)
 	if err != nil {
 		log.Log.Error(err, "GetNetdevMTU(): fail to get Link ", "device", ifaceName)
 		return 0
@@ -184,10 +187,21 @@ func (n *network) SetNetdevMTU(pciAddr string, mtu int) error {
 			return fmt.Errorf("failed to get netdevice for device %s", pciAddr)
 		}
 
+		// Try standard LinkByName first. For InfiniBand devices with many VFs, this may fail
+		// with EMSGSIZE because the kernel's netlink response exceeds the message limit.
+		// In that case, fall back to LinkByNameForSetVf which reads minimal info from sysfs.
+		// LinkSetMTU only needs the link index.
 		link, err := n.netlinkLib.LinkByName(ifaceName)
 		if err != nil {
-			log.Log.Error(err, "SetNetdevMTU(): fail to get Link ", "device", ifaceName)
-			return err
+			if errors.Is(err, syscall.EMSGSIZE) {
+				log.Log.V(2).Info("SetNetdevMTU(): LinkByName failed with EMSGSIZE, using sysfs fallback",
+					"device", ifaceName)
+				link, err = n.netlinkLib.LinkByNameForSetVf(ifaceName)
+			}
+			if err != nil {
+				log.Log.Error(err, "SetNetdevMTU(): fail to get Link ", "device", ifaceName)
+				return err
+			}
 		}
 		return n.netlinkLib.LinkSetMTU(link, mtu)
 	}, backoff.WithMaxRetries(b, 10))
@@ -203,7 +217,9 @@ func (n *network) SetNetdevMTU(pciAddr string, mtu int) error {
 // retrieved.
 func (n *network) GetNetDevMac(ifaceName string) string {
 	log.Log.V(2).Info("GetNetDevMac(): get Mac", "device", ifaceName)
-	link, err := n.netlinkLib.LinkByName(ifaceName)
+	// Use LinkByNameWithBasicInfo which falls back to sysfs on EMSGSIZE.
+	// This handles InfiniBand devices with many VFs.
+	link, err := n.netlinkLib.LinkByNameWithBasicInfo(ifaceName)
 	if err != nil {
 		log.Log.Error(err, "GetNetDevMac(): failed to get Link", "device", ifaceName)
 		return ""
diff --git a/pkg/host/internal/network/network_test.go b/pkg/host/internal/network/network_test.go
@@ -358,14 +358,14 @@ var _ = Describe("Network", func() {
 		})
 		It("should return 0 if not able to get interface by name", func() {
 			dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil)
-			netlinkLibMock.EXPECT().LinkByName("eno1").Return(nil, fmt.Errorf("failed to get interface"))
+			netlinkLibMock.EXPECT().LinkByNameWithBasicInfo("eno1").Return(nil, fmt.Errorf("failed to get interface"))
 			mtu := n.GetNetdevMTU("0000:d8:00.0")
 			Expect(mtu).To(Equal(0))
 		})
 		It("should return mtu for interface", func() {
 			dputilsLibMock.EXPECT().GetNetNames("0000:d8:00.0").Return([]string{"eno1"}, nil)
 			link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1", MTU: 1500}}
-			netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil)
+			netlinkLibMock.EXPECT().LinkByNameWithBasicInfo("eno1").Return(link, nil)
 			mtu := n.GetNetdevMTU("0000:d8:00.0")
 			Expect(mtu).To(Equal(1500))
 		})
@@ -386,13 +386,13 @@ var _ = Describe("Network", func() {
 	})
 	Context("GetNetDevMac", func() {
 		It("should return empty mac address if not able to get interface by link", func() {
-			netlinkLibMock.EXPECT().LinkByName("eno1").Return(nil, fmt.Errorf("failed to find intreface"))
+			netlinkLibMock.EXPECT().LinkByNameWithBasicInfo("eno1").Return(nil, fmt.Errorf("failed to find intreface"))
 			mac := n.GetNetDevMac("eno1")
 			Expect(mac).To(BeEmpty())
 		})
 		It("should return interface mac address", func() {
 			link := &netlink.GenericLink{LinkType: "PF", LinkAttrs: netlink.LinkAttrs{Name: "eno1", HardwareAddr: net.HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}}}
-			netlinkLibMock.EXPECT().LinkByName("eno1").Return(link, nil)
+			netlinkLibMock.EXPECT().LinkByNameWithBasicInfo("eno1").Return(link, nil)
 			mac := n.GetNetDevMac("eno1")
 			Expect(mac).To(Equal("00:00:5e:00:53:01"))
 		})
diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go
@@ -149,7 +149,9 @@ func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devi
 	}
 
 	if name := s.networkHelper.TryGetInterfaceName(vfAddr); name != "" {
-		link, err := s.netlinkLib.LinkByName(name)
+		// Use LinkByNameWithBasicInfo which falls back to sysfs on EMSGSIZE.
+		// This handles InfiniBand VFs where netlink can fail.
+		link, err := s.netlinkLib.LinkByNameWithBasicInfo(name)
 		if err != nil {
 			log.Log.Error(err, "getVfInfo(): unable to get VF Link Object", "name", name, "device", vfAddr)
 		} else {
@@ -270,7 +272,10 @@ func (s *sriov) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]sri
 			continue
 		}
 
-		link, err := s.netlinkLib.LinkByName(pfNetName)
+		// Use LinkByNameWithBasicInfo which falls back to sysfs if netlink fails with EMSGSIZE.
+		// This handles InfiniBand devices with many VFs where the netlink response can exceed
+		// the kernel's message size limit.
+		link, err := s.netlinkLib.LinkByNameWithBasicInfo(pfNetName)
 		if err != nil {
 			log.Log.Error(err, "DiscoverSriovDevices(): unable to get Link for device, skipping", "device", device.Address)
 			continue
@@ -507,10 +512,20 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
 		if err != nil {
 			log.Log.Error(err, "configSriovVFDevices(): unable to parse VFs for device", "device", iface.PciAddress)
 		}
+		// Try standard LinkByName first. For InfiniBand devices with many VFs, this may fail
+		// with EMSGSIZE because the kernel's netlink response exceeds the message limit.
+		// In that case, fall back to LinkByNameForSetVf which reads minimal info from sysfs.
 		pfLink, err := s.netlinkLib.LinkByName(iface.Name)
 		if err != nil {
-			log.Log.Error(err, "configSriovVFDevices(): unable to get PF link for device", "device", iface)
-			return err
+			if errors.Is(err, syscall.EMSGSIZE) {
+				log.Log.V(2).Info("configSriovVFDevices(): LinkByName failed with EMSGSIZE, using sysfs fallback",
+					"device", iface.Name)
+				pfLink, err = s.netlinkLib.LinkByNameForSetVf(iface.Name)
+			}
+			if err != nil {
+				log.Log.Error(err, "configSriovVFDevices(): unable to get PF link for device", "device", iface)
+				return err
+			}
 		}
 
 		for _, addr := range vfAddrs {
@@ -655,10 +670,23 @@ func (s *sriov) configSriovDevice(iface *sriovnetworkv1.Interface, skipVFConfigu
 		return err
 	}
 	// Set PF link up
+	// Try standard LinkByName first. For InfiniBand devices with many VFs, this may fail
+	// with EMSGSIZE because the kernel's netlink response exceeds the message limit.
+	// In that case, fall back to LinkByNameForSetVf which reads minimal info from sysfs.
 	pfLink, err := s.netlinkLib.LinkByName(iface.Name)
 	if err != nil {
-		return err
+		if errors.Is(err, syscall.EMSGSIZE) {
+			log.Log.V(2).Info("configSriovDevice(): LinkByName failed with EMSGSIZE, using sysfs fallback",
+				"device", iface.Name)
+			pfLink, err = s.netlinkLib.LinkByNameForSetVf(iface.Name)
+		}
+		if err != nil {
+			return err
+		}
 	}
+	// Note: With the sysfs fallback, IsLinkAdminStateUp will always return false because
+	// the minimal link object doesn't have the Flags field populated. However, LinkSetUp
+	// is idempotent - calling it on an already-up link is a no-op.
 	if !s.netlinkLib.IsLinkAdminStateUp(pfLink) {
 		err = s.netlinkLib.LinkSetUp(pfLink)
 		if err != nil {
@@ -1042,7 +1070,9 @@ func (s *sriov) SetNicSriovMode(pciAddress string, mode string) error {
 
 func (s *sriov) GetLinkType(name string) string {
 	log.Log.V(2).Info("GetLinkType()", "name", name)
-	link, err := s.netlinkLib.LinkByName(name)
+	// Use LinkByNameWithBasicInfo which falls back to sysfs if netlink fails with EMSGSIZE.
+	// This handles InfiniBand devices with many VFs.
+	link, err := s.netlinkLib.LinkByNameWithBasicInfo(name)
 	if err != nil {
 		log.Log.Error(err, "GetLinkType(): failed to get link", "device", name)
 		return ""
diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go
