feat: Add RDMA device support with CDI integration

Implements RDMA capability detection and device exposure using the
Mellanox rdmamap library. Adds discovery of RDMA devices (mlx5_*),
protocol detection (RoCE/InfiniBand/iWARP), and automatic injection
of character devices (/dev/infiniband/*) into containers via CDI.

Key changes:
- Add RdmaProvider interface with rdmamap integration
- Expose RDMA character devices (uverbs, umad, issm, rdma_cm)
- Set environment variables for device paths and names
- Add comprehensive unit tests for RDMA functionality

Matches feature parity with sriov-network-device-plugin for RDMA.

Tested on Mellanox ConnectX hardware with RoCE.

Signed-off-by: Fred Rolland <frolland@nvidia.com>

Author: rollandf

pkg/devicestate/state.go

@@ -215,6 +215,16 @@ func (s *Manager) applyConfigOnDevice(ctx context.Context, ifNameIndex *int, cla
 		})
 	}
 
+	// If device is RDMA capable, add RDMA character devices
+	if rdmaCapableAttr, ok := deviceInfo.Attributes[consts.AttributeRDMACapable]; ok && rdmaCapableAttr.BoolValue != nil && *rdmaCapableAttr.BoolValue {
+		rdmaDeviceNodes, rdmaEnvs, err := s.handleRDMADevice(ctx, pciAddress, result.Device)
+		if err != nil {
+			return nil, fmt.Errorf("error handling RDMA device: %w", err)
+		}
+		deviceNodes = append(deviceNodes, rdmaDeviceNodes...)
+		envs = append(envs, rdmaEnvs...)
+	}
+
 	edits := &cdispec.ContainerEdits{
 		Env:         envs,
 		DeviceNodes: deviceNodes,
@@ -254,6 +264,79 @@ func (s *Manager) applyConfigOnDevice(ctx context.Context, ifNameIndex *int, cla
 	return preparedDevice, nil
 }
 
+// handleRDMADevice handles RDMA device configuration and returns device nodes, environment variables, or an error
+func (s *Manager) handleRDMADevice(ctx context.Context, pciAddress, deviceName string) ([]*cdispec.DeviceNode, []string, error) {
+	logger := klog.FromContext(ctx).WithName("handleRDMADevice")
+	var deviceNodes []*cdispec.DeviceNode
+	var envs []string
+
+	rdmaDevices, err := host.GetHelpers().GetRDMADeviceForPCI(pciAddress)
+	if err != nil {
+		logger.Error(err, "Failed to get RDMA devices for PCI address",
+			"device", pciAddress)
+		return nil, nil, err
+	}
+
+	if len(rdmaDevices) == 0 {
+		logger.V(2).Info("No RDMA devices found for PCI address", "device", pciAddress)
+		return nil, nil, fmt.Errorf("no RDMA devices found for PCI address %s", pciAddress)
+	}
+
+	logger.V(2).Info("Device is RDMA capable, adding RDMA character devices",
+		"device", pciAddress, "rdmaDevices", rdmaDevices)
+
+	for _, rdmaDevice := range rdmaDevices {
+		// Get character devices for this RDMA device
+		charDevices, err := host.GetHelpers().GetRDMACharDevices(rdmaDevice)
+		if err != nil {
+			logger.Error(err, "Failed to get RDMA character devices, skipping",
+				"device", pciAddress, "rdmaDevice", rdmaDevice)
+			return nil, nil, err
+		}
+
+		if len(charDevices) == 0 {
+			logger.V(2).Info("No RDMA character devices found",
+				"device", pciAddress, "rdmaDevice", rdmaDevice)
+			return nil, nil, fmt.Errorf("no RDMA character devices found for RDMA device %s (PCI: %s)", rdmaDevice, pciAddress)
+		}
+
+		// Use RDMA device name in env var key to support multiple RDMA devices
+		rdmaDeviceSanitized := strings.ReplaceAll(rdmaDevice, "_", "")
+		devicePrefix := strings.ReplaceAll(deviceName, "-", "_")
+
+		// Add each character device to the CDI spec
+		for _, charDev := range charDevices {
+			deviceNodes = append(deviceNodes, &cdispec.DeviceNode{
+				Path:     charDev,
+				HostPath: charDev,
+				Type:     "c", // character device
+			})
+
+			// Add environment variable for each character device type
+			// Include RDMA device name to avoid collisions with multiple RDMA devices
+			switch {
+			case strings.Contains(charDev, "uverbs"):
+				envs = append(envs, fmt.Sprintf("SRIOVNETWORK_%s_%s_RDMA_UVERBS=%s", devicePrefix, rdmaDeviceSanitized, charDev))
+			case strings.Contains(charDev, "umad"):
+				envs = append(envs, fmt.Sprintf("SRIOVNETWORK_%s_%s_RDMA_UMAD=%s", devicePrefix, rdmaDeviceSanitized, charDev))
+			case strings.Contains(charDev, "issm"):
+				envs = append(envs, fmt.Sprintf("SRIOVNETWORK_%s_%s_RDMA_ISSM=%s", devicePrefix, rdmaDeviceSanitized, charDev))
+			case strings.Contains(charDev, "rdma_cm"):
+				envs = append(envs, fmt.Sprintf("SRIOVNETWORK_%s_%s_RDMA_CM=%s", devicePrefix, rdmaDeviceSanitized, charDev))
+			}
+		}
+
+		logger.Info("Added RDMA character devices for device",
+			"device", pciAddress, "rdmaDevice", rdmaDevice, "charDevices", charDevices, "envs", envs)
+
+		// Add RDMA device name to environment variables
+		envs = append(envs, fmt.Sprintf("SRIOVNETWORK_%s_%s_RDMA_DEVICE=%s",
+			devicePrefix, rdmaDeviceSanitized, rdmaDevice))
+	}
+
+	return deviceNodes, envs, nil
+}
+
 func (s *Manager) getNetAttachDefRawConfig(ctx context.Context, namespace string, netAttachDefName string) (string, error) {
 	// Get the net attach def information
 	netAttachDef := &netattdefv1.NetworkAttachmentDefinition{}

pkg/devicestate/state_test.go

@@ -2,13 +2,18 @@ package devicestate
 
 import (
 	"context"
+	"fmt"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"go.uber.org/mock/gomock"
+	"k8s.io/utils/ptr"
 
 	resourceapi "k8s.io/api/resource/v1"
 
 	"github.com/k8snetworkplumbingwg/dra-driver-sriov/pkg/consts"
+	"github.com/k8snetworkplumbingwg/dra-driver-sriov/pkg/host"
+	mock_host "github.com/k8snetworkplumbingwg/dra-driver-sriov/pkg/host/mock"
 )
 
 var _ = Describe("Manager", func() {
@@ -45,4 +50,212 @@ var _ = Describe("Manager", func() {
 			Expect(exists).To(BeFalse())
 		})
 	})
+
+	Context("RDMA Device Preparation", func() {
+		It("should skip RDMA preparation when device is not RDMA capable", func() {
+			// Create device without RDMA capability
+			nonRdmaDevice := &resourceapi.Device{
+				Name: "0000-08-00-1",
+				Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
+					consts.AttributePciAddress: {
+						StringValue: ptr.To("0000:08:00.1"),
+					},
+					consts.AttributeRDMACapable: {
+						BoolValue: ptr.To(false),
+					},
+				},
+			}
+
+			// Verify device is not RDMA capable
+			rdmaCapable, exists := nonRdmaDevice.Attributes[consts.AttributeRDMACapable]
+			Expect(exists).To(BeTrue())
+			Expect(rdmaCapable.BoolValue).ToNot(BeNil())
+			Expect(*rdmaCapable.BoolValue).To(BeFalse())
+
+			// Test the conditional logic that determines if RDMA preparation should occur
+			// This replicates the production code condition:
+			// if rdmaCapableAttr, ok := deviceInfo.Attributes[consts.AttributeRDMACapable]; ok && rdmaCapableAttr.BoolValue != nil && *rdmaCapableAttr.BoolValue
+			shouldPrepareRDMA := exists && rdmaCapable.BoolValue != nil && *rdmaCapable.BoolValue
+			Expect(shouldPrepareRDMA).To(BeFalse(), "RDMA preparation should be skipped for non-RDMA capable devices")
+
+			// When this condition is false, the production code never calls:
+			// - GetRDMADeviceForPCI
+			// - GetRDMACharDevices
+			// This test verifies the condition evaluates correctly for non-RDMA devices
+		})
+	})
+
+	Context("handleRDMADevice", func() {
+		var (
+			mockCtrl    *gomock.Controller
+			mockHost    *mock_host.MockInterface
+			origHelpers host.Interface
+			manager     *Manager
+		)
+
+		BeforeEach(func() {
+			mockCtrl = gomock.NewController(GinkgoT())
+			mockHost = mock_host.NewMockInterface(mockCtrl)
+			// Save original helpers and replace with mock
+			_ = host.GetHelpers()
+			origHelpers = host.Helpers
+			host.Helpers = mockHost
+
+			manager = &Manager{}
+		})
+
+		AfterEach(func() {
+			// Restore original helpers
+			host.Helpers = origHelpers
+			mockCtrl.Finish()
+		})
+
+		It("should return device nodes and environment variables for RDMA device", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+			rdmaDeviceName := "mlx5_0"
+
+			// Mock GetRDMADeviceForPCI to return one RDMA device
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return([]string{rdmaDeviceName}, nil)
+
+			// Mock GetRDMACharDevices to return various character devices
+			mockHost.EXPECT().GetRDMACharDevices(rdmaDeviceName).Return([]string{
+				"/dev/infiniband/uverbs0",
+				"/dev/infiniband/umad0",
+				"/dev/infiniband/issm0",
+				"/dev/infiniband/rdma_cm",
+			}, nil)
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify no error
+			Expect(err).ToNot(HaveOccurred())
+
+			// Verify device nodes
+			Expect(deviceNodes).To(HaveLen(4))
+			Expect(deviceNodes[0].Path).To(Equal("/dev/infiniband/uverbs0"))
+			Expect(deviceNodes[0].HostPath).To(Equal("/dev/infiniband/uverbs0"))
+			Expect(deviceNodes[0].Type).To(Equal("c"))
+			Expect(deviceNodes[1].Path).To(Equal("/dev/infiniband/umad0"))
+			Expect(deviceNodes[2].Path).To(Equal("/dev/infiniband/issm0"))
+			Expect(deviceNodes[3].Path).To(Equal("/dev/infiniband/rdma_cm"))
+
+			// Verify environment variables
+			Expect(envs).To(HaveLen(5))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_UVERBS=/dev/infiniband/uverbs0"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_UMAD=/dev/infiniband/umad0"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_ISSM=/dev/infiniband/issm0"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_CM=/dev/infiniband/rdma_cm"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_DEVICE=mlx5_0"))
+		})
+
+		It("should handle multiple RDMA devices", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+
+			// Mock GetRDMADeviceForPCI to return two RDMA devices
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return([]string{"mlx5_0", "mlx5_1"}, nil)
+
+			// Mock GetRDMACharDevices for first device
+			mockHost.EXPECT().GetRDMACharDevices("mlx5_0").Return([]string{"/dev/infiniband/uverbs0"}, nil)
+
+			// Mock GetRDMACharDevices for second device
+			mockHost.EXPECT().GetRDMACharDevices("mlx5_1").Return([]string{"/dev/infiniband/uverbs1"}, nil)
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify no error
+			Expect(err).ToNot(HaveOccurred())
+
+			// Verify device nodes for both RDMA devices
+			Expect(deviceNodes).To(HaveLen(2))
+			Expect(deviceNodes[0].Path).To(Equal("/dev/infiniband/uverbs0"))
+			Expect(deviceNodes[1].Path).To(Equal("/dev/infiniband/uverbs1"))
+
+			// Verify environment variables for both RDMA devices
+			Expect(envs).To(HaveLen(4))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_UVERBS=/dev/infiniband/uverbs0"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx50_RDMA_DEVICE=mlx5_0"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx51_RDMA_UVERBS=/dev/infiniband/uverbs1"))
+			Expect(envs).To(ContainElement("SRIOVNETWORK_device_1_mlx51_RDMA_DEVICE=mlx5_1"))
+		})
+
+		It("should return error when GetRDMADeviceForPCI fails", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+
+			// Mock GetRDMADeviceForPCI to return an error
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return(nil, fmt.Errorf("failed to get RDMA devices"))
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify error is returned
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("failed to get RDMA devices"))
+			Expect(deviceNodes).To(BeNil())
+			Expect(envs).To(BeNil())
+		})
+
+		It("should return error when no RDMA devices found", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+
+			// Mock GetRDMADeviceForPCI to return empty list
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return([]string{}, nil)
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify error is returned
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no RDMA devices found"))
+			Expect(deviceNodes).To(BeNil())
+			Expect(envs).To(BeNil())
+		})
+
+		It("should return error when GetRDMACharDevices fails", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+			rdmaDeviceName := "mlx5_0"
+
+			// Mock GetRDMADeviceForPCI to return one RDMA device
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return([]string{rdmaDeviceName}, nil)
+
+			// Mock GetRDMACharDevices to return an error
+			mockHost.EXPECT().GetRDMACharDevices(rdmaDeviceName).Return(nil, fmt.Errorf("failed to get char devices"))
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify error is returned
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("failed to get char devices"))
+			Expect(deviceNodes).To(BeNil())
+			Expect(envs).To(BeNil())
+		})
+
+		It("should return error when no character devices found", func() {
+			pciAddress := "0000:08:00.1"
+			deviceName := "device-1"
+			rdmaDeviceName := "mlx5_0"
+
+			// Mock GetRDMADeviceForPCI to return one RDMA device
+			mockHost.EXPECT().GetRDMADeviceForPCI(pciAddress).Return([]string{rdmaDeviceName}, nil)
+
+			// Mock GetRDMACharDevices to return empty list
+			mockHost.EXPECT().GetRDMACharDevices(rdmaDeviceName).Return([]string{}, nil)
+
+			// Call the function
+			deviceNodes, envs, err := manager.handleRDMADevice(context.Background(), pciAddress, deviceName)
+
+			// Verify error is returned
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no RDMA character devices found"))
+			Expect(deviceNodes).To(BeNil())
+			Expect(envs).To(BeNil())
+		})
+	})
 })

pkg/host/host.go

@@ -114,6 +114,7 @@ type Interface interface {
 	// RDMA device functions
 	GetRDMADeviceForPCI(pciAddr string) ([]string, error)
 	VerifyRDMACapability(pciAddr string) (bool, error)
+	GetRDMACharDevices(rdmaDeviceName string) ([]string, error)
 }
 
 // Host provides unified host system functionality for SR-IOV, PCI operations, and driver management
@@ -783,6 +784,21 @@ func (h *Host) EnsureVhostModulesLoaded() error {
 // GetRDMADeviceForPCI returns the RDMA device names associated with a PCI address
 // Uses the rdmamap library from Mellanox for RDMA device detection
 func (h *Host) GetRDMADeviceForPCI(pciAddr string) ([]string, error) {
+	// Validate input
+	if pciAddr == "" {
+		return nil, fmt.Errorf("pciAddr cannot be empty")
+	}
+	// Validate PCI address format (should be DDDD:BB:DD.F)
+	parts := strings.Split(pciAddr, ":")
+	if len(parts) != 3 {
+		return nil, fmt.Errorf("invalid PCI address format: %s (expected format: DDDD:BB:DD.F)", pciAddr)
+	}
+	// Validate the last part contains device.function format
+	deviceFunc := strings.Split(parts[2], ".")
+	if len(deviceFunc) != 2 {
+		return nil, fmt.Errorf("invalid PCI address format: %s (expected format: DDDD:BB:DD.F)", pciAddr)
+	}
+
 	h.log.V(2).Info("GetRDMADeviceForPCI(): getting RDMA devices for PCI address", "device", pciAddr)
 
 	// Use rdmaProvider to get RDMA devices for this PCI address
@@ -812,3 +828,27 @@ func (h *Host) VerifyRDMACapability(pciAddr string) (bool, error) {
 
 	return hasRDMA, nil
 }
+
+// GetRDMACharDevices returns the character device paths for an RDMA device
+// These are the actual device nodes (e.g., /dev/infiniband/uverbs0) that need to be
+// exposed to containers for RDMA functionality
+func (h *Host) GetRDMACharDevices(rdmaDeviceName string) ([]string, error) {
+	// Validate input
+	if rdmaDeviceName == "" {
+		return nil, fmt.Errorf("rdmaDeviceName cannot be empty")
+	}
+
+	h.log.Info("GetRDMACharDevices(): getting character devices for RDMA device", "rdmaDevice", rdmaDeviceName)
+
+	// Use rdmaProvider to get character devices for this RDMA device
+	charDevices := h.rdmaProvider.GetRdmaCharDevices(rdmaDeviceName)
+
+	if len(charDevices) == 0 {
+		h.log.Info("GetRDMACharDevices(): no character devices found", "rdmaDevice", rdmaDeviceName)
+		return nil, nil
+	}
+
+	h.log.Info("GetRDMACharDevices(): found character devices",
+		"rdmaDevice", rdmaDeviceName, "charDevices", charDevices)
+	return charDevices, nil
+}