nvmeof: fix CSI node plugin crash on immutable Linux distributions

On distributions like Talos Linux,
NVMe modules are compiled directly into the kernel
(CONFIG_NVME_TCP=y, CONFIG_NVME_FABRICS=y) instead of
being loadable .ko files.
Explicitly calling modprobe on nvme_fabrics fails on
these systems since there is no .ko file present,
causing the CSI node plugin to crash on startup.

Removing nvme_fabrics from the module load
list is safe because it is always a dependency
of nvme_tcp.
On normal distributions modprobe loads it automatically
as part of the nvme_tcp dependency chain. On immutable
distributions it is already baked into the kernel.

We verify the fabrics framework is functional
after loading nvme_tcp by checking that
/dev/nvme-fabrics exists.
This device node is created by the kernel on init
regardless of whether nvme_fabrics was loaded as a
module or compiled in, making it a
reliable indicator that NVMe-oF TCP is ready to use.

Signed-off-by: gadi-didi <gadi.didi@ibm.com>
(cherry picked from commit 992d720)

Author: gadididi

internal/nvmeof/nvmeof_initiator.go

@@ -125,20 +125,27 @@ func NewNVMeInitiator() NVMeInitiator {
 
 // LoadKernelModules ensures required kernel modules are loaded.
 func (ni *nvmeInitiator) LoadKernelModules(ctx context.Context) error {
-	modules := []string{
-		"nvme_tcp",
-		"nvme_fabrics",
-	}
-	log.DebugLog(ctx, "Loading NVMe-oF kernel modules: %s, and %s", modules[0], modules[1])
+	module := "nvme_tcp"
+	log.DebugLog(ctx, "Loading NVMe-oF kernel module: %s", module)
 
-	for _, module := range modules {
-		err := kmod.Modprobe(ctx, module)
-		if err != nil {
-			return fmt.Errorf("failed to load kernel module %q: %w", module, err)
-		}
+	err := kmod.Modprobe(ctx, module)
+	if err != nil {
+		return fmt.Errorf("failed to load kernel module %q: %w", module, err)
 	}
+	log.DebugLog(ctx, "NVMe-oF kernel module: %s, is loaded successfully", module)
+	// verify nvme_fabrics is functional by checking its device node.
+	// On immutable Operating Systems like Talos Linux (CONFIG_NVME_FABRICS=y), the module
+	// is baked into the kernel and /sys/module/nvme_fabrics may not exist,
+	// but /dev/nvme-fabrics is always created by the kernel on init if the
+	// fabrics framework is operational.
+	if _, err := os.Stat("/dev/nvme-fabrics"); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("nvme_fabrics is not functional, /dev/nvme-fabrics not found: %w", err)
+		}
 
-	log.DebugLog(ctx, "All NVMe-oF kernel modules: %s, and %s, loaded successfully", modules[0], modules[1])
+		return fmt.Errorf("nvme_fabrics is not functional, failed to stat /dev/nvme-fabrics: %w", err)
+	}
+	log.DebugLog(ctx, "NVMe-oF fabrics framework is functional with /dev/nvme-fabrics present")
 
 	return nil
 }