[nvpassthrough] load kmods without needing the host's rootfs or chroot

Prior to this change, BindToVFIODriver() loaded the vfio-pci kernel
module by running "chroot $hostRoot modprobe vfio_pci". This commit
moves to using github.com/pmorjan/kmod instead, which only depends
on having read access to the host's /lib/modules directory.

Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>

Author: cdesiniotis

go.mod

@@ -7,7 +7,10 @@ toolchain go1.24.12
 require (
 	github.com/NVIDIA/go-nvml v0.13.0-1
 	github.com/google/uuid v1.6.0
+	github.com/klauspost/compress v1.18.6
+	github.com/pmorjan/kmod v1.1.1
 	github.com/stretchr/testify v1.11.1
+	github.com/ulikunitz/xz v0.5.15
 	golang.org/x/sys v0.40.0
 )
 

go.sum

@@ -4,10 +4,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmorjan/kmod v1.1.1 h1:Vfw6bMaOg/sYSBCqJPT9TbqHHf5zK00GbaL5JQLO4r0=
+github.com/pmorjan/kmod v1.1.1/go.mod h1:jR4fVosEpQ6b5U0rpxaqoShTDPvCjLIP8vEESZyvnqQ=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
+github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
 golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/nvpassthrough/kmod.go

@@ -17,21 +17,70 @@
 package nvpassthrough
 
 import (
-	"os/exec"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
+	"golang.org/x/sys/unix"
 )
 
-type kernelModules struct {
-	root string
-}
+// modInitFunc supports uncompressed files and gzip and xz compressed files.
+//
+// This code is taken from:
+//
+//	https://github.com/pmorjan/kmod/blob/d0ca1d5ed38616f2dc65c69add06b55f7cc091a7/cmd/modprobe/modprobe.go#L132
+func modInitFunc(path, params string, flags int) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	switch filepath.Ext(path) {
+	case ".gz":
+		rd, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		defer rd.Close()
+		return initModule(rd, params)
+	case ".xz":
+		rd, err := xz.NewReader(f)
+		if err != nil {
+			return err
+		}
+		return initModule(rd, params)
+	case ".zst":
+		rd, err := zstd.NewReader(f)
+		if err != nil {
+			return err
+		}
+		defer rd.Close()
+		return initModule(rd, params)
+	}
 
-func newKernelModules(root string) *kernelModules {
-	km := &kernelModules{
-		root: root,
+	// uncompressed file, first try finitModule then initModule
+	if err := finitModule(int(f.Fd()), params); err != nil {
+		if err == unix.ENOSYS {
+			return initModule(f, params)
+		}
 	}
-	return km
+	return nil
+}
+
+// finitModule inserts a module file via syscall finit_module(2)
+func finitModule(fd int, params string) error {
+	return unix.FinitModule(fd, params, 0)
 }
 
-func (km *kernelModules) load(module string) error {
-	cmd := exec.Command("chroot", km.root, "modprobe", module)
-	return cmd.Run()
+// initModule inserts a module via syscall init_module(2)
+func initModule(rd io.Reader, params string) error {
+	buf, err := io.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+	return unix.InitModule(buf, params)
 }

pkg/nvpassthrough/modalias.go

@@ -132,13 +132,13 @@ func getKernelVersion() (string, error) {
 	return string(release), nil
 }
 
-func getVFIOAliases() ([]vfioAlias, error) {
+func (n *nvpassthrough) getVFIOAliases() ([]vfioAlias, error) {
 	kernelVersion, err := getKernelVersion()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get kernel version: %w", err)
 	}
 
-	modulesAliasFilePath := filepath.Join(libModulesRoot, kernelVersion, "modules.alias")
+	modulesAliasFilePath := filepath.Join(n.libModulesRoot, kernelVersion, "modules.alias")
 	modulesAliasContent, err := os.ReadFile(modulesAliasFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file %s: %w", modulesAliasFilePath, err)

pkg/nvpassthrough/nvpassthrough.go

@@ -22,6 +22,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/pmorjan/kmod"
+
 	"github.com/NVIDIA/go-nvlib/pkg/nvpci"
 )
 
@@ -42,9 +44,9 @@ type Interface interface {
 }
 
 type nvpassthrough struct {
-	logger   basicLogger
-	hostRoot string
-	nvpciLib nvpci.Interface
+	logger         basicLogger
+	libModulesRoot string
+	nvpciLib       nvpci.Interface
 }
 
 var _ Interface = (*nvpassthrough)(nil)
@@ -63,8 +65,8 @@ func New(opts ...Option) Interface {
 	if n.logger == nil {
 		n.logger = &nullLogger{}
 	}
-	if n.hostRoot == "" {
-		n.hostRoot = "/"
+	if n.libModulesRoot == "" {
+		n.libModulesRoot = libModulesRoot
 	}
 	if n.nvpciLib == nil {
 		n.nvpciLib = nvpci.New()
@@ -83,10 +85,10 @@ func WithLogger(logger basicLogger) Option {
 	}
 }
 
-// WithHostRoot provides an Option to set the path to the host root filesystem.
-func WithHostRoot(hostRoot string) Option {
+// WithLibModulesRoot provides an Option to set the path to the modules root.
+func WithLibModulesRoot(libModulesRoot string) Option {
 	return func(w *nvpassthrough) {
-		w.hostRoot = hostRoot
+		w.libModulesRoot = libModulesRoot
 	}
 }
 
@@ -116,7 +118,7 @@ func (n *nvpassthrough) FindBestVFIOVariant(address string) (string, error) {
 		return "", fmt.Errorf("device at %q is not an NVIDIA PCI device", address)
 	}
 
-	vfioAliases, err := getVFIOAliases()
+	vfioAliases, err := n.getVFIOAliases()
 	if err != nil {
 		return "", fmt.Errorf("failed to get vfio_pci aliases in modules.alias file: %w", err)
 	}
@@ -166,8 +168,14 @@ func (n *nvpassthrough) BindToVFIODriver(address string) error {
 		return fmt.Errorf("failed to find best vfio variant driver: %w", err)
 	}
 
-	km := newKernelModules(n.hostRoot)
-	if err := km.load(vfioDriverName); err != nil {
+	k, err := kmod.New(
+		kmod.SetInitFunc(modInitFunc),
+		kmod.SetRootDir(n.libModulesRoot),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to initialize kmod library: %w", err)
+	}
+	if err := k.Load(vfioDriverName, "", 0); err != nil {
 		return fmt.Errorf("failed to load %q driver: %w", vfioDriverName, err)
 	}