Add config option for ContainerConfigRoot

This change allows the container compat root for nvgpu (e.g. Orin) systems
to be specified either as the

nvidia-container-runtime.modes.csv.container-compat-root option in the config.toml
file, or with the --csv.container-compat-root (NVIDIA_CTK_CDI_GENERATE_CSV_CONTAINER_COMPAT_ROOT)
option when generating CDI specifications.

A WithCSVContainerCompatRoot option is also exposed in the nvcdi API.

Note that this option is only relevant when nvgpu devices are detected.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -70,8 +70,9 @@ type options struct {
 	featureFlags []string
 
 	csv struct {
-		files          []string
-		ignorePatterns []string
+		files               []string
+		ignorePatterns      []string
+		containerCompatRoot string
 	}
 
 	noAllDevice bool
@@ -212,6 +213,12 @@ func (m command) build() *cli.Command {
 				Destination: &opts.csv.ignorePatterns,
 				Sources:     cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_CSV_IGNORE_PATTERNS"),
 			},
+			&cli.StringFlag{
+				Name:        "csv.container-compat-root",
+				Usage:       "specify the container folder to use for CUDA Forward Compatibility in non-standard containers",
+				Destination: &opts.csv.containerCompatRoot,
+				Sources:     cli.EnvVars("NVIDIA_CTK_CDI_GENERATE_CSV_CONTAINER_COMPAT_ROOT"),
+			},
 			&cli.StringSliceFlag{
 				Name:    "disable-hook",
 				Aliases: []string{"disable-hooks"},
@@ -384,6 +391,7 @@ func (m command) generateSpecs(opts *options) ([]generatedSpecs, error) {
 		nvcdi.WithLibrarySearchPaths(opts.librarySearchPaths),
 		nvcdi.WithCSVFiles(opts.csv.files),
 		nvcdi.WithCSVIgnorePatterns(opts.csv.ignorePatterns),
+		nvcdi.WithCSVContainerCompatRoot(opts.csv.containerCompatRoot),
 		nvcdi.WithDisabledHooks(opts.disabledHooks...),
 		nvcdi.WithEnabledHooks(opts.enabledHooks...),
 		nvcdi.WithFeatureFlags(opts.featureFlags...),

internal/config/runtime.go

@@ -53,6 +53,9 @@ type jitCDIModeConfig struct {
 
 type csvModeConfig struct {
 	MountSpecPath string `toml:"mount-spec-path"`
+	// ContainerCompatRoot specifies the compat root used when the the standard
+	// CUDA compat libraries should not be used.
+	ContainerCompatRoot string `toml:"container-compat-root"`
 }
 
 type legacyModeConfig struct {

internal/modifier/csv.go

@@ -59,6 +59,7 @@ func NewCSVModifier(logger logger.Interface, cfg *config.Config, container image
 		nvcdi.WithNVIDIACDIHookPath(cfg.NVIDIACTKConfig.Path),
 		nvcdi.WithMode(nvcdi.ModeCSV),
 		nvcdi.WithCSVFiles(csvFiles),
+		nvcdi.WithCSVContainerCompatRoot(cfg.NVIDIAContainerRuntimeConfig.Modes.CSV.ContainerCompatRoot),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct CDI library: %v", err)

pkg/nvcdi/lib-csv.go

@@ -37,8 +37,9 @@ import (
 )
 
 type csvOptions struct {
-	Files          []string
-	IgnorePatterns []string
+	Files               []string
+	IgnorePatterns      []string
+	ContainerCompatRoot string
 }
 
 type csvlib nvcdilib
@@ -52,6 +53,9 @@ func (l *nvcdilib) asCSVLib() *csvlib {
 	if len(l.csv.Files) == 0 {
 		l.csv.Files = csv.DefaultFileList()
 	}
+	if l.csv.ContainerCompatRoot == "" {
+		l.csv.ContainerCompatRoot = "/usr/local/cuda/compat-orin"
+	}
 	return (*csvlib)(l)
 }
 
@@ -481,7 +485,7 @@ func (l *csvlib) cudaCompatDiscoverer() discover.Discover {
 		// TODO: Should this be overridable through a feature flag / config option?
 		if strings.Contains(name, "Orin (nvgpu)") {
 			// TODO: This should probably be a constant or configurable.
-			cudaCompatContainerRoot = "/usr/local/cuda/compat-orin"
+			cudaCompatContainerRoot = l.csv.ContainerCompatRoot
 			break
 		}
 	}

pkg/nvcdi/options.go

@@ -142,6 +142,14 @@ func WithCSVIgnorePatterns(csvIgnorePatterns []string) Option {
 	}
 }
 
+// WithCSVContainerCompatRoot sets the compat root to use for the container in
+// the case of nvgpu-only devices.
+func WithCSVContainerCompatRoot(csvContainerCompatRoot string) Option {
+	return func(o *nvcdilib) {
+		o.csv.ContainerCompatRoot = csvContainerCompatRoot
+	}
+}
+
 // WithConfigSearchPaths sets the search paths for config files.
 func WithConfigSearchPaths(paths []string) Option {
 	return func(o *nvcdilib) {