Add support for drop-in configs

This change adds explicit support for drop-in configs as supported
by containerd and cri-o.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

controllers/object_controls.go

@@ -1327,12 +1327,18 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
 		setContainerEnv(mainContainer, "CONTAINERD_RUNTIME_CLASS", getRuntimeClass(config))
 	}
 
+	// For runtime config files we have top-level configs and drop-in files.
+	// These are supported as follows:
+	//   * Docker only supports top-level config files.
+	//   * Containerd supports drop-in files, but required modification to the top-level config
+	//   * Crio supports drop-in files at a predefined location. The top-level config may be read
+	//     but should not be updated.
+
 	// setup mounts for runtime config file
-	runtimeConfigFile, err := getRuntimeConfigFile(mainContainer, runtime)
+	runtimeConfigFiles, err := getRuntimeConfigFiles(mainContainer, runtime)
 	if err != nil {
 		return fmt.Errorf("error getting path to runtime config file: %v", err)
 	}
-	sourceConfigFileName := path.Base(runtimeConfigFile)
 
 	var configEnvvarName string
 	switch runtime {
@@ -1344,15 +1350,38 @@ func transformForRuntime(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,
 		configEnvvarName = "CRIO_CONFIG"
 	}
 
-	setContainerEnv(mainContainer, "RUNTIME_CONFIG", DefaultRuntimeConfigTargetDir+sourceConfigFileName)
-	setContainerEnv(mainContainer, configEnvvarName, DefaultRuntimeConfigTargetDir+sourceConfigFileName)
+	// Handle the top-level configs
+	if runtimeConfigFiles.topLevelConfigFile != "" {
+		sourceConfigFileName := path.Base(runtimeConfigFiles.topLevelConfigFile)
+		sourceConfigDir := path.Dir(runtimeConfigFiles.topLevelConfigFile)
+		containerConfigDir := DefaultRuntimeConfigTargetDir
+		setContainerEnv(mainContainer, "RUNTIME_CONFIG", containerConfigDir+sourceConfigFileName)
+		setContainerEnv(mainContainer, configEnvvarName, containerConfigDir+sourceConfigFileName)
+
+		volMountConfigName := fmt.Sprintf("%s-config", runtime)
+		volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: containerConfigDir}
+		mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
+
+		configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: sourceConfigDir, Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
+		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
+	}
+
+	// Handle the drop-in configs
+	if runtimeConfigFiles.dropInConfigFile != "" {
+		sourceConfigFileName := path.Base(runtimeConfigFiles.dropInConfigFile)
+		sourceConfigDir := path.Dir(runtimeConfigFiles.dropInConfigFile)
+		// TODO: This should be a constant.
+		containerConfigDir := "/runtime/config-dir.d/"
+		setContainerEnv(mainContainer, "RUNTIME_DROP_IN_CONFIG", containerConfigDir+sourceConfigFileName)
+		setContainerEnv(mainContainer, "RUNTIME_DROP_IN_CONFIG_HOST_PATH", runtimeConfigFiles.dropInConfigFile)
 
-	volMountConfigName := fmt.Sprintf("%s-config", runtime)
-	volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: DefaultRuntimeConfigTargetDir}
-	mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
+		volMountConfigName := fmt.Sprintf("%s-drop-in-config", runtime)
+		volMountConfig := corev1.VolumeMount{Name: volMountConfigName, MountPath: containerConfigDir}
+		mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMountConfig)
 
-	configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Dir(runtimeConfigFile), Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
-	obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
+		configVol := corev1.Volume{Name: volMountConfigName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: sourceConfigDir, Type: newHostPathType(corev1.HostPathDirectoryOrCreate)}}}
+		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, configVol)
+	}
 
 	// setup mounts for runtime socket file
 	runtimeSocketFile, err := getRuntimeSocketFile(mainContainer, runtime)
@@ -2357,30 +2386,62 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol
 	return nil
 }
 
+type runtimeConfigFiles struct {
+	// TODO: Add envvarName as a member here so that we don't have to query this
+	// again.
+	// envvarName         string
+	topLevelConfigFile string
+	dropInConfigFile   string
+}
+
 // get runtime(docker, containerd) config file path based on toolkit container env or default
-func getRuntimeConfigFile(c *corev1.Container, runtime string) (string, error) {
-	var runtimeConfigFile string
+func getRuntimeConfigFiles(c *corev1.Container, runtime string) (runtimeConfigFiles, error) {
 	switch runtime {
 	case gpuv1.Docker.String():
-		runtimeConfigFile = DefaultDockerConfigFile
+		topLevelConfigFile := DefaultDockerConfigFile
 		if value := getContainerEnv(c, "DOCKER_CONFIG"); value != "" {
-			runtimeConfigFile = value
+			topLevelConfigFile = value
 		}
+		return runtimeConfigFiles{
+			topLevelConfigFile: topLevelConfigFile,
+			// Docker does not support drop-in files.
+			dropInConfigFile: "",
+		}, nil
 	case gpuv1.Containerd.String():
-		runtimeConfigFile = DefaultContainerdConfigFile
+		topLevelConfigFile := DefaultContainerdConfigFile
+		// TODO: We should also read RUNTIME_CONFIG here
 		if value := getContainerEnv(c, "CONTAINERD_CONFIG"); value != "" {
-			runtimeConfigFile = value
+			topLevelConfigFile = value
+		}
+		// TODO: This should be a sane default.
+		dropInConfigFile := "/run/toolkit/config/99-nvidia.toml"
+		if value := getContainerEnv(c, "RUNTIME_DROP_IN_CONFIG"); value != "" {
+			dropInConfigFile = value
 		}
+		return runtimeConfigFiles{
+			topLevelConfigFile: topLevelConfigFile,
+			dropInConfigFile:   dropInConfigFile,
+		}, nil
 	case gpuv1.CRIO.String():
-		runtimeConfigFile = DefaultCRIOConfigFile
+		// TODO: We should still allow the top-level config to be specified
+		// TODO: This should be DefaultCRIOConfigFile, but this is not the correct value.
+		// TODO: We should also read RUNTIME_CONFIG here
+		topLevelConfigFile := "/etc/crio/config.toml"
 		if value := getContainerEnv(c, "CRIO_CONFIG"); value != "" {
-			runtimeConfigFile = value
+			topLevelConfigFile = value
 		}
+		// TODO: The constant should have a different name (e.g. DefaultCRIODropInFile)
+		dropInConfigFile := DefaultCRIOConfigFile
+		if value := getContainerEnv(c, "RUNTIME_DROP_IN_CONFIG"); value != "" {
+			dropInConfigFile = value
+		}
+		return runtimeConfigFiles{
+			topLevelConfigFile: topLevelConfigFile,
+			dropInConfigFile:   dropInConfigFile,
+		}, nil
 	default:
-		return "", fmt.Errorf("invalid runtime: %s", runtime)
+		return runtimeConfigFiles{}, fmt.Errorf("invalid runtime: %s", runtime)
 	}
-
-	return runtimeConfigFile, nil
 }
 
 // get runtime(docker, containerd) socket file path based on toolkit container env or default