Feat: Allow to configure kubelet root directory

Signed-off-by: Aleksandr Zhitnikov <aleksandr.zhitnikov@gcore.com>
Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>
Co-authored-by: Tariq <tibrahim@nvidia.com>

Author: apten-fors

api/nvidia/v1/clusterpolicy_types.go

@@ -174,6 +174,11 @@ type HostPathsSpec struct {
 	// DriverInstallDir represents the root at which driver files including libraries,
 	// config files, and executables can be found.
 	DriverInstallDir string `json:"driverInstallDir,omitempty"`
+
+	// KubeletRootDir represents the location of the kubelet root directory.
+	// If empty, it will default to "/var/lib/kubelet".
+	// +kubebuilder:default="/var/lib/kubelet"
+	KubeletRootDir string `json:"kubeletRootDir,omitempty"`
 }
 
 // EnvVar represents an environment variable present in a Container.

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -1504,6 +1504,12 @@ spec:
                       DriverInstallDir represents the root at which driver files including libraries,
                       config files, and executables can be found.
                     type: string
+                  kubeletRootDir:
+                    default: /var/lib/kubelet
+                    description: |-
+                      KubeletRootDir represents the location of the kubelet root directory.
+                      If empty, it will default to "/var/lib/kubelet".
+                    type: string
                   rootFS:
                     description: |-
                       RootFS represents the path to the root filesystem of the host.

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -1504,6 +1504,12 @@ spec:
                       DriverInstallDir represents the root at which driver files including libraries,
                       config files, and executables can be found.
                     type: string
+                  kubeletRootDir:
+                    default: /var/lib/kubelet
+                    description: |-
+                      KubeletRootDir represents the location of the kubelet root directory.
+                      If empty, it will default to "/var/lib/kubelet".
+                    type: string
                   rootFS:
                     description: |-
                       RootFS represents the path to the root filesystem of the host.

controllers/object_controls.go

@@ -181,6 +181,8 @@ const (
 	HostRootEnvName = "HOST_ROOT"
 	// DefaultDriverInstallDir represents the default path of a driver container installation
 	DefaultDriverInstallDir = "/run/nvidia/driver"
+	// DefaultKubeletRootDir represents the default path of a kubelet root directory
+	DefaultKubeletRootDir = "/var/lib/kubelet"
 	// DriverInstallDirEnvName is the name of the envvar used by the driver-validator to represent the driver install dir
 	DriverInstallDirEnvName = "DRIVER_INSTALL_DIR"
 	// DriverInstallDirCtrPathEnvName is the name of the envvar used by the driver-validator to represent the path
@@ -1850,6 +1852,18 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_COLLECTORS", MetricsConfigMountPath)
 	}
 
+	const podResourcesVolume = "pod-gpu-resources"
+	kubeletRootDir := config.HostPaths.KubeletRootDir
+	if len(kubeletRootDir) > 0 && kubeletRootDir != DefaultKubeletRootDir {
+		for i := range obj.Spec.Template.Spec.Volumes {
+			volume := &obj.Spec.Template.Spec.Volumes[i]
+			if volume.Name == podResourcesVolume {
+				volume.HostPath.Path = filepath.Join(kubeletRootDir, "pod-resources")
+				break
+			}
+		}
+	}
+
 	for _, env := range config.DCGMExporter.Env {
 		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), env.Name, env.Value)
 	}

controllers/transforms_test.go

@@ -1248,7 +1248,8 @@ func TestTransformDCGMExporter(t *testing.T) {
 			description: "transform dcgm exporter",
 			ds: NewDaemonset().
 				WithContainer(corev1.Container{Name: "dcgm-exporter"}).
-				WithContainer(corev1.Container{Name: "dummy"}),
+				WithContainer(corev1.Container{Name: "dummy"}).
+				WithHostPathVolume("pod-resources", "/var/lib/kubelet/pod-resources", nil),
 			cpSpec: &gpuv1.ClusterPolicySpec{
 				DCGMExporter: gpuv1.DCGMExporterSpec{
 					Repository:       "nvcr.io/nvidia/cloud-native",
@@ -1274,7 +1275,9 @@ func TestTransformDCGMExporter(t *testing.T) {
 					{Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"},
 					{Name: "foo", Value: "bar"},
 				},
-			}).WithContainer(corev1.Container{Name: "dummy"}).WithPullSecret("pull-secret").WithRuntimeClassName("nvidia"),
+			}).WithContainer(corev1.Container{Name: "dummy"}).WithPullSecret("pull-secret").
+				WithRuntimeClassName("nvidia").
+				WithHostPathVolume("pod-resources", "/var/lib/kubelet/pod-resources", nil),
 		},
 		{
 			description: "transform dcgm exporter with hostPID enabled",
@@ -1607,6 +1610,32 @@ func TestTransformDCGMExporter(t *testing.T) {
 				}).
 				WithRuntimeClassName("nvidia"),
 		},
+		{
+			description: "transform dcgm exporter with custom kubelet root",
+			ds: NewDaemonset().
+				WithContainer(corev1.Container{Name: "dcgm-exporter"}).
+				WithHostPathVolume("pod-gpu-resources", "/var/lib/kubelet/pod-resources", nil),
+			cpSpec: &gpuv1.ClusterPolicySpec{
+				HostPaths: gpuv1.HostPathsSpec{
+					KubeletRootDir: "/custom-kubelet",
+				},
+				DCGMExporter: gpuv1.DCGMExporterSpec{
+					Repository:      "nvcr.io/nvidia/cloud-native",
+					Image:           "dcgm-exporter",
+					Version:         "v1.0.0",
+					ImagePullPolicy: "IfNotPresent",
+				},
+			},
+			expectedDs: NewDaemonset().WithContainer(corev1.Container{
+				Name:            "dcgm-exporter",
+				Image:           "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0",
+				ImagePullPolicy: corev1.PullIfNotPresent,
+				Env: []corev1.EnvVar{
+					{Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"},
+				},
+			}).WithRuntimeClassName("nvidia").
+				WithHostPathVolume("pod-gpu-resources", "/custom-kubelet/pod-resources", nil),
+		},
 	}
 
 	for _, tc := range testCases {

deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml

@@ -1504,6 +1504,12 @@ spec:
                       DriverInstallDir represents the root at which driver files including libraries,
                       config files, and executables can be found.
                     type: string
+                  kubeletRootDir:
+                    default: /var/lib/kubelet
+                    description: |-
+                      KubeletRootDir represents the location of the kubelet root directory.
+                      If empty, it will default to "/var/lib/kubelet".
+                    type: string
                   rootFS:
                     description: |-
                       RootFS represents the path to the root filesystem of the host.

deployments/gpu-operator/templates/clusterpolicy.yaml

@@ -15,6 +15,9 @@ spec:
   hostPaths:
     rootFS: {{ .Values.hostPaths.rootFS }}
     driverInstallDir: {{ .Values.hostPaths.driverInstallDir }}
+    {{- if .Values.hostPaths.kubeletRootDir }}
+    kubeletRootDir: {{ .Values.hostPaths.kubeletRootDir }}
+    {{- end }}
   operator:
     {{- if .Values.operator.runtimeClass }}
     runtimeClass: {{ .Values.operator.runtimeClass }}

deployments/gpu-operator/values.yaml

@@ -34,6 +34,10 @@ hostPaths:
   # config files, and executables can be found.
   driverInstallDir: "/run/nvidia/driver"
 
+  # kubeletRootDir represents the root with the kubelet base directory
+  # if empty will use /var/lib/kubelet as the default path
+  # kubeletRootDir: ""
+
 daemonsets:
   labels: {}
   annotations: {}