Skip to content

Commit b698050

Browse files
apten-forstariq1890
apten-fors
and
tariq1890
committed
Feat: Allow to configure kubelet root directory
Signed-off-by: Aleksandr Zhitnikov <aleksandr.zhitnikov@gcore.com> Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com> Co-authored-by: Tariq <tibrahim@nvidia.com>
1 parent ad9b4a3 commit b698050

8 files changed

Lines changed: 161 additions & 0 deletions

File tree

‎api/nvidia/v1/clusterpolicy_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,11 @@ type HostPathsSpec struct {
174174
// DriverInstallDir represents the root at which driver files including libraries,
175175
// config files, and executables can be found.
176176
DriverInstallDir string `json:"driverInstallDir,omitempty"`
177+
178+
// KubeletRootDir represents the location of the kubelet root directory.
179+
// If empty, it will default to "/var/lib/kubelet".
180+
// +kubebuilder:default="/var/lib/kubelet"
181+
KubeletRootDir string `json:"kubeletRootDir,omitempty"`
177182
}
178183

179184
// EnvVar represents an environment variable present in a Container.

‎bundle/manifests/nvidia.com_clusterpolicies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,12 @@ spec:
14961496
DriverInstallDir represents the root at which driver files including libraries,
14971497
config files, and executables can be found.
14981498
type: string
1499+
kubeletRootDir:
1500+
default: /var/lib/kubelet
1501+
description: |-
1502+
KubeletRootDir represents the location of the kubelet root directory.
1503+
If empty, it will default to "/var/lib/kubelet".
1504+
type: string
14991505
rootFS:
15001506
description: |-
15011507
RootFS represents the path to the root filesystem of the host.

‎config/crd/bases/nvidia.com_clusterpolicies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,12 @@ spec:
14961496
DriverInstallDir represents the root at which driver files including libraries,
14971497
config files, and executables can be found.
14981498
type: string
1499+
kubeletRootDir:
1500+
default: /var/lib/kubelet
1501+
description: |-
1502+
KubeletRootDir represents the location of the kubelet root directory.
1503+
If empty, it will default to "/var/lib/kubelet".
1504+
type: string
14991505
rootFS:
15001506
description: |-
15011507
RootFS represents the path to the root filesystem of the host.

‎controllers/object_controls.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,8 @@ const (
181181
HostRootEnvName = "HOST_ROOT"
182182
// DefaultDriverInstallDir represents the default path of a driver container installation
183183
DefaultDriverInstallDir = "/run/nvidia/driver"
184+
// DefaultKubeletRootDir represents the default path of a kubelet root directory
185+
DefaultKubeletRootDir = "/var/lib/kubelet"
184186
// DriverInstallDirEnvName is the name of the envvar used by the driver-validator to represent the driver install dir
185187
DriverInstallDirEnvName = "DRIVER_INSTALL_DIR"
186188
// DriverInstallDirCtrPathEnvName is the name of the envvar used by the driver-validator to represent the path
@@ -735,6 +737,9 @@ func preProcessDaemonSet(obj *appsv1.DaemonSet, n ClusterPolicyController) error
735737
// transform the driver-root volume if a custom driver install dir is configured with the operator
736738
transformForDriverInstallDir(obj, n.singleton.Spec.HostPaths.DriverInstallDir)
737739

740+
// transform the kubelet root dir if a custom kubelet root dir is configured with the operator
741+
transformForKubeletRootDir(obj, n.singleton.Spec.HostPaths.KubeletRootDir)
742+
738743
// apply per operand Daemonset config
739744
err = t(obj, &n.singleton.Spec, n)
740745
if err != nil {
@@ -874,6 +879,23 @@ func transformForDriverInstallDir(obj *appsv1.DaemonSet, driverInstallDir string
874879
}
875880
}
876881

882+
// apply necessary transforms if a custom kubelet root directory is configured
883+
func transformForKubeletRootDir(obj *appsv1.DaemonSet, kubeletRootDir string) {
884+
if kubeletRootDir == "" || kubeletRootDir == DefaultKubeletRootDir {
885+
return
886+
}
887+
888+
for i := range obj.Spec.Template.Spec.Volumes {
889+
volume := &obj.Spec.Template.Spec.Volumes[i]
890+
switch volume.Name {
891+
case "pod-gpu-resources":
892+
volume.HostPath.Path = filepath.Join(kubeletRootDir, "pod-resources")
893+
case "device-plugin":
894+
volume.HostPath.Path = filepath.Join(kubeletRootDir, "device-plugins")
895+
}
896+
}
897+
}
898+
877899
// TransformGPUDiscoveryPlugin transforms GPU discovery daemonset with required config as per ClusterPolicy
878900
func TransformGPUDiscoveryPlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
879901
// update validation container

‎controllers/transforms_test.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,115 @@ func TestTransformForHostRoot(t *testing.T) {
300300
}
301301
}
302302

303+
func TestTransformForKubeletRootDir(t *testing.T) {
304+
podGPUResourcesVolumeName := "pod-gpu-resources"
305+
devicePluginVolumeName := "device-plugin"
306+
testCases := []struct {
307+
description string
308+
kubeletRootDir string
309+
input Daemonset
310+
expectedOutput Daemonset
311+
}{
312+
{
313+
description: "no pod-gpu-resources volume in daemonset",
314+
kubeletRootDir: "/custom-kubelet",
315+
input: NewDaemonset(),
316+
expectedOutput: NewDaemonset(),
317+
},
318+
{
319+
description: "empty kubeletRootDir is a no-op",
320+
kubeletRootDir: "",
321+
input: NewDaemonset().
322+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil),
323+
expectedOutput: NewDaemonset().
324+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil),
325+
},
326+
{
327+
description: "default kubeletRootDir is a no-op",
328+
kubeletRootDir: DefaultKubeletRootDir,
329+
input: NewDaemonset().
330+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil),
331+
expectedOutput: NewDaemonset().
332+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil),
333+
},
334+
{
335+
description: "custom kubeletRootDir with pod-gpu-resources volume only",
336+
kubeletRootDir: "/custom-kubelet",
337+
input: NewDaemonset().
338+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil),
339+
expectedOutput: NewDaemonset().
340+
WithHostPathVolume(podGPUResourcesVolumeName, "/custom-kubelet/pod-resources", nil),
341+
},
342+
{
343+
description: "custom kubeletRootDir with pod-gpu-resources volume and container mount",
344+
kubeletRootDir: "/custom-kubelet",
345+
input: NewDaemonset().
346+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil).
347+
WithContainer(corev1.Container{
348+
Name: "test-container",
349+
VolumeMounts: []corev1.VolumeMount{
350+
{Name: podGPUResourcesVolumeName, MountPath: "/var/lib/kubelet/pod-resources"},
351+
},
352+
}),
353+
expectedOutput: NewDaemonset().
354+
WithHostPathVolume(podGPUResourcesVolumeName, "/custom-kubelet/pod-resources", nil).
355+
WithContainer(corev1.Container{
356+
Name: "test-container",
357+
VolumeMounts: []corev1.VolumeMount{
358+
{Name: podGPUResourcesVolumeName, MountPath: "/var/lib/kubelet/pod-resources"},
359+
},
360+
}),
361+
},
362+
{
363+
description: "custom kubeletRootDir with pod-gpu-resources volume and init container mount",
364+
kubeletRootDir: "/custom-kubelet",
365+
input: NewDaemonset().
366+
WithHostPathVolume(podGPUResourcesVolumeName, "/var/lib/kubelet/pod-resources", nil).
367+
WithInitContainer(corev1.Container{
368+
Name: "test-init-container",
369+
VolumeMounts: []corev1.VolumeMount{
370+
{Name: podGPUResourcesVolumeName, MountPath: "/var/lib/kubelet/pod-resources"},
371+
},
372+
}),
373+
expectedOutput: NewDaemonset().
374+
WithHostPathVolume(podGPUResourcesVolumeName, "/custom-kubelet/pod-resources", nil).
375+
WithInitContainer(corev1.Container{
376+
Name: "test-init-container",
377+
VolumeMounts: []corev1.VolumeMount{
378+
{Name: podGPUResourcesVolumeName, MountPath: "/var/lib/kubelet/pod-resources"},
379+
},
380+
}),
381+
},
382+
{
383+
description: "custom kubeletRootDir with device-plugin volume and container mount",
384+
kubeletRootDir: "/custom-kubelet",
385+
input: NewDaemonset().
386+
WithHostPathVolume(devicePluginVolumeName, "/var/lib/kubelet/device-plugins", nil).
387+
WithContainer(corev1.Container{
388+
Name: "test-container",
389+
VolumeMounts: []corev1.VolumeMount{
390+
{Name: devicePluginVolumeName, MountPath: "/var/lib/kubelet/device-plugins"},
391+
},
392+
}),
393+
expectedOutput: NewDaemonset().
394+
WithHostPathVolume(devicePluginVolumeName, "/custom-kubelet/device-plugins", nil).
395+
WithContainer(corev1.Container{
396+
Name: "test-container",
397+
VolumeMounts: []corev1.VolumeMount{
398+
{Name: devicePluginVolumeName, MountPath: "/var/lib/kubelet/device-plugins"},
399+
},
400+
}),
401+
},
402+
}
403+
404+
for _, tc := range testCases {
405+
t.Run(tc.description, func(t *testing.T) {
406+
transformForKubeletRootDir(tc.input.DaemonSet, tc.kubeletRootDir)
407+
require.EqualValues(t, tc.expectedOutput, tc.input)
408+
})
409+
}
410+
}
411+
303412
func TestTransformForDriverInstallDir(t *testing.T) {
304413
driverInstallDirVolumeName := "driver-install-dir"
305414
testCases := []struct {

‎deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,12 @@ spec:
14961496
DriverInstallDir represents the root at which driver files including libraries,
14971497
config files, and executables can be found.
14981498
type: string
1499+
kubeletRootDir:
1500+
default: /var/lib/kubelet
1501+
description: |-
1502+
KubeletRootDir represents the location of the kubelet root directory.
1503+
If empty, it will default to "/var/lib/kubelet".
1504+
type: string
14991505
rootFS:
15001506
description: |-
15011507
RootFS represents the path to the root filesystem of the host.

‎deployments/gpu-operator/templates/clusterpolicy.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ spec:
1515
hostPaths:
1616
rootFS: {{ .Values.hostPaths.rootFS }}
1717
driverInstallDir: {{ .Values.hostPaths.driverInstallDir }}
18+
{{- if .Values.hostPaths.kubeletRootDir }}
19+
kubeletRootDir: {{ .Values.hostPaths.kubeletRootDir }}
20+
{{- end }}
1821
operator:
1922
{{- if .Values.operator.runtimeClass }}
2023
runtimeClass: {{ .Values.operator.runtimeClass }}

‎deployments/gpu-operator/values.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ hostPaths:
3434
# config files, and executables can be found.
3535
driverInstallDir: "/run/nvidia/driver"
3636

37+
# kubeletRootDir represents the root with the kubelet base directory
38+
# if empty will use /var/lib/kubelet as the default path
39+
# kubeletRootDir: ""
40+
3741
daemonsets:
3842
labels: {}
3943
annotations: {}

0 commit comments

Comments
 (0)