Parameterize kubelet root directory(/var/lib/kubelet)

Author: ant31

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -327,15 +327,15 @@ insecure_registries = {{ crio_insecure_registries }}
 default_transport = "docker://"
 
 # The path to a file containing credentials necessary for pulling images from
-# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+# secure registries. The file is similar to that of {{kubelet_root_dir}}/config.json
 global_auth_file = "/etc/crio/config.json"
 
 # The image used to instantiate infra containers.
 # This option supports live configuration reload.
 pause_image = "{{ crio_pause_image }}"
 
 # The path to a file containing credentials specific for pulling the pause_image from
-# above. The file is similar to that of /var/lib/kubelet/config.json
+# above. The file is similar to that of {{kubelet_root_dir}}/config.json
 # This option supports live configuration reload.
 pause_image_auth_file = ""
 

roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2

@@ -34,7 +34,7 @@ spec:
       volumes:
       - name: device-plugin
         hostPath:
-          path: /var/lib/kubelet/device-plugins
+          path: {{kubelet_root_dir}}/device-plugins
       - name: dev
         hostPath:
           path: /dev

roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2

@@ -37,7 +37,7 @@ spec:
               value: unix:/csi/csi.sock
           volumeMounts:
             - name: kubelet-dir
-              mountPath: /var/lib/kubelet
+              mountPath: {{kubelet_root_dir}}
               mountPropagation: "Bidirectional"
             - name: plugin-dir
               mountPath: /csi
@@ -69,7 +69,7 @@ spec:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+              value: {{kubelet_root_dir}}/plugins/ebs.csi.aws.com/csi.sock
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
@@ -85,15 +85,15 @@ spec:
       volumes:
         - name: kubelet-dir
           hostPath:
-            path: /var/lib/kubelet
+            path: {{kubelet_root_dir}}
             type: Directory
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+            path: {{kubelet_root_dir}}/plugins/ebs.csi.aws.com/
             type: DirectoryOrCreate
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{kubelet_root_dir}}/plugins_registry/
             type: Directory
         - name: device-dir
           hostPath:

roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2

@@ -65,7 +65,7 @@ spec:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock
+              value: {{kubelet_root_dir}}/plugins/disk.csi.azure.com/csi.sock
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
@@ -120,7 +120,7 @@ spec:
           volumeMounts:
             - mountPath: /csi
               name: socket-dir
-            - mountPath: /var/lib/kubelet/
+            - mountPath: {{kubelet_root_dir}}/
               mountPropagation: Bidirectional
               name: mountpoint-dir
             - mountPath: /etc/kubernetes/
@@ -139,15 +139,15 @@ spec:
               memory: 20Mi
       volumes:
         - hostPath:
-            path: /var/lib/kubelet/plugins/disk.csi.azure.com
+            path: {{kubelet_root_dir}}/plugins/disk.csi.azure.com
             type: DirectoryOrCreate
           name: socket-dir
         - hostPath:
-            path: /var/lib/kubelet/
+            path: {{kubelet_root_dir}}/
             type: DirectoryOrCreate
           name: mountpoint-dir
         - hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{kubelet_root_dir}}/plugins_registry/
             type: DirectoryOrCreate
           name: registration-dir
         - secret:

roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2

@@ -31,7 +31,7 @@ spec:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock
+              value: {{kubelet_root_dir}}/plugins/cinder.csi.openstack.org/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
@@ -81,7 +81,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
             - name: kubelet-dir
-              mountPath: /var/lib/kubelet
+              mountPath: {{kubelet_root_dir}}
               mountPropagation: "Bidirectional"
             - name: pods-probe-dir
               mountPath: /dev
@@ -107,15 +107,15 @@ spec:
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/cinder.csi.openstack.org
+            path: {{kubelet_root_dir}}/plugins/cinder.csi.openstack.org
             type: DirectoryOrCreate
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{kubelet_root_dir}}/plugins_registry/
             type: Directory
         - name: kubelet-dir
           hostPath:
-            path: /var/lib/kubelet
+            path: {{kubelet_root_dir}}
             type: Directory
         - name: pods-probe-dir
           hostPath:

roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2

@@ -25,7 +25,7 @@ spec:
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
-            - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
+            - "--kubelet-registration-path={{kubelet_root_dir}}/plugins/pd.csi.storage.gke.io/csi.sock"
           lifecycle:
             preStop:
               exec:
@@ -52,7 +52,7 @@ spec:
             - "--run-controller-service=false"
           volumeMounts:
             - name: kubelet-dir
-              mountPath: /var/lib/kubelet
+              mountPath: {{kubelet_root_dir}}
               mountPropagation: "Bidirectional"
             - name: plugin-dir
               mountPath: /csi
@@ -73,15 +73,15 @@ spec:
       volumes:
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{kubelet_root_dir}}/plugins_registry/
             type: Directory
         - name: kubelet-dir
           hostPath:
-            path: /var/lib/kubelet
+            path: {{kubelet_root_dir}}
             type: Directory
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/
+            path: {{kubelet_root_dir}}/plugins/pd.csi.storage.gke.io/
             type: DirectoryOrCreate
         - name: device-dir
           hostPath:

roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2

@@ -27,7 +27,7 @@ spec:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/storage.csi.upcloud.com/csi.sock
+              value: {{kubelet_root_dir}}/plugins/storage.csi.upcloud.com/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
@@ -70,7 +70,7 @@ spec:
             - name: plugin-dir
               mountPath: /csi
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet
+              mountPath: {{kubelet_root_dir}}
               # needed so that any mounts setup inside this container are
               # propagated back to the host machine.
               mountPropagation: "Bidirectional"
@@ -81,15 +81,15 @@ spec:
       volumes:
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{kubelet_root_dir}}/plugins_registry/
             type: DirectoryOrCreate
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/storage.csi.upcloud.com
+            path: {{kubelet_root_dir}}/plugins/storage.csi.upcloud.com
             type: DirectoryOrCreate
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet
+            path: {{kubelet_root_dir}}
             type: Directory
         - name: device-dir
           hostPath:

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2

@@ -48,7 +48,7 @@ spec:
         - name: ADDRESS
           value: /csi/csi.sock
         - name: DRIVER_REG_SOCK_PATH
-          value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
+          value: {{kubelet_root_dir}}/plugins/csi.vsphere.vmware.com/csi.sock
         volumeMounts:
         - name: plugin-dir
           mountPath: /csi
@@ -58,7 +58,7 @@ spec:
           exec:
             command:
             - /csi-node-driver-registrar
-            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
+            - --kubelet-registration-path={{kubelet_root_dir}}/plugins/csi.vsphere.vmware.com/csi.sock
             - --mode=kubelet-registration-probe
           initialDelaySeconds: 3
       - name: vsphere-csi-node
@@ -104,7 +104,7 @@ spec:
         - name: plugin-dir
           mountPath: /csi
         - name: pods-mount-dir
-          mountPath: /var/lib/kubelet
+          mountPath: {{kubelet_root_dir}}
           # needed so that any mounts setup inside this container are
           # propagated back to the host machine.
           mountPropagation: "Bidirectional"
@@ -142,15 +142,15 @@ spec:
       volumes:
       - name: registration-dir
         hostPath:
-          path: /var/lib/kubelet/plugins_registry
+          path: {{kubelet_root_dir}}/plugins_registry
           type: Directory
       - name: plugin-dir
         hostPath:
-          path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com
+          path: {{kubelet_root_dir}}/plugins/csi.vsphere.vmware.com
           type: DirectoryOrCreate
       - name: pods-mount-dir
         hostPath:
-          path: /var/lib/kubelet
+          path: {{kubelet_root_dir}}
           type: Directory
       - name: device-dir
         hostPath:

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -12,7 +12,7 @@
 
 - name: Kubeadm | Check if kubeadm has already run
   stat:
-    path: "/var/lib/kubelet/config.yaml"
+    path: "{{kubelet_root_dir}}/config.yaml"
     get_attributes: false
     get_checksum: false
     get_mime: false

roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml

@@ -3,7 +3,7 @@
   lineinfile:
     path: "{{ kube_config_dir }}/kubelet.conf"
     regexp: '^    client-certificate-data: '
-    line: '    client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    line: "    client-certificate: {{kubelet_root_dir}}/pki/kubelet-client-current.pem"
     backup: true
   notify:
     - "Control plane | reload kubelet"
@@ -12,7 +12,7 @@
   lineinfile:
     path: "{{ kube_config_dir }}/kubelet.conf"
     regexp: '^    client-key-data: '
-    line: '    client-key: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    line: "    client-key: {{kubelet_root_dir}}/pki/kubelet-client-current.pem"
     backup: true
   notify:
     - "Control plane | reload kubelet"

roles/kubernetes/node/defaults/main.yml

@@ -110,7 +110,7 @@ loadbalancer_apiserver_pod_name: "{% if loadbalancer_apiserver_type == 'nginx' %
 # A port range to reserve for services with NodePort visibility.
 # Inclusive at both ends of the range.
 kube_apiserver_node_port_range: "30000-32767"
-
+kubelet_root_dir: /var/lib/kubelet
 # Configure the amount of pods able to run on single node
 # default is equal to application default
 kubelet_max_pods: 110

roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml

@@ -15,7 +15,7 @@
 
 - name: Kube-vip | Check if kubeadm has already run
   stat:
-    path: "/var/lib/kubelet/config.yaml"
+    path: "{{kubelet_root_dir}}/config.yaml"
     get_attributes: false
     get_checksum: false
     get_mime: false

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -12,6 +12,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --kubeconfig={{ kube_config_dir }}/kubelet.conf \
 {# end kubeadm specific settings #}
 --runtime-cgroups={{ kubelet_runtime_cgroups }} \
+--root-dir={{ kubelet_root_dir }} \
 {% endset %}
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}"

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -148,5 +148,5 @@
 
 - name: Set alternate flexvolume path
   set_fact:
-    kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins
+    kubelet_flexvolumes_plugins_dir: "{{kubelet_root_dir}}/volumeplugins"
   when: not usr.stat.writeable

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -107,6 +107,12 @@
     - kube_network_plugin not in ['calico', 'none']
     - ipv4_stack | bool
 
+- name: "Kubelet: check root-dir is not set as custom-flag too"
+  assert:
+    that: "{{item.split('=')[0] != '--root-dir'}}"
+    fail_msg: "kubelet root dir is set via `kubelet_root_dir` variable. Remove it from the kubelet_custom_flags and set the new var"
+  loop: "{{kubelet_custom_flags | d([])}}"
+
 - name: Stop if ip var does not match local ips
   assert:
     that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses)

roles/kubespray-defaults/defaults/main/main.yml

@@ -565,6 +565,7 @@ kubelet_rotate_server_certificates: false
 # If set to true, kubelet errors if any of kernel tunables is different than kubelet defaults
 kubelet_protect_kernel_defaults: true
 
+kubelet_root_dir: /var/lib/kubelet
 # Set additional sysctl variables to modify Linux kernel variables, for example:
 # additional_sysctl:
 #  - { name: kernel.pid_max, value: 131072 }

roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2

@@ -517,7 +517,7 @@ spec:
             path: /lib/modules
         - name: shared-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: {{kubelet_root_dir}}/pods
         - name: systemid
           hostPath:
             path: /etc/origin/openvswitch

roles/reset/tasks/main.yml

@@ -159,7 +159,7 @@
         - services
 
 - name: Reset | gather mounted kubelet dirs
-  shell: set -o pipefail && mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac
+  shell: set -o pipefail && mount | grep {{kubelet_root_dir}}/ | awk '{print $3}' | tac
   args:
     executable: /bin/bash
   check_mode: false
@@ -241,23 +241,23 @@
     - enable_nodelocaldns | default(false) | bool
     - nodelocaldns_device.stat.exists
 
-- name: Reset | Check whether /var/lib/kubelet directory exists
+- name: Reset | Check whether {{kubelet_root_dir}} directory exists
   stat:
-    path: /var/lib/kubelet
+    path: "{{kubelet_root_dir}}"
     get_attributes: false
     get_checksum: false
     get_mime: false
   register: var_lib_kubelet_directory
 
-- name: Reset | Find files/dirs with immutable flag in /var/lib/kubelet
-  command: lsattr -laR /var/lib/kubelet/
+- name: Reset | Find files/dirs with immutable flag in {{kubelet_root_dir}}
+  command: lsattr -laR {{kubelet_root_dir}}/
   become: true
   register: var_lib_kubelet_files_dirs_w_attrs
   changed_when: false
   no_log: true
   when: var_lib_kubelet_directory.stat.exists
 
-- name: Reset | Remove immutable flag from files/dirs in /var/lib/kubelet
+- name: Reset | Remove immutable flag from files/dirs in {{kubelet_root_dir}}
   file:
     path: "{{ filedir_path }}"
     state: touch
@@ -277,7 +277,7 @@
     state: absent
   with_items:
     - "{{ kube_config_dir }}"
-    - /var/lib/kubelet
+    - {{kubelet_root_dir}}
     - "{{ containerd_storage_dir }}"
     - "{{ ansible_env.HOME | default('/root') }}/.kube"
     - "{{ ansible_env.HOME | default('/root') }}/.helm"