Add support for Kubernetes v1.16 (#259)

* When running on Kubernetes v1.9.0 or newer, apps/v1 DaemonSet API will
be used to create reconcile/nodereport ds
* On Kubernetes older than v1.9.0 legacy extensions/v1beta1 API will be
used (the same that got deprecated in v1.16).
* Update tests.
* Add RBAC rules that allow CMK clusterinit to modify apps/v1 DS resources.
* Update template DaemonSet specs and move deprecated ones to
'*-legacy.yaml' files.

Signed-off-by: Przemysław Lal <[email protected]>

Author: przemeklal

intel/clusterinit.py

@@ -78,7 +78,7 @@ def cluster_init(host_list, all_hosts, cmd_list, cmk_img, cmk_img_pol,
                  exclusive_mode, namespace)
 
     # Run mutating webhook admission controller on supported cluster
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     if version >= util.parse_version("v1.9.0"):
         deploy_webhook(namespace, conf_dir, install_dir, serviceaccount,
                        cmk_img)
@@ -132,12 +132,12 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
                  install_dir, num_exclusive_cores, num_shared_cores,
                  cmk_node_list, pull_secret, serviceaccount, shared_mode,
                  exclusive_mode, namespace):
+    version = util.parse_version(k8s.get_kube_version(None))
     pod = k8s.get_pod_template()
     if pull_secret:
         update_pod_with_pull_secret(pod, pull_secret)
     if cmd_list:
         update_pod(pod, "Always", conf_dir, install_dir, serviceaccount)
-        version = util.parse_version(k8s.get_kubelet_version(None))
         if version >= util.parse_version("v1.7.0"):
             pod["spec"]["tolerations"] = [{
                 "operator": "Exists"}]
@@ -179,13 +179,14 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
     for node_name in cmk_node_list:
         if cmd_list:
             update_pod_with_node_details(pod, node_name, cmd_list)
-            daemon_set = k8s.ds_from(pod=pod)
+            daemon_set = k8s.ds_from(pod=pod, version=version)
         elif cmd_init_list:
             update_pod_with_node_details(pod, node_name, cmd_init_list)
 
         try:
             if cmd_list:
-                cr_pod_resp = k8s.create_ds(None, daemon_set, namespace)
+                cr_pod_resp = k8s.create_ds(None, daemon_set, namespace,
+                                            version)
                 logging.debug("Response while creating ds for {} command(s): "
                               "{}".format(cmd_list, cr_pod_resp))
             elif cmd_init_list:
@@ -375,7 +376,7 @@ def update_pod_with_init_container(pod, cmd, cmk_img, cmk_img_pol, args):
     container_template["name"] = cmd
     pod_init_containers_list = []
 
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
 
     if version >= util.parse_version("v1.7.0"):
         pod["spec"]["initContainers"] = [container_template]

intel/discover.py

@@ -29,7 +29,7 @@
 # the appropriate CMK node labels and taints.
 def discover(conf_dir):
 
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     if version == util.parse_version("v1.8.0"):
         logging.fatal("K8s 1.8.0 is not supported. Update K8s to "
                       "version >=1.8.1 or rollback to previous versions")
@@ -139,7 +139,7 @@ def add_node_taint():
         logging.error("Aborting discover ...")
         sys.exit(1)
 
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     node_taints_list = []
     node_taints = []
 

intel/k8s.py

@@ -14,6 +14,7 @@
 
 import logging
 
+from intel import util
 from kubernetes import client as k8sclient, config as k8sconfig
 from kubernetes.client import V1Namespace, V1DeleteOptions
 
@@ -58,25 +59,52 @@ def get_pod_template(saname="cmk-serviceaccount"):
     return pod_template
 
 
-def ds_from(pod):
-    ds_template = {
-        "apiVersion": "extensions/v1beta1",
-        "kind": "DaemonSet",
-        "metadata": {
-            "name": pod["metadata"]["name"].replace("pod", "ds")
-        },
-        "spec": {
-            "template": {
-                "metadata": {
-                    "labels": {
-                        "app":
-                            pod["metadata"]["name"].replace("pod", "ds")
+def ds_from(pod, version):
+    ds_template = {}
+    if version >= util.parse_version("v1.9.0"):
+        ds_template = {
+            "apiVersion": "apps/v1",
+            "kind": "DaemonSet",
+            "metadata": {
+                "name": pod["metadata"]["name"].replace("pod", "ds")
+            },
+            "spec": {
+                "selector": {
+                    "matchLabels": {
+                        "app": pod["metadata"]["name"].replace("pod", "ds")
                     }
                 },
-                "spec": pod["spec"]
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app":
+                                pod["metadata"]["name"].replace("pod", "ds")
+                        }
+                    },
+                    "spec": pod["spec"]
+                }
+            }
+        }
+    # for k8s versions older than 1.9.0 use extensions/v1beta1 API
+    else:
+        ds_template = {
+            "apiVersion": "extensions/v1beta1",
+            "kind": "DaemonSet",
+            "metadata": {
+                "name": pod["metadata"]["name"].replace("pod", "ds")
+            },
+            "spec": {
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app":
+                                pod["metadata"]["name"].replace("pod", "ds")
+                        }
+                    },
+                    "spec": pod["spec"]
+                }
             }
         }
-    }
     return ds_template
 
 
@@ -217,11 +245,15 @@ def create_pod(config, podspec, ns_name):
     return k8s_api.create_namespaced_pod(ns_name, podspec)
 
 
-# create_ds() sends a request to the Kubernetes API server to create a
-# ds based on podspec.
-def create_ds(config, podspec, ns_name):
-    k8s_api = extensions_client_from_config(config)
-    return k8s_api.create_namespaced_daemon_set(ns_name, podspec)
+# create_legacy_ds() sends a request to the Kubernetes API server to create a
+# ds based on deamonset spec.
+def create_ds(config, spec, ns_name, version):
+    if version >= util.parse_version("v1.9.0"):
+        k8s_api = apps_api_client_from_config(config)
+        return k8s_api.create_namespaced_daemon_set(ns_name, spec)
+    else:
+        k8s_api = extensions_client_from_config(config)
+        return k8s_api.create_namespaced_daemon_set(ns_name, spec)
 
 
 def create_service(config, spec, ns_name):
@@ -295,8 +327,8 @@ def get_namespaces(config):
     return k8s_api.list_namespace().to_dict()
 
 
-# Get kubelet version from node
-def get_kubelet_version(config):
+# Get k8s version
+def get_kube_version(config):
     k8s_api = version_api_client_from_config(config)
     version_info = k8s_api.get_code()
     return version_info.git_version
@@ -319,14 +351,22 @@ def delete_pod(config, name, ns_name="default", body=V1DeleteOptions()):
 # in cascade deletion in k8s, first delete the ds, then the pod
 # https://github.com/kubernetes-incubator/client-python/issues/162
 # https://github.com/kubernetes/kubernetes/issues/44046
-def delete_ds(config, ds_name, ns_name="default", body=V1DeleteOptions()):
-    k8s_api_ext = extensions_client_from_config(config)
+def delete_ds(config, version, ds_name, ns_name="default",
+              body=V1DeleteOptions()):
     k8s_api_core = client_from_config(config)
 
-    k8s_api_ext.delete_namespaced_daemon_set(ds_name,
-                                             ns_name,
-                                             grace_period_seconds=0,
-                                             orphan_dependents=False)
+    if version >= util.parse_version("v1.9.0"):
+        k8s_api_apps = apps_api_client_from_config(config)
+        k8s_api_apps.delete_namespaced_daemon_set(ds_name,
+                                                  ns_name,
+                                                  grace_period_seconds=0,
+                                                  orphan_dependents=False)
+    else:
+        k8s_api_ext = extensions_client_from_config(config)
+        k8s_api_ext.delete_namespaced_daemon_set(ds_name,
+                                                 ns_name,
+                                                 grace_period_seconds=0,
+                                                 orphan_dependents=False)
 
     # Pod in ds has fixed label so we use label selector
     data = k8s_api_core.list_namespaced_pod(

intel/nodereport.py

@@ -40,7 +40,7 @@ def nodereport(conf_dir, seconds, publish):
             k8sconfig.load_incluster_config()
             v1beta = k8sclient.ExtensionsV1beta1Api()
 
-            version = util.parse_version(k8s.get_kubelet_version(None))
+            version = util.parse_version(k8s.get_kube_version(None))
 
             if version >= util.parse_version("v1.7.0"):
                 node_report_type = \

intel/reconcile.py

@@ -44,7 +44,7 @@ def reconcile(conf_dir, seconds, publish):
             k8sconfig.load_incluster_config()
             v1beta = k8sclient.ExtensionsV1beta1Api()
 
-            version = util.parse_version(k8s.get_kubelet_version(None))
+            version = util.parse_version(k8s.get_kube_version(None))
 
             if version >= util.parse_version("v1.7.0"):
                 reconcile_report_type = \

intel/uninstall.py

@@ -68,7 +68,7 @@ def remove_binary(install_dir):
 
 
 def remove_all_report():
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
 
     if version >= util.parse_version("v1.7.0"):
         remove_report_crd("cmk-nodereport", ["cmk-nr"])
@@ -146,7 +146,8 @@ def delete_cmk_pod(pod_base_name, namespace, postfix=None,):
             # Pod is part of DaemonSet - remove ds otherwise ds
             # controller will restart pod
             logging.info("\"{}\" is DaemonSet".format(pod_name))
-            k8s.delete_ds(None, pod_name, namespace)
+            api_version = util.parse_version(k8s.get_kube_version(None))
+            k8s.delete_ds(None, api_version, pod_name, namespace)
         else:
             k8s.delete_pod(None, pod_name, namespace)
     except K8sApiException as err:
@@ -259,7 +260,7 @@ def remove_node_taint():
                       "\"{}\" obj: {}".format(node_name, err))
         sys.exit(1)
 
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     node_taints_list = []
 
     if version >= util.parse_version("v1.7.0"):
@@ -299,7 +300,7 @@ def remove_node_taint():
 
 
 def remove_resource_tracking():
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     if version == util.parse_version("v1.8.0"):
         logging.warning("Unsupported Kubernetes version")
     elif version >= util.parse_version("v1.8.1"):
@@ -354,7 +355,7 @@ def remove_node_cmk_er():
 
 
 def remove_webhook_resources(prefix, namespace):
-    version = util.parse_version(k8s.get_kubelet_version(None))
+    version = util.parse_version(k8s.get_kube_version(None))
     if version >= util.parse_version("v1.9.0"):
         try:
             k8s.delete_mutating_webhook_configuration(

resources/authorization/cmk-rbac-rules.yaml

@@ -27,8 +27,8 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: cmk-daemonset-controller
 rules:
-- apiGroups: ["extensions"]
-  resources: ["daemonsets", "daemonsets.extensions"]
+- apiGroups: ["extensions", "apps"]
+  resources: ["daemonsets", "daemonsets.extensions", "daemonsets.apps"]
   verbs: ["*"]
 ---
 kind: ClusterRole
@@ -49,6 +49,15 @@ rules:
   resources: ["secrets", "configmaps", "deployments", "services", "mutatingwebhookconfigurations"]
   verbs: ["*"]
 ---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: cmk-node-lister
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["*"]
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
@@ -126,3 +135,16 @@ subjects:
 - kind: ServiceAccount
   name: cmk-serviceaccount
   namespace: cmk-namespace
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: cmk-role-binding-node-lister
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cmk-node-lister
+subjects:
+- kind: ServiceAccount
+  name: cmk-serviceaccount
+  namespace: cmk-namespace

resources/pods/cmk-nodereport-daemonset-legacy.yaml

@@ -0,0 +1,67 @@
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    app: cmk-node-report-ds-all
+# Needed for k8s < 1.7
+#  annotations:
+#    "scheduler.alpha.kubernetes.io/tolerations": '[{"key":"cmk", "value":"true"}]'
+  name: cmk-node-report-ds-all
+  namespace: cmk-namespace
+spec:
+  template:
+    metadata:
+      labels:
+        app: cmk-node-report-ds-all
+    spec:
+      serviceAccountName: cmk-serviceaccount
+# Needed for k8s >= 1.7
+#      tolerations:
+#      - operator: "Exists"
+      containers:
+      - args:
+        - "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- node-report --interval=$CMK_NODE_REPORT_SLEEP_TIME --publish"
+        command:
+        - "/bin/bash"
+        - "-c"
+        env:
+        - name: CMK_NODE_REPORT_SLEEP_TIME
+          # Change this to modify the sleep interval between consecutive
+          # cmk node report runs. The value is specified in seconds.
+          value: '60'
+        - name: CMK_PROC_FS
+          value: "/host/proc"
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: cmk:v1.3.1
+        name: cmk-nodereport
+        volumeMounts:
+        - mountPath: "/host/proc"
+          name: host-proc
+          readOnly: true
+        - mountPath: "/etc/cmk"
+          name: cmk-conf-dir
+      volumes:
+      - hostPath:
+          path: "/proc"
+        name: host-proc
+      - hostPath:
+          # Change this to modify the CMK config dir in the host file system.
+          path: "/etc/cmk"
+        name: cmk-conf-dir

resources/pods/cmk-nodereport-daemonset.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   labels:
@@ -23,6 +23,9 @@ metadata:
   name: cmk-node-report-ds-all
   namespace: cmk-namespace
 spec:
+  selector:
+    matchLabels:
+      app: cmk-node-report-ds-all
   template:
     metadata:
       labels: