Skip to content
This repository was archived by the owner on Jan 18, 2023. It is now read-only.

Commit e3df769

Browse files
przemeklallmdaly
authored andcommitted
Add support for Kubernetes v1.16 (#259)
* When running on Kubernetes v1.9.0 or newer, apps/v1 DaemonSet API will be used to create reconcile/nodereport ds * On Kubernetes older than v1.9.0 legacy extensions/v1beta1 API will be used (the same that got deprecated in v1.16). * Update tests. * Add RBAC rules that allow CMK clusterinit to modify apps/v1 DS resources. * Update template DaemonSet specs and move deprecated ones to '*-legacy.yaml' files. Signed-off-by: Przemysław Lal <[email protected]>
1 parent f7def5f commit e3df769

17 files changed

+366
-71
lines changed

intel/clusterinit.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def cluster_init(host_list, all_hosts, cmd_list, cmk_img, cmk_img_pol,
7878
exclusive_mode, namespace)
7979

8080
# Run mutating webhook admission controller on supported cluster
81-
version = util.parse_version(k8s.get_kubelet_version(None))
81+
version = util.parse_version(k8s.get_kube_version(None))
8282
if version >= util.parse_version("v1.9.0"):
8383
deploy_webhook(namespace, conf_dir, install_dir, serviceaccount,
8484
cmk_img)
@@ -132,12 +132,12 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
132132
install_dir, num_exclusive_cores, num_shared_cores,
133133
cmk_node_list, pull_secret, serviceaccount, shared_mode,
134134
exclusive_mode, namespace):
135+
version = util.parse_version(k8s.get_kube_version(None))
135136
pod = k8s.get_pod_template()
136137
if pull_secret:
137138
update_pod_with_pull_secret(pod, pull_secret)
138139
if cmd_list:
139140
update_pod(pod, "Always", conf_dir, install_dir, serviceaccount)
140-
version = util.parse_version(k8s.get_kubelet_version(None))
141141
if version >= util.parse_version("v1.7.0"):
142142
pod["spec"]["tolerations"] = [{
143143
"operator": "Exists"}]
@@ -179,13 +179,14 @@ def run_cmd_pods(cmd_list, cmd_init_list, cmk_img, cmk_img_pol, conf_dir,
179179
for node_name in cmk_node_list:
180180
if cmd_list:
181181
update_pod_with_node_details(pod, node_name, cmd_list)
182-
daemon_set = k8s.ds_from(pod=pod)
182+
daemon_set = k8s.ds_from(pod=pod, version=version)
183183
elif cmd_init_list:
184184
update_pod_with_node_details(pod, node_name, cmd_init_list)
185185

186186
try:
187187
if cmd_list:
188-
cr_pod_resp = k8s.create_ds(None, daemon_set, namespace)
188+
cr_pod_resp = k8s.create_ds(None, daemon_set, namespace,
189+
version)
189190
logging.debug("Response while creating ds for {} command(s): "
190191
"{}".format(cmd_list, cr_pod_resp))
191192
elif cmd_init_list:
@@ -375,7 +376,7 @@ def update_pod_with_init_container(pod, cmd, cmk_img, cmk_img_pol, args):
375376
container_template["name"] = cmd
376377
pod_init_containers_list = []
377378

378-
version = util.parse_version(k8s.get_kubelet_version(None))
379+
version = util.parse_version(k8s.get_kube_version(None))
379380

380381
if version >= util.parse_version("v1.7.0"):
381382
pod["spec"]["initContainers"] = [container_template]

intel/discover.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
# the appropriate CMK node labels and taints.
3030
def discover(conf_dir):
3131

32-
version = util.parse_version(k8s.get_kubelet_version(None))
32+
version = util.parse_version(k8s.get_kube_version(None))
3333
if version == util.parse_version("v1.8.0"):
3434
logging.fatal("K8s 1.8.0 is not supported. Update K8s to "
3535
"version >=1.8.1 or rollback to previous versions")
@@ -139,7 +139,7 @@ def add_node_taint():
139139
logging.error("Aborting discover ...")
140140
sys.exit(1)
141141

142-
version = util.parse_version(k8s.get_kubelet_version(None))
142+
version = util.parse_version(k8s.get_kube_version(None))
143143
node_taints_list = []
144144
node_taints = []
145145

intel/k8s.py

Lines changed: 68 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import logging
1616

17+
from intel import util
1718
from kubernetes import client as k8sclient, config as k8sconfig
1819
from kubernetes.client import V1Namespace, V1DeleteOptions
1920

@@ -58,25 +59,52 @@ def get_pod_template(saname="cmk-serviceaccount"):
5859
return pod_template
5960

6061

61-
def ds_from(pod):
62-
ds_template = {
63-
"apiVersion": "extensions/v1beta1",
64-
"kind": "DaemonSet",
65-
"metadata": {
66-
"name": pod["metadata"]["name"].replace("pod", "ds")
67-
},
68-
"spec": {
69-
"template": {
70-
"metadata": {
71-
"labels": {
72-
"app":
73-
pod["metadata"]["name"].replace("pod", "ds")
62+
def ds_from(pod, version):
63+
ds_template = {}
64+
if version >= util.parse_version("v1.9.0"):
65+
ds_template = {
66+
"apiVersion": "apps/v1",
67+
"kind": "DaemonSet",
68+
"metadata": {
69+
"name": pod["metadata"]["name"].replace("pod", "ds")
70+
},
71+
"spec": {
72+
"selector": {
73+
"matchLabels": {
74+
"app": pod["metadata"]["name"].replace("pod", "ds")
7475
}
7576
},
76-
"spec": pod["spec"]
77+
"template": {
78+
"metadata": {
79+
"labels": {
80+
"app":
81+
pod["metadata"]["name"].replace("pod", "ds")
82+
}
83+
},
84+
"spec": pod["spec"]
85+
}
86+
}
87+
}
88+
# for k8s versions older than 1.9.0 use extensions/v1beta1 API
89+
else:
90+
ds_template = {
91+
"apiVersion": "extensions/v1beta1",
92+
"kind": "DaemonSet",
93+
"metadata": {
94+
"name": pod["metadata"]["name"].replace("pod", "ds")
95+
},
96+
"spec": {
97+
"template": {
98+
"metadata": {
99+
"labels": {
100+
"app":
101+
pod["metadata"]["name"].replace("pod", "ds")
102+
}
103+
},
104+
"spec": pod["spec"]
105+
}
77106
}
78107
}
79-
}
80108
return ds_template
81109

82110

@@ -217,11 +245,15 @@ def create_pod(config, podspec, ns_name):
217245
return k8s_api.create_namespaced_pod(ns_name, podspec)
218246

219247

220-
# create_ds() sends a request to the Kubernetes API server to create a
221-
# ds based on podspec.
222-
def create_ds(config, podspec, ns_name):
223-
k8s_api = extensions_client_from_config(config)
224-
return k8s_api.create_namespaced_daemon_set(ns_name, podspec)
248+
# create_legacy_ds() sends a request to the Kubernetes API server to create a
249+
# ds based on deamonset spec.
250+
def create_ds(config, spec, ns_name, version):
251+
if version >= util.parse_version("v1.9.0"):
252+
k8s_api = apps_api_client_from_config(config)
253+
return k8s_api.create_namespaced_daemon_set(ns_name, spec)
254+
else:
255+
k8s_api = extensions_client_from_config(config)
256+
return k8s_api.create_namespaced_daemon_set(ns_name, spec)
225257

226258

227259
def create_service(config, spec, ns_name):
@@ -295,8 +327,8 @@ def get_namespaces(config):
295327
return k8s_api.list_namespace().to_dict()
296328

297329

298-
# Get kubelet version from node
299-
def get_kubelet_version(config):
330+
# Get k8s version
331+
def get_kube_version(config):
300332
k8s_api = version_api_client_from_config(config)
301333
version_info = k8s_api.get_code()
302334
return version_info.git_version
@@ -319,14 +351,22 @@ def delete_pod(config, name, ns_name="default", body=V1DeleteOptions()):
319351
# in cascade deletion in k8s, first delete the ds, then the pod
320352
# https://github.com/kubernetes-incubator/client-python/issues/162
321353
# https://github.com/kubernetes/kubernetes/issues/44046
322-
def delete_ds(config, ds_name, ns_name="default", body=V1DeleteOptions()):
323-
k8s_api_ext = extensions_client_from_config(config)
354+
def delete_ds(config, version, ds_name, ns_name="default",
355+
body=V1DeleteOptions()):
324356
k8s_api_core = client_from_config(config)
325357

326-
k8s_api_ext.delete_namespaced_daemon_set(ds_name,
327-
ns_name,
328-
grace_period_seconds=0,
329-
orphan_dependents=False)
358+
if version >= util.parse_version("v1.9.0"):
359+
k8s_api_apps = apps_api_client_from_config(config)
360+
k8s_api_apps.delete_namespaced_daemon_set(ds_name,
361+
ns_name,
362+
grace_period_seconds=0,
363+
orphan_dependents=False)
364+
else:
365+
k8s_api_ext = extensions_client_from_config(config)
366+
k8s_api_ext.delete_namespaced_daemon_set(ds_name,
367+
ns_name,
368+
grace_period_seconds=0,
369+
orphan_dependents=False)
330370

331371
# Pod in ds has fixed label so we use label selector
332372
data = k8s_api_core.list_namespaced_pod(

intel/nodereport.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def nodereport(conf_dir, seconds, publish):
4040
k8sconfig.load_incluster_config()
4141
v1beta = k8sclient.ExtensionsV1beta1Api()
4242

43-
version = util.parse_version(k8s.get_kubelet_version(None))
43+
version = util.parse_version(k8s.get_kube_version(None))
4444

4545
if version >= util.parse_version("v1.7.0"):
4646
node_report_type = \

intel/reconcile.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def reconcile(conf_dir, seconds, publish):
4444
k8sconfig.load_incluster_config()
4545
v1beta = k8sclient.ExtensionsV1beta1Api()
4646

47-
version = util.parse_version(k8s.get_kubelet_version(None))
47+
version = util.parse_version(k8s.get_kube_version(None))
4848

4949
if version >= util.parse_version("v1.7.0"):
5050
reconcile_report_type = \

intel/uninstall.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def remove_binary(install_dir):
6868

6969

7070
def remove_all_report():
71-
version = util.parse_version(k8s.get_kubelet_version(None))
71+
version = util.parse_version(k8s.get_kube_version(None))
7272

7373
if version >= util.parse_version("v1.7.0"):
7474
remove_report_crd("cmk-nodereport", ["cmk-nr"])
@@ -146,7 +146,8 @@ def delete_cmk_pod(pod_base_name, namespace, postfix=None,):
146146
# Pod is part of DaemonSet - remove ds otherwise ds
147147
# controller will restart pod
148148
logging.info("\"{}\" is DaemonSet".format(pod_name))
149-
k8s.delete_ds(None, pod_name, namespace)
149+
api_version = util.parse_version(k8s.get_kube_version(None))
150+
k8s.delete_ds(None, api_version, pod_name, namespace)
150151
else:
151152
k8s.delete_pod(None, pod_name, namespace)
152153
except K8sApiException as err:
@@ -259,7 +260,7 @@ def remove_node_taint():
259260
"\"{}\" obj: {}".format(node_name, err))
260261
sys.exit(1)
261262

262-
version = util.parse_version(k8s.get_kubelet_version(None))
263+
version = util.parse_version(k8s.get_kube_version(None))
263264
node_taints_list = []
264265

265266
if version >= util.parse_version("v1.7.0"):
@@ -299,7 +300,7 @@ def remove_node_taint():
299300

300301

301302
def remove_resource_tracking():
302-
version = util.parse_version(k8s.get_kubelet_version(None))
303+
version = util.parse_version(k8s.get_kube_version(None))
303304
if version == util.parse_version("v1.8.0"):
304305
logging.warning("Unsupported Kubernetes version")
305306
elif version >= util.parse_version("v1.8.1"):
@@ -354,7 +355,7 @@ def remove_node_cmk_er():
354355

355356

356357
def remove_webhook_resources(prefix, namespace):
357-
version = util.parse_version(k8s.get_kubelet_version(None))
358+
version = util.parse_version(k8s.get_kube_version(None))
358359
if version >= util.parse_version("v1.9.0"):
359360
try:
360361
k8s.delete_mutating_webhook_configuration(

resources/authorization/cmk-rbac-rules.yaml

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
2727
metadata:
2828
name: cmk-daemonset-controller
2929
rules:
30-
- apiGroups: ["extensions"]
31-
resources: ["daemonsets", "daemonsets.extensions"]
30+
- apiGroups: ["extensions", "apps"]
31+
resources: ["daemonsets", "daemonsets.extensions", "daemonsets.apps"]
3232
verbs: ["*"]
3333
---
3434
kind: ClusterRole
@@ -49,6 +49,15 @@ rules:
4949
resources: ["secrets", "configmaps", "deployments", "services", "mutatingwebhookconfigurations"]
5050
verbs: ["*"]
5151
---
52+
kind: ClusterRole
53+
apiVersion: rbac.authorization.k8s.io/v1beta1
54+
metadata:
55+
name: cmk-node-lister
56+
rules:
57+
- apiGroups: [""]
58+
resources: ["nodes"]
59+
verbs: ["*"]
60+
---
5261
apiVersion: rbac.authorization.k8s.io/v1beta1
5362
kind: ClusterRoleBinding
5463
metadata:
@@ -126,3 +135,16 @@ subjects:
126135
- kind: ServiceAccount
127136
name: cmk-serviceaccount
128137
namespace: cmk-namespace
138+
---
139+
apiVersion: rbac.authorization.k8s.io/v1beta1
140+
kind: ClusterRoleBinding
141+
metadata:
142+
name: cmk-role-binding-node-lister
143+
roleRef:
144+
apiGroup: rbac.authorization.k8s.io
145+
kind: ClusterRole
146+
name: cmk-node-lister
147+
subjects:
148+
- kind: ServiceAccount
149+
name: cmk-serviceaccount
150+
namespace: cmk-namespace
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright (c) 2017 Intel Corporation
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
apiVersion: extensions/v1beta1
16+
kind: DaemonSet
17+
metadata:
18+
labels:
19+
app: cmk-node-report-ds-all
20+
# Needed for k8s < 1.7
21+
# annotations:
22+
# "scheduler.alpha.kubernetes.io/tolerations": '[{"key":"cmk", "value":"true"}]'
23+
name: cmk-node-report-ds-all
24+
namespace: cmk-namespace
25+
spec:
26+
template:
27+
metadata:
28+
labels:
29+
app: cmk-node-report-ds-all
30+
spec:
31+
serviceAccountName: cmk-serviceaccount
32+
# Needed for k8s >= 1.7
33+
# tolerations:
34+
# - operator: "Exists"
35+
containers:
36+
- args:
37+
- "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- node-report --interval=$CMK_NODE_REPORT_SLEEP_TIME --publish"
38+
command:
39+
- "/bin/bash"
40+
- "-c"
41+
env:
42+
- name: CMK_NODE_REPORT_SLEEP_TIME
43+
# Change this to modify the sleep interval between consecutive
44+
# cmk node report runs. The value is specified in seconds.
45+
value: '60'
46+
- name: CMK_PROC_FS
47+
value: "/host/proc"
48+
- name: NODE_NAME
49+
valueFrom:
50+
fieldRef:
51+
fieldPath: spec.nodeName
52+
image: cmk:v1.3.1
53+
name: cmk-nodereport
54+
volumeMounts:
55+
- mountPath: "/host/proc"
56+
name: host-proc
57+
readOnly: true
58+
- mountPath: "/etc/cmk"
59+
name: cmk-conf-dir
60+
volumes:
61+
- hostPath:
62+
path: "/proc"
63+
name: host-proc
64+
- hostPath:
65+
# Change this to modify the CMK config dir in the host file system.
66+
path: "/etc/cmk"
67+
name: cmk-conf-dir

resources/pods/cmk-nodereport-daemonset.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
#
15-
apiVersion: extensions/v1beta1
15+
apiVersion: apps/v1
1616
kind: DaemonSet
1717
metadata:
1818
labels:
@@ -23,6 +23,9 @@ metadata:
2323
name: cmk-node-report-ds-all
2424
namespace: cmk-namespace
2525
spec:
26+
selector:
27+
matchLabels:
28+
app: cmk-node-report-ds-all
2629
template:
2730
metadata:
2831
labels:

0 commit comments

Comments
 (0)