Skip to content

Commit ae4c7ef

Browse files
ChengYanJinChengYanJin
committed
MK8S-184 - Add restart script to react when control plane ingress change
1 parent ff83f83 commit ae4c7ef

File tree

6 files changed

+237
-0
lines changed

6 files changed

+237
-0
lines changed

‎salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/usr/bin/env python3
2+
import hashlib
3+
import json
4+
import os
5+
import ssl
6+
import sys
7+
import urllib.error
8+
import urllib.request
9+
from datetime import datetime, timezone
10+
11+
CA_DIR = "/tmp/secrets"
12+
HASH_FILE = os.path.join(CA_DIR, ".ca-hash-previous")
13+
14+
15+
def hash_dir(path):
16+
h = hashlib.sha256()
17+
for fname in sorted(os.listdir(path)):
18+
if fname.startswith("."):
19+
continue
20+
fpath = os.path.join(path, fname)
21+
if os.path.isfile(fpath):
22+
with open(fpath, "rb") as f:
23+
h.update(fname.encode())
24+
h.update(f.read())
25+
return h.hexdigest()
26+
27+
28+
def trigger_restart(namespace, deployment):
29+
token = open("/var/run/secrets/kubernetes.io/serviceaccount/token").read()
30+
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
31+
body = json.dumps(
32+
{
33+
"spec": {
34+
"template": {
35+
"metadata": {
36+
"annotations": {"kubectl.kubernetes.io/restartedAt": timestamp}
37+
}
38+
}
39+
}
40+
}
41+
).encode()
42+
ctx = ssl.create_default_context(
43+
cafile="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
44+
)
45+
url = (
46+
f"https://kubernetes.default.svc/apis/apps/v1"
47+
f"/namespaces/{namespace}/deployments/{deployment}"
48+
)
49+
req = urllib.request.Request(
50+
url,
51+
data=body,
52+
method="PATCH",
53+
headers={
54+
"Authorization": "Bearer " + token,
55+
"Content-Type": "application/strategic-merge-patch+json",
56+
},
57+
)
58+
urllib.request.urlopen(req, context=ctx)
59+
60+
61+
def main():
62+
if not [f for f in os.listdir(CA_DIR) if not f.startswith(".")]:
63+
print("CA directory empty, skipping")
64+
return
65+
66+
current_hash = hash_dir(CA_DIR)
67+
68+
if not os.path.exists(HASH_FILE):
69+
with open(HASH_FILE, "w") as f:
70+
f.write(current_hash)
71+
print("Initial CA load, skipping restart")
72+
return
73+
74+
with open(HASH_FILE) as f:
75+
previous_hash = f.read().strip()
76+
77+
if current_hash == previous_hash:
78+
return
79+
80+
namespace = os.environ["POD_NAMESPACE"]
81+
deployment = os.environ["DEPLOYMENT_NAME"]
82+
83+
try:
84+
trigger_restart(namespace, deployment)
85+
except urllib.error.URLError as e:
86+
print(
87+
f"Failed to trigger restart for {deployment}: {e}",
88+
file=sys.stderr,
89+
)
90+
sys.exit(1)
91+
92+
# Persist hash only after successful restart
93+
with open(HASH_FILE, "w") as f:
94+
f.write(current_hash)
95+
print(f"Rolling restart triggered for {deployment}")
96+
97+
98+
if __name__ == "__main__":
99+
main()

‎salt/metalk8s/addons/prometheus-operator/deployed/init.sls‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ include:
1111
- .thanos-query-sd-files
1212
- .thanos-chart
1313
- .oidc-proxy-rbac
14+
- .oidc-proxy-restart-script
1415
- .oidc-proxy-prometheus
1516
- .oidc-proxy-alertmanager

‎salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,18 @@ Create oauth2-proxy-alertmanager Deployment:
6060
value: secret
6161
- name: UNIQUE_FILENAMES
6262
value: "true"
63+
- name: SCRIPT
64+
value: /scripts/restart-on-ca-change.sh
65+
- name: POD_NAMESPACE
66+
value: metalk8s-monitoring
67+
- name: DEPLOYMENT_NAME
68+
value: oauth2-proxy-alertmanager
6369
volumeMounts:
6470
- name: secrets-volume
6571
mountPath: /tmp/secrets
72+
- name: restart-script
73+
mountPath: /scripts
74+
readOnly: true
6675
containers:
6776
- name: oauth2-proxy
6877
image: {{ build_image_name("oauth2-proxy") }}
@@ -93,6 +102,9 @@ Create oauth2-proxy-alertmanager Deployment:
93102
volumes:
94103
- name: secrets-volume
95104
emptyDir: {}
105+
- name: restart-script
106+
configMap:
107+
name: oidc-proxy-restart-script
96108
97109
Create oauth2-proxy-alertmanager Service:
98110
metalk8s_kubernetes.object_present:

‎salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,18 @@ Create oauth2-proxy-prometheus Deployment:
5959
value: secret
6060
- name: UNIQUE_FILENAMES
6161
value: "true"
62+
- name: SCRIPT
63+
value: /scripts/restart-on-ca-change.sh
64+
- name: POD_NAMESPACE
65+
value: metalk8s-monitoring
66+
- name: DEPLOYMENT_NAME
67+
value: oauth2-proxy-prometheus
6268
volumeMounts:
6369
- name: secrets-volume
6470
mountPath: /tmp/secrets
71+
- name: restart-script
72+
mountPath: /scripts
73+
readOnly: true
6574
containers:
6675
- name: oauth2-proxy
6776
image: {{ build_image_name("oauth2-proxy") }}
@@ -92,6 +101,9 @@ Create oauth2-proxy-prometheus Deployment:
92101
volumes:
93102
- name: secrets-volume
94103
emptyDir: {}
104+
- name: restart-script
105+
configMap:
106+
name: oidc-proxy-restart-script
95107
96108
Create oauth2-proxy-prometheus Service:
97109
metalk8s_kubernetes.object_present:

‎salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,37 @@ Create oidc-proxy-prometheus-secret-reader-binding RoleBinding in {{ prometheus_
6767
name: oidc-proxy-prometheus-secret-reader
6868
apiGroup: rbac.authorization.k8s.io
6969

70+
Create oidc-proxy-prometheus-deployment-restarter Role:
71+
metalk8s_kubernetes.object_present:
72+
- manifest:
73+
apiVersion: rbac.authorization.k8s.io/v1
74+
kind: Role
75+
metadata:
76+
name: oidc-proxy-prometheus-deployment-restarter
77+
namespace: metalk8s-monitoring
78+
rules:
79+
- apiGroups: ["apps"]
80+
resources: ["deployments"]
81+
resourceNames: ["oauth2-proxy-prometheus"]
82+
verbs: ["get", "patch"]
83+
84+
Create oidc-proxy-prometheus-deployment-restarter-binding RoleBinding:
85+
metalk8s_kubernetes.object_present:
86+
- manifest:
87+
apiVersion: rbac.authorization.k8s.io/v1
88+
kind: RoleBinding
89+
metadata:
90+
name: oidc-proxy-prometheus-deployment-restarter-binding
91+
namespace: metalk8s-monitoring
92+
subjects:
93+
- kind: ServiceAccount
94+
name: oidc-proxy-prometheus
95+
namespace: metalk8s-monitoring
96+
roleRef:
97+
kind: Role
98+
name: oidc-proxy-prometheus-deployment-restarter
99+
apiGroup: rbac.authorization.k8s.io
100+
70101
{%- else %}
71102

72103
Ensure oidc-proxy-prometheus ServiceAccount does not exist:
@@ -90,6 +121,20 @@ Ensure oidc-proxy-prometheus-secret-reader-binding RoleBinding does not exist in
90121
- kind: RoleBinding
91122
- apiVersion: rbac.authorization.k8s.io/v1
92123

124+
Ensure oidc-proxy-prometheus-deployment-restarter Role does not exist:
125+
metalk8s_kubernetes.object_absent:
126+
- name: oidc-proxy-prometheus-deployment-restarter
127+
- namespace: metalk8s-monitoring
128+
- kind: Role
129+
- apiVersion: rbac.authorization.k8s.io/v1
130+
131+
Ensure oidc-proxy-prometheus-deployment-restarter-binding RoleBinding does not exist:
132+
metalk8s_kubernetes.object_absent:
133+
- name: oidc-proxy-prometheus-deployment-restarter-binding
134+
- namespace: metalk8s-monitoring
135+
- kind: RoleBinding
136+
- apiVersion: rbac.authorization.k8s.io/v1
137+
93138
{%- endif %}
94139

95140
{%- if alertmanager_oidc_enabled %}
@@ -133,6 +178,37 @@ Create oidc-proxy-alertmanager-secret-reader-binding RoleBinding in {{ alertmana
133178
name: oidc-proxy-alertmanager-secret-reader
134179
apiGroup: rbac.authorization.k8s.io
135180

181+
Create oidc-proxy-alertmanager-deployment-restarter Role:
182+
metalk8s_kubernetes.object_present:
183+
- manifest:
184+
apiVersion: rbac.authorization.k8s.io/v1
185+
kind: Role
186+
metadata:
187+
name: oidc-proxy-alertmanager-deployment-restarter
188+
namespace: metalk8s-monitoring
189+
rules:
190+
- apiGroups: ["apps"]
191+
resources: ["deployments"]
192+
resourceNames: ["oauth2-proxy-alertmanager"]
193+
verbs: ["get", "patch"]
194+
195+
Create oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding:
196+
metalk8s_kubernetes.object_present:
197+
- manifest:
198+
apiVersion: rbac.authorization.k8s.io/v1
199+
kind: RoleBinding
200+
metadata:
201+
name: oidc-proxy-alertmanager-deployment-restarter-binding
202+
namespace: metalk8s-monitoring
203+
subjects:
204+
- kind: ServiceAccount
205+
name: oidc-proxy-alertmanager
206+
namespace: metalk8s-monitoring
207+
roleRef:
208+
kind: Role
209+
name: oidc-proxy-alertmanager-deployment-restarter
210+
apiGroup: rbac.authorization.k8s.io
211+
136212
{%- else %}
137213

138214
Ensure oidc-proxy-alertmanager ServiceAccount does not exist:
@@ -156,4 +232,18 @@ Ensure oidc-proxy-alertmanager-secret-reader-binding RoleBinding does not exist
156232
- kind: RoleBinding
157233
- apiVersion: rbac.authorization.k8s.io/v1
158234

235+
Ensure oidc-proxy-alertmanager-deployment-restarter Role does not exist:
236+
metalk8s_kubernetes.object_absent:
237+
- name: oidc-proxy-alertmanager-deployment-restarter
238+
- namespace: metalk8s-monitoring
239+
- kind: Role
240+
- apiVersion: rbac.authorization.k8s.io/v1
241+
242+
Ensure oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding does not exist:
243+
metalk8s_kubernetes.object_absent:
244+
- name: oidc-proxy-alertmanager-deployment-restarter-binding
245+
- namespace: metalk8s-monitoring
246+
- kind: RoleBinding
247+
- apiVersion: rbac.authorization.k8s.io/v1
248+
159249
{%- endif %}

‎salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{%- set script_content = salt['cp.get_file_str'](
2+
'salt://metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py',
3+
saltenv=saltenv
4+
)
5+
%}
6+
7+
Create oidc-proxy-restart-script ConfigMap:
8+
metalk8s_kubernetes.object_present:
9+
- manifest:
10+
apiVersion: v1
11+
kind: ConfigMap
12+
metadata:
13+
name: oidc-proxy-restart-script
14+
namespace: metalk8s-monitoring
15+
labels:
16+
app.kubernetes.io/managed-by: salt
17+
app.kubernetes.io/part-of: metalk8s
18+
heritage: metalk8s
19+
data:
20+
restart-on-ca-change.py: |-
21+
{{ script_content | indent(12, first=True) }}
22+
restart-on-ca-change.sh: |
23+
python3 /scripts/restart-on-ca-change.py

0 commit comments

Comments
 (0)