MK8S-184 - Add restart script to react when control plane ingress change

buildchain/buildchain/salt_tree.py

@@ -408,6 +408,14 @@ def _download_ui_operator_crds() -> str:
     Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-chart.sls"),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-query-sd-files.sls"),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls"),
+    Path(
+        "salt/metalk8s/addons/prometheus-operator/deployed/",
+        "oidc-proxy-restart-script.sls",
+    ),
+    Path(
+        "salt/metalk8s/addons/prometheus-operator/deployed/files/",
+        "restart-on-ca-change.py",
+    ),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls"),
     Path(
         "salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls"

salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls

@@ -7,6 +7,8 @@ kind: Secret
 metadata:
   name: ingress-control-plane-default-certificate
   namespace: metalk8s-ingress
+  labels:
+    metalk8s.scality.com/oidc-ca: "true"
 type: Opaque
 data:
   tls.crt: "{{

salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import hashlib
+import json
+import os
+import ssl
+import sys
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+
+CA_DIR = "/tmp/secrets"
+HASH_FILE = os.path.join(CA_DIR, ".ca-hash-previous")
+
+
+def hash_dir(path):
+    h = hashlib.sha256()
+    for fname in sorted(os.listdir(path)):
+        if fname.startswith("."):
+            continue
+        fpath = os.path.join(path, fname)
+        if os.path.isfile(fpath):
+            with open(fpath, "rb") as f:
+                h.update(fname.encode())
+                h.update(f.read())
+    return h.hexdigest()
+
+
+def trigger_restart(namespace, deployment):
+    with open("/var/run/secrets/kubernetes.io/serviceaccount/token") as f:
+        token = f.read()
+    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    body = json.dumps(
+        {
+            "spec": {
+                "template": {
+                    "metadata": {
+                        "annotations": {"kubectl.kubernetes.io/restartedAt": timestamp}
+                    }
+                }
+            }
+        }
+    ).encode()
+    ctx = ssl.create_default_context(
+        cafile="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    )
+    url = (
+        f"https://kubernetes.default.svc/apis/apps/v1"
+        f"/namespaces/{namespace}/deployments/{deployment}"
+    )
+    req = urllib.request.Request(
+        url,
+        data=body,
+        method="PATCH",
+        headers={
+            "Authorization": "Bearer " + token,
+            "Content-Type": "application/strategic-merge-patch+json",
+        },
+    )
+    urllib.request.urlopen(req, context=ctx)
+
+
+def main():
+    if not [f for f in os.listdir(CA_DIR) if not f.startswith(".")]:
+        print("CA directory empty, skipping")
+        return
+
+    current_hash = hash_dir(CA_DIR)
+
+    if not os.path.exists(HASH_FILE):
+        with open(HASH_FILE, "w") as f:
+            f.write(current_hash)
+        print("Initial CA load, skipping restart")
+        return
+
+    with open(HASH_FILE) as f:
+        previous_hash = f.read().strip()
+
+    if current_hash == previous_hash:
+        return
+
+    namespace = os.environ["POD_NAMESPACE"]
+    deployment = os.environ["DEPLOYMENT_NAME"]
+
+    try:
+        trigger_restart(namespace, deployment)
+    except urllib.error.URLError as e:
+        print(
+            f"Failed to trigger restart for {deployment}: {e}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    # Persist hash only after successful restart
+    with open(HASH_FILE, "w") as f:
+        f.write(current_hash)
+    print(f"Rolling restart triggered for {deployment}")
+
+
+if __name__ == "__main__":
+    main()

salt/metalk8s/addons/prometheus-operator/deployed/init.sls

@@ -11,5 +11,6 @@ include:
   - .thanos-query-sd-files
   - .thanos-chart
   - .oidc-proxy-rbac
+  - .oidc-proxy-restart-script
   - .oidc-proxy-prometheus
   - .oidc-proxy-alertmanager

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls

@@ -60,9 +60,18 @@ Create oauth2-proxy-alertmanager Deployment:
                   value: secret
                 - name: UNIQUE_FILENAMES
                   value: "true"
+                - name: SCRIPT
+                  value: /scripts/restart-on-ca-change.sh
+                - name: POD_NAMESPACE
+                  value: metalk8s-monitoring
+                - name: DEPLOYMENT_NAME
+                  value: oauth2-proxy-alertmanager
                 volumeMounts:
                 - name: secrets-volume
                   mountPath: /tmp/secrets
+                - name: restart-script
+                  mountPath: /scripts
+                  readOnly: true
               containers:
               - name: oauth2-proxy
                 image: {{ build_image_name("oauth2-proxy") }}
@@ -93,6 +102,9 @@ Create oauth2-proxy-alertmanager Deployment:
               volumes:
               - name: secrets-volume
                 emptyDir: {}
+              - name: restart-script
+                configMap:
+                  name: oidc-proxy-restart-script
 
 Create oauth2-proxy-alertmanager Service:
   metalk8s_kubernetes.object_present:

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls

@@ -59,9 +59,18 @@ Create oauth2-proxy-prometheus Deployment:
                   value: secret
                 - name: UNIQUE_FILENAMES
                   value: "true"
+                - name: SCRIPT
+                  value: /scripts/restart-on-ca-change.sh
+                - name: POD_NAMESPACE
+                  value: metalk8s-monitoring
+                - name: DEPLOYMENT_NAME
+                  value: oauth2-proxy-prometheus
                 volumeMounts:
                 - name: secrets-volume
                   mountPath: /tmp/secrets
+                - name: restart-script
+                  mountPath: /scripts
+                  readOnly: true
               containers:
               - name: oauth2-proxy
                 image: {{ build_image_name("oauth2-proxy") }}
@@ -92,6 +101,9 @@ Create oauth2-proxy-prometheus Deployment:
               volumes:
               - name: secrets-volume
                 emptyDir: {}
+              - name: restart-script
+                configMap:
+                  name: oidc-proxy-restart-script
 
 Create oauth2-proxy-prometheus Service:
   metalk8s_kubernetes.object_present:

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls

@@ -67,6 +67,37 @@ Create oidc-proxy-prometheus-secret-reader-binding RoleBinding in {{ prometheus_
           name: oidc-proxy-prometheus-secret-reader
           apiGroup: rbac.authorization.k8s.io
 
+Create oidc-proxy-prometheus-deployment-restarter Role:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: Role
+        metadata:
+          name: oidc-proxy-prometheus-deployment-restarter
+          namespace: metalk8s-monitoring
+        rules:
+        - apiGroups: ["apps"]
+          resources: ["deployments"]
+          resourceNames: ["oauth2-proxy-prometheus"]
+          verbs: ["get", "patch"]
+
+Create oidc-proxy-prometheus-deployment-restarter-binding RoleBinding:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: RoleBinding
+        metadata:
+          name: oidc-proxy-prometheus-deployment-restarter-binding
+          namespace: metalk8s-monitoring
+        subjects:
+        - kind: ServiceAccount
+          name: oidc-proxy-prometheus
+          namespace: metalk8s-monitoring
+        roleRef:
+          kind: Role
+          name: oidc-proxy-prometheus-deployment-restarter
+          apiGroup: rbac.authorization.k8s.io
+
 {%- else %}
 
 Ensure oidc-proxy-prometheus ServiceAccount does not exist:
@@ -90,6 +121,20 @@ Ensure oidc-proxy-prometheus-secret-reader-binding RoleBinding does not exist in
     - kind: RoleBinding
     - apiVersion: rbac.authorization.k8s.io/v1
 
+Ensure oidc-proxy-prometheus-deployment-restarter Role does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-prometheus-deployment-restarter
+    - namespace: metalk8s-monitoring
+    - kind: Role
+    - apiVersion: rbac.authorization.k8s.io/v1
+
+Ensure oidc-proxy-prometheus-deployment-restarter-binding RoleBinding does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-prometheus-deployment-restarter-binding
+    - namespace: metalk8s-monitoring
+    - kind: RoleBinding
+    - apiVersion: rbac.authorization.k8s.io/v1
+
 {%- endif %}
 
 {%- if alertmanager_oidc_enabled %}
@@ -133,6 +178,37 @@ Create oidc-proxy-alertmanager-secret-reader-binding RoleBinding in {{ alertmana
           name: oidc-proxy-alertmanager-secret-reader
           apiGroup: rbac.authorization.k8s.io
 
+Create oidc-proxy-alertmanager-deployment-restarter Role:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: Role
+        metadata:
+          name: oidc-proxy-alertmanager-deployment-restarter
+          namespace: metalk8s-monitoring
+        rules:
+        - apiGroups: ["apps"]
+          resources: ["deployments"]
+          resourceNames: ["oauth2-proxy-alertmanager"]
+          verbs: ["get", "patch"]
+
+Create oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: RoleBinding
+        metadata:
+          name: oidc-proxy-alertmanager-deployment-restarter-binding
+          namespace: metalk8s-monitoring
+        subjects:
+        - kind: ServiceAccount
+          name: oidc-proxy-alertmanager
+          namespace: metalk8s-monitoring
+        roleRef:
+          kind: Role
+          name: oidc-proxy-alertmanager-deployment-restarter
+          apiGroup: rbac.authorization.k8s.io
+
 {%- else %}
 
 Ensure oidc-proxy-alertmanager ServiceAccount does not exist:
@@ -156,4 +232,18 @@ Ensure oidc-proxy-alertmanager-secret-reader-binding RoleBinding does not exist
     - kind: RoleBinding
     - apiVersion: rbac.authorization.k8s.io/v1
 
+Ensure oidc-proxy-alertmanager-deployment-restarter Role does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-alertmanager-deployment-restarter
+    - namespace: metalk8s-monitoring
+    - kind: Role
+    - apiVersion: rbac.authorization.k8s.io/v1
+
+Ensure oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-alertmanager-deployment-restarter-binding
+    - namespace: metalk8s-monitoring
+    - kind: RoleBinding
+    - apiVersion: rbac.authorization.k8s.io/v1
+
 {%- endif %}

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls

@@ -0,0 +1,61 @@
+{%- set prometheus_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/prometheus-operator/config/prometheus.yaml',
+        saltenv=saltenv
+    )
+%}
+
+{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-monitoring', 'metalk8s-prometheus-config', prometheus_defaults
+    )
+%}
+
+{%- set alertmanager_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/prometheus-operator/config/alertmanager.yaml',
+        saltenv=saltenv
+    )
+%}
+
+{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults
+    )
+%}
+
+{%- set prometheus_oidc_enabled = prometheus.spec.get('config', {}).get('enable_oidc_authentication', False) %}
+{%- set alertmanager_oidc_enabled = alertmanager.spec.get('config', {}).get('enable_oidc_authentication', False) %}
+
+{%- if prometheus_oidc_enabled or alertmanager_oidc_enabled %}
+
+{%- set script_content = salt['cp.get_file_str'](
+        'salt://metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py',
+        saltenv=saltenv
+    )
+%}
+
+Create oidc-proxy-restart-script ConfigMap:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: oidc-proxy-restart-script
+          namespace: metalk8s-monitoring
+          labels:
+            app.kubernetes.io/managed-by: salt
+            app.kubernetes.io/part-of: metalk8s
+            heritage: metalk8s
+        data:
+          restart-on-ca-change.py: |-
+{{ script_content | indent(12, first=True) }}
+          restart-on-ca-change.sh: |
+            python3 /scripts/restart-on-ca-change.py
+
+{%- else %}
+
+Ensure oidc-proxy-restart-script ConfigMap does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-restart-script
+    - namespace: metalk8s-monitoring
+    - kind: ConfigMap
+    - apiVersion: v1
+
+{%- endif %}

salt/tests/unit/formulas/fixtures/salt.py

@@ -412,6 +412,9 @@ def slsutil_renderer(salt_mock: SaltMock, source: str, **_kwargs: Any) -> Any:
 register_basic("file.find")(MagicMock(return_value=[]))
 register_basic("file.join")(lambda *args: "/".join(args))
 register_basic("file.read")(MagicMock(return_value="<file contents>"))
+
+
+register_basic("cp.get_file_str")(MagicMock(return_value="<file contents>"))
 register_basic("hashutil.base64_b64decode")(lambda input_data: input_data)
 register_basic("hashutil.base64_encodefile")(
     MagicMock(return_value="<b64-encoded data>")