MK8S-184 - Add restart script to react when control plane ingress change

buildchain/buildchain/salt_tree.py

@@ -408,6 +408,14 @@ def _download_ui_operator_crds() -> str:
     Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-chart.sls"),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-query-sd-files.sls"),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls"),
+    Path(
+        "salt/metalk8s/addons/prometheus-operator/deployed/",
+        "oidc-proxy-restart-script.sls",
+    ),
+    Path(
+        "salt/metalk8s/addons/prometheus-operator/deployed/files/",
+        "restart-on-ca-change.py",
+    ),
     Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls"),
     Path(
         "salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls"

salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls

@@ -7,6 +7,8 @@ kind: Secret
 metadata:
   name: ingress-control-plane-default-certificate
   namespace: metalk8s-ingress
+  labels:
+    metalk8s.scality.com/oidc-ca: "true"
 type: Opaque
 data:
   tls.crt: "{{

salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+import hashlib
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import requests
+
+HASH_FILE_NAME = ".ca-hash-previous"
+
+SA_TOKEN = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
+SA_CA = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+K8S_API = "https://kubernetes.default.svc"
+
+
+def hash_file(file_path: Path) -> str:
+    h = hashlib.sha256()
+    h.update(file_path.read_bytes())
+    return h.hexdigest()
+
+
+def trigger_restart(namespace: str, deployment: str) -> None:
+    token = SA_TOKEN.read_text()
+    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    body = {
+        "spec": {
+            "template": {
+                "metadata": {
+                    "annotations": {"kubectl.kubernetes.io/restartedAt": timestamp}
+                }
+            }
+        }
+    }
+    url = f"{K8S_API}/apis/apps/v1/namespaces/{namespace}/deployments/{deployment}"
+    response = requests.patch(
+        url,
+        json=body,
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/strategic-merge-patch+json",
+        },
+        verify=SA_CA,
+    )
+    response.raise_for_status()
+
+
+def main() -> None:
+    ca_dir = Path(os.environ["CA_DIR"])
+    ca_file = ca_dir / os.environ["CA_FILE_NAME"]
+    hash_file_path = ca_dir / HASH_FILE_NAME
+
+    if not ca_file.exists():
+        print(f"CA file {ca_file} does not exist, skipping")
+        return
+
+    current_hash = hash_file(ca_file)
+
+    if not hash_file_path.exists():
+        hash_file_path.write_text(current_hash)
+        print("Initial CA load, skipping restart")
+        return
+
+    previous_hash = hash_file_path.read_text().strip()
+
+    if current_hash == previous_hash:
+        return
+
+    namespace = os.environ["DEPLOYMENT_NAMESPACE"]
+    deployment = os.environ["DEPLOYMENT_NAME"]
+
+    try:
+        trigger_restart(namespace, deployment)
+    except requests.RequestException as e:
+        print(
+            f"Failed to trigger restart for {deployment}: {e}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    # Persist hash only after successful restart
+    hash_file_path.write_text(current_hash)
+    print(f"Rolling restart triggered for {deployment}")
+
+
+if __name__ == "__main__":
+    main()

salt/metalk8s/addons/prometheus-operator/deployed/init.sls

@@ -11,5 +11,6 @@ include:
   - .thanos-query-sd-files
   - .thanos-chart
   - .oidc-proxy-rbac
+  - .oidc-proxy-restart-script
   - .oidc-proxy-prometheus
   - .oidc-proxy-alertmanager

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls

@@ -60,9 +60,22 @@ Create oauth2-proxy-alertmanager Deployment:
                   value: secret
                 - name: UNIQUE_FILENAMES
                   value: "true"
+                - name: SCRIPT
+                  value: /scripts/restart-on-ca-change.py
+                - name: DEPLOYMENT_NAMESPACE
+                  value: metalk8s-monitoring
+                - name: DEPLOYMENT_NAME
+                  value: oauth2-proxy-alertmanager
+                - name: CA_DIR
+                  value: /tmp/secrets
+                - name: CA_FILE_NAME
+                  value: {{ ca_file }}
                 volumeMounts:
                 - name: secrets-volume
                   mountPath: /tmp/secrets
+                - name: restart-script
+                  mountPath: /scripts
+                  readOnly: true
               containers:
               - name: oauth2-proxy
                 image: {{ build_image_name("oauth2-proxy") }}
@@ -93,6 +106,10 @@ Create oauth2-proxy-alertmanager Deployment:
               volumes:
               - name: secrets-volume
                 emptyDir: {}
+              - name: restart-script
+                configMap:
+                  name: oidc-proxy-restart-script
+                  defaultMode: "0555"
 
 Create oauth2-proxy-alertmanager Service:
   metalk8s_kubernetes.object_present:

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls

@@ -59,9 +59,22 @@ Create oauth2-proxy-prometheus Deployment:
                   value: secret
                 - name: UNIQUE_FILENAMES
                   value: "true"
+                - name: SCRIPT
+                  value: /scripts/restart-on-ca-change.py
+                - name: DEPLOYMENT_NAMESPACE
+                  value: metalk8s-monitoring
+                - name: DEPLOYMENT_NAME
+                  value: oauth2-proxy-prometheus
+                - name: CA_DIR
+                  value: /tmp/secrets
+                - name: CA_FILE_NAME
+                  value: {{ ca_file }}
                 volumeMounts:
                 - name: secrets-volume
                   mountPath: /tmp/secrets
+                - name: restart-script
+                  mountPath: /scripts
+                  readOnly: true
               containers:
               - name: oauth2-proxy
                 image: {{ build_image_name("oauth2-proxy") }}
@@ -92,6 +105,10 @@ Create oauth2-proxy-prometheus Deployment:
               volumes:
               - name: secrets-volume
                 emptyDir: {}
+              - name: restart-script
+                configMap:
+                  name: oidc-proxy-restart-script
+                  defaultMode: "0555"
 
 Create oauth2-proxy-prometheus Service:
   metalk8s_kubernetes.object_present:

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls

@@ -67,6 +67,37 @@ Create oidc-proxy-prometheus-secret-reader-binding RoleBinding in {{ prometheus_
           name: oidc-proxy-prometheus-secret-reader
           apiGroup: rbac.authorization.k8s.io
 
+Create oidc-proxy-prometheus-deployment-restarter Role:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: Role
+        metadata:
+          name: oidc-proxy-prometheus-deployment-restarter
+          namespace: metalk8s-monitoring
+        rules:
+        - apiGroups: ["apps"]
+          resources: ["deployments"]
+          resourceNames: ["oauth2-proxy-prometheus"]
+          verbs: ["get", "patch"]
+
+Create oidc-proxy-prometheus-deployment-restarter-binding RoleBinding:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: RoleBinding
+        metadata:
+          name: oidc-proxy-prometheus-deployment-restarter-binding
+          namespace: metalk8s-monitoring
+        subjects:
+        - kind: ServiceAccount
+          name: oidc-proxy-prometheus
+          namespace: metalk8s-monitoring
+        roleRef:
+          kind: Role
+          name: oidc-proxy-prometheus-deployment-restarter
+          apiGroup: rbac.authorization.k8s.io
+
 {%- else %}
 
 Ensure oidc-proxy-prometheus ServiceAccount does not exist:
@@ -90,6 +121,20 @@ Ensure oidc-proxy-prometheus-secret-reader-binding RoleBinding does not exist in
     - kind: RoleBinding
     - apiVersion: rbac.authorization.k8s.io/v1
 
+Ensure oidc-proxy-prometheus-deployment-restarter Role does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-prometheus-deployment-restarter
+    - namespace: metalk8s-monitoring
+    - kind: Role
+    - apiVersion: rbac.authorization.k8s.io/v1
+
+Ensure oidc-proxy-prometheus-deployment-restarter-binding RoleBinding does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-prometheus-deployment-restarter-binding
+    - namespace: metalk8s-monitoring
+    - kind: RoleBinding
+    - apiVersion: rbac.authorization.k8s.io/v1
+
 {%- endif %}
 
 {%- if alertmanager_oidc_enabled %}
@@ -133,6 +178,37 @@ Create oidc-proxy-alertmanager-secret-reader-binding RoleBinding in {{ alertmana
           name: oidc-proxy-alertmanager-secret-reader
           apiGroup: rbac.authorization.k8s.io
 
+Create oidc-proxy-alertmanager-deployment-restarter Role:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: Role
+        metadata:
+          name: oidc-proxy-alertmanager-deployment-restarter
+          namespace: metalk8s-monitoring
+        rules:
+        - apiGroups: ["apps"]
+          resources: ["deployments"]
+          resourceNames: ["oauth2-proxy-alertmanager"]
+          verbs: ["get", "patch"]
+
+Create oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: RoleBinding
+        metadata:
+          name: oidc-proxy-alertmanager-deployment-restarter-binding
+          namespace: metalk8s-monitoring
+        subjects:
+        - kind: ServiceAccount
+          name: oidc-proxy-alertmanager
+          namespace: metalk8s-monitoring
+        roleRef:
+          kind: Role
+          name: oidc-proxy-alertmanager-deployment-restarter
+          apiGroup: rbac.authorization.k8s.io
+
 {%- else %}
 
 Ensure oidc-proxy-alertmanager ServiceAccount does not exist:
@@ -156,4 +232,18 @@ Ensure oidc-proxy-alertmanager-secret-reader-binding RoleBinding does not exist
     - kind: RoleBinding
     - apiVersion: rbac.authorization.k8s.io/v1
 
+Ensure oidc-proxy-alertmanager-deployment-restarter Role does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-alertmanager-deployment-restarter
+    - namespace: metalk8s-monitoring
+    - kind: Role
+    - apiVersion: rbac.authorization.k8s.io/v1
+
+Ensure oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-alertmanager-deployment-restarter-binding
+    - namespace: metalk8s-monitoring
+    - kind: RoleBinding
+    - apiVersion: rbac.authorization.k8s.io/v1
+
 {%- endif %}

salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls

@@ -0,0 +1,59 @@
+{%- set prometheus_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/prometheus-operator/config/prometheus.yaml',
+        saltenv=saltenv
+    )
+%}
+
+{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-monitoring', 'metalk8s-prometheus-config', prometheus_defaults
+    )
+%}
+
+{%- set alertmanager_defaults = salt.slsutil.renderer(
+        'salt://metalk8s/addons/prometheus-operator/config/alertmanager.yaml',
+        saltenv=saltenv
+    )
+%}
+
+{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf(
+        'metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults
+    )
+%}
+
+{%- set prometheus_oidc_enabled = prometheus.spec.get('config', {}).get('enable_oidc_authentication', False) %}
+{%- set alertmanager_oidc_enabled = alertmanager.spec.get('config', {}).get('enable_oidc_authentication', False) %}
+
+{%- if prometheus_oidc_enabled or alertmanager_oidc_enabled %}
+
+{%- set script_content = salt['cp.get_file_str'](
+        'salt://metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py',
+        saltenv=saltenv
+    )
+%}
+
+Create oidc-proxy-restart-script ConfigMap:
+  metalk8s_kubernetes.object_present:
+    - manifest:
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: oidc-proxy-restart-script
+          namespace: metalk8s-monitoring
+          labels:
+            app.kubernetes.io/managed-by: salt
+            app.kubernetes.io/part-of: metalk8s
+            heritage: metalk8s
+        data:
+          restart-on-ca-change.py: |-
+            {{ script_content | indent(12) }}
+
+{%- else %}
+
+Ensure oidc-proxy-restart-script ConfigMap does not exist:
+  metalk8s_kubernetes.object_absent:
+    - name: oidc-proxy-restart-script
+    - namespace: metalk8s-monitoring
+    - kind: ConfigMap
+    - apiVersion: v1
+
+{%- endif %}

salt/tests/unit/formulas/fixtures/salt.py

@@ -412,6 +412,7 @@ def slsutil_renderer(salt_mock: SaltMock, source: str, **_kwargs: Any) -> Any:
 register_basic("file.find")(MagicMock(return_value=[]))
 register_basic("file.join")(lambda *args: "/".join(args))
 register_basic("file.read")(MagicMock(return_value="<file contents>"))
+register_basic("cp.get_file_str")(MagicMock(return_value="<file contents>"))
 register_basic("hashutil.base64_b64decode")(lambda input_data: input_data)
 register_basic("hashutil.base64_encodefile")(
     MagicMock(return_value="<b64-encoded data>")

salt/tests/unit/scripts/__init__.py

@@ -0,0 +1 @@
+