diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index f3237bde2b..aea4ee3bff 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -408,6 +408,14 @@ def _download_ui_operator_crds() -> str: Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-chart.sls"), Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-query-sd-files.sls"), Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls"), + Path( + "salt/metalk8s/addons/prometheus-operator/deployed/", + "oidc-proxy-restart-script.sls", + ), + Path( + "salt/metalk8s/addons/prometheus-operator/deployed/files/", + "restart-on-ca-change.py", + ), Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls"), Path( "salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls" diff --git a/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls b/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls index d4e6fad3c7..02ec068e3f 100644 --- a/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls +++ b/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/tls-secret.sls @@ -7,6 +7,8 @@ kind: Secret metadata: name: ingress-control-plane-default-certificate namespace: metalk8s-ingress + labels: + metalk8s.scality.com/oidc-ca: "true" type: Opaque data: tls.crt: "{{ diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py b/salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py new file mode 100644 index 0000000000..da10ececef --- /dev/null +++ b/salt/metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +import hashlib +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +import requests + +HASH_FILE_NAME = ".ca-hash-previous" + +SA_TOKEN = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") +SA_CA = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") +K8S_API = "https://kubernetes.default.svc" + + +def hash_file(file_path: Path) -> str: + h = hashlib.sha256() + h.update(file_path.read_bytes()) + return h.hexdigest() + + +def trigger_restart(namespace: str, deployment: str) -> None: + token = SA_TOKEN.read_text() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + body = { + "spec": { + "template": { + "metadata": { + "annotations": {"kubectl.kubernetes.io/restartedAt": timestamp} + } + } + } + } + url = f"{K8S_API}/apis/apps/v1/namespaces/{namespace}/deployments/{deployment}" + response = requests.patch( + url, + json=body, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/strategic-merge-patch+json", + }, + verify=SA_CA, + ) + response.raise_for_status() + + +def main() -> None: + ca_dir = Path(os.environ["CA_DIR"]) + ca_file = ca_dir / os.environ["CA_FILE_NAME"] + hash_file_path = ca_dir / HASH_FILE_NAME + + if not ca_file.exists(): + print(f"CA file {ca_file} does not exist, skipping") + return + + current_hash = hash_file(ca_file) + + if not hash_file_path.exists(): + hash_file_path.write_text(current_hash) + print("Initial CA load, skipping restart") + return + + previous_hash = hash_file_path.read_text().strip() + + if current_hash == previous_hash: + return + + namespace = os.environ["DEPLOYMENT_NAMESPACE"] + deployment = os.environ["DEPLOYMENT_NAME"] + + try: + trigger_restart(namespace, deployment) + except requests.RequestException as e: + print( + f"Failed to trigger restart for {deployment}: {e}", + file=sys.stderr, + ) + sys.exit(1) + + # Persist hash only after successful restart + hash_file_path.write_text(current_hash) + print(f"Rolling restart triggered for {deployment}") + + +if __name__ == "__main__": + main() diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/init.sls b/salt/metalk8s/addons/prometheus-operator/deployed/init.sls index 2ce71ca972..79f287e8be 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/init.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/init.sls @@ -11,5 +11,6 @@ include: - .thanos-query-sd-files - .thanos-chart - .oidc-proxy-rbac + - .oidc-proxy-restart-script - .oidc-proxy-prometheus - .oidc-proxy-alertmanager diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls index b8bb48cb9a..b8b3d5d58e 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls @@ -60,9 +60,22 @@ Create oauth2-proxy-alertmanager Deployment: value: secret - name: UNIQUE_FILENAMES value: "true" + - name: SCRIPT + value: /scripts/restart-on-ca-change.py + - name: DEPLOYMENT_NAMESPACE + value: metalk8s-monitoring + - name: DEPLOYMENT_NAME + value: oauth2-proxy-alertmanager + - name: CA_DIR + value: /tmp/secrets + - name: CA_FILE_NAME + value: {{ ca_file }} volumeMounts: - name: secrets-volume mountPath: /tmp/secrets + - name: restart-script + mountPath: /scripts + readOnly: true containers: - name: oauth2-proxy image: {{ build_image_name("oauth2-proxy") }} @@ -93,6 +106,10 @@ Create oauth2-proxy-alertmanager Deployment: volumes: - name: secrets-volume emptyDir: {} + - name: restart-script + configMap: + name: oidc-proxy-restart-script + defaultMode: "0555" Create oauth2-proxy-alertmanager Service: metalk8s_kubernetes.object_present: diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls index 22252a42ba..e80123c84b 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls @@ -59,9 +59,22 @@ Create oauth2-proxy-prometheus Deployment: value: secret - name: UNIQUE_FILENAMES value: "true" + - name: SCRIPT + value: /scripts/restart-on-ca-change.py + - name: DEPLOYMENT_NAMESPACE + value: metalk8s-monitoring + - name: DEPLOYMENT_NAME + value: oauth2-proxy-prometheus + - name: CA_DIR + value: /tmp/secrets + - name: CA_FILE_NAME + value: {{ ca_file }} volumeMounts: - name: secrets-volume mountPath: /tmp/secrets + - name: restart-script + mountPath: /scripts + readOnly: true containers: - name: oauth2-proxy image: {{ build_image_name("oauth2-proxy") }} @@ -92,6 +105,10 @@ Create oauth2-proxy-prometheus Deployment: volumes: - name: secrets-volume emptyDir: {} + - name: restart-script + configMap: + name: oidc-proxy-restart-script + defaultMode: "0555" Create oauth2-proxy-prometheus Service: metalk8s_kubernetes.object_present: diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls index 92530c10f0..59ee07deca 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls @@ -67,6 +67,37 @@ Create oidc-proxy-prometheus-secret-reader-binding RoleBinding in {{ prometheus_ name: oidc-proxy-prometheus-secret-reader apiGroup: rbac.authorization.k8s.io +Create oidc-proxy-prometheus-deployment-restarter Role: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: oidc-proxy-prometheus-deployment-restarter + namespace: metalk8s-monitoring + rules: + - apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["oauth2-proxy-prometheus"] + verbs: ["get", "patch"] + +Create oidc-proxy-prometheus-deployment-restarter-binding RoleBinding: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: oidc-proxy-prometheus-deployment-restarter-binding + namespace: metalk8s-monitoring + subjects: + - kind: ServiceAccount + name: oidc-proxy-prometheus + namespace: metalk8s-monitoring + roleRef: + kind: Role + name: oidc-proxy-prometheus-deployment-restarter + apiGroup: rbac.authorization.k8s.io + {%- else %} Ensure oidc-proxy-prometheus ServiceAccount does not exist: @@ -90,6 +121,20 @@ Ensure oidc-proxy-prometheus-secret-reader-binding RoleBinding does not exist in - kind: RoleBinding - apiVersion: rbac.authorization.k8s.io/v1 +Ensure oidc-proxy-prometheus-deployment-restarter Role does not exist: + metalk8s_kubernetes.object_absent: + - name: oidc-proxy-prometheus-deployment-restarter + - namespace: metalk8s-monitoring + - kind: Role + - apiVersion: rbac.authorization.k8s.io/v1 + +Ensure oidc-proxy-prometheus-deployment-restarter-binding RoleBinding does not exist: + metalk8s_kubernetes.object_absent: + - name: oidc-proxy-prometheus-deployment-restarter-binding + - namespace: metalk8s-monitoring + - kind: RoleBinding + - apiVersion: rbac.authorization.k8s.io/v1 + {%- endif %} {%- if alertmanager_oidc_enabled %} @@ -133,6 +178,37 @@ Create oidc-proxy-alertmanager-secret-reader-binding RoleBinding in {{ alertmana name: oidc-proxy-alertmanager-secret-reader apiGroup: rbac.authorization.k8s.io +Create oidc-proxy-alertmanager-deployment-restarter Role: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: oidc-proxy-alertmanager-deployment-restarter + namespace: metalk8s-monitoring + rules: + - apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["oauth2-proxy-alertmanager"] + verbs: ["get", "patch"] + +Create oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: oidc-proxy-alertmanager-deployment-restarter-binding + namespace: metalk8s-monitoring + subjects: + - kind: ServiceAccount + name: oidc-proxy-alertmanager + namespace: metalk8s-monitoring + roleRef: + kind: Role + name: oidc-proxy-alertmanager-deployment-restarter + apiGroup: rbac.authorization.k8s.io + {%- else %} Ensure oidc-proxy-alertmanager ServiceAccount does not exist: @@ -156,4 +232,18 @@ Ensure oidc-proxy-alertmanager-secret-reader-binding RoleBinding does not exist - kind: RoleBinding - apiVersion: rbac.authorization.k8s.io/v1 +Ensure oidc-proxy-alertmanager-deployment-restarter Role does not exist: + metalk8s_kubernetes.object_absent: + - name: oidc-proxy-alertmanager-deployment-restarter + - namespace: metalk8s-monitoring + - kind: Role + - apiVersion: rbac.authorization.k8s.io/v1 + +Ensure oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding does not exist: + metalk8s_kubernetes.object_absent: + - name: oidc-proxy-alertmanager-deployment-restarter-binding + - namespace: metalk8s-monitoring + - kind: RoleBinding + - apiVersion: rbac.authorization.k8s.io/v1 + {%- endif %} diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls new file mode 100644 index 0000000000..1edd868e8d --- /dev/null +++ b/salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-restart-script.sls @@ -0,0 +1,59 @@ +{%- set prometheus_defaults = salt.slsutil.renderer( + 'salt://metalk8s/addons/prometheus-operator/config/prometheus.yaml', + saltenv=saltenv + ) +%} + +{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf( + 'metalk8s-monitoring', 'metalk8s-prometheus-config', prometheus_defaults + ) +%} + +{%- set alertmanager_defaults = salt.slsutil.renderer( + 'salt://metalk8s/addons/prometheus-operator/config/alertmanager.yaml', + saltenv=saltenv + ) +%} + +{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf( + 'metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults + ) +%} + +{%- set prometheus_oidc_enabled = prometheus.spec.get('config', {}).get('enable_oidc_authentication', False) %} +{%- set alertmanager_oidc_enabled = alertmanager.spec.get('config', {}).get('enable_oidc_authentication', False) %} + +{%- if prometheus_oidc_enabled or alertmanager_oidc_enabled %} + +{%- set script_content = salt['cp.get_file_str']( + 'salt://metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py', + saltenv=saltenv + ) +%} + +Create oidc-proxy-restart-script ConfigMap: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: oidc-proxy-restart-script + namespace: metalk8s-monitoring + labels: + app.kubernetes.io/managed-by: salt + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s + data: + restart-on-ca-change.py: |- + {{ script_content | indent(12) }} + +{%- else %} + +Ensure oidc-proxy-restart-script ConfigMap does not exist: + metalk8s_kubernetes.object_absent: + - name: oidc-proxy-restart-script + - namespace: metalk8s-monitoring + - kind: ConfigMap + - apiVersion: v1 + +{%- endif %} diff --git a/salt/tests/unit/formulas/fixtures/salt.py b/salt/tests/unit/formulas/fixtures/salt.py index d95c004281..695133ba14 100644 --- a/salt/tests/unit/formulas/fixtures/salt.py +++ b/salt/tests/unit/formulas/fixtures/salt.py @@ -412,6 +412,7 @@ def slsutil_renderer(salt_mock: SaltMock, source: str, **_kwargs: Any) -> Any: register_basic("file.find")(MagicMock(return_value=[])) register_basic("file.join")(lambda *args: "/".join(args)) register_basic("file.read")(MagicMock(return_value="")) +register_basic("cp.get_file_str")(MagicMock(return_value="")) register_basic("hashutil.base64_b64decode")(lambda input_data: input_data) register_basic("hashutil.base64_encodefile")( MagicMock(return_value="") diff --git a/salt/tests/unit/scripts/__init__.py b/salt/tests/unit/scripts/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/salt/tests/unit/scripts/__init__.py @@ -0,0 +1 @@ + diff --git a/salt/tests/unit/scripts/test_restart_on_ca_change.py b/salt/tests/unit/scripts/test_restart_on_ca_change.py new file mode 100644 index 0000000000..6e0ba46da9 --- /dev/null +++ b/salt/tests/unit/scripts/test_restart_on_ca_change.py @@ -0,0 +1,203 @@ +"""Tests for the restart-on-ca-change.py script.""" + +import importlib.util +import os +import tempfile +from pathlib import Path + +import requests +from unittest import TestCase +from unittest.mock import patch + +# The script has a hyphenated filename, so we need importlib to load it +_SCRIPT_PATH = ( + Path(__file__).resolve().parents[3] + / "metalk8s" + / "addons" + / "prometheus-operator" + / "deployed" + / "files" + / "restart-on-ca-change.py" +) +_spec = importlib.util.spec_from_file_location("restart_on_ca_change", _SCRIPT_PATH) +restart_on_ca_change = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(restart_on_ca_change) + +ENV_VARS = { + "CA_DIR": "/tmp/secrets", + "CA_FILE_NAME": "ca.crt", + "DEPLOYMENT_NAMESPACE": "metalk8s-monitoring", + "DEPLOYMENT_NAME": "oauth2-proxy-prometheus", +} + + +class TestHashFile(TestCase): + """Tests for hash_file function.""" + + def test_returns_sha256_hex(self): + """hash_file returns a 64-char hex string.""" + with tempfile.NamedTemporaryFile() as f: + f.write(b"cert-data") + f.flush() + result = restart_on_ca_change.hash_file(Path(f.name)) + self.assertIsInstance(result, str) + self.assertEqual(len(result), 64) + + def test_different_content_different_hash(self): + """hash_file returns different hashes for different file contents.""" + with tempfile.NamedTemporaryFile() as f1, tempfile.NamedTemporaryFile() as f2: + f1.write(b"cert-v1") + f1.flush() + hash_v1 = restart_on_ca_change.hash_file(Path(f1.name)) + + f2.write(b"cert-v2") + f2.flush() + hash_v2 = restart_on_ca_change.hash_file(Path(f2.name)) + + self.assertNotEqual(hash_v1, hash_v2) + + def test_same_content_same_hash(self): + """hash_file returns the same hash for the same content.""" + with tempfile.NamedTemporaryFile() as f1, tempfile.NamedTemporaryFile() as f2: + f1.write(b"cert-data") + f1.flush() + hash_1 = restart_on_ca_change.hash_file(Path(f1.name)) + + f2.write(b"cert-data") + f2.flush() + hash_2 = restart_on_ca_change.hash_file(Path(f2.name)) + + self.assertEqual(hash_1, hash_2) + + +@patch.dict(os.environ, ENV_VARS) +class TestMain(TestCase): + """Tests for main function.""" + + def test_ca_file_missing_skips(self): + """main skips when CA file does not exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print") as mock_print: + restart_on_ca_change.main() + mock_print.assert_called_once_with( + f"CA file {Path(tmpdir) / 'ca.crt'} does not exist, skipping" + ) + + def test_initial_load_skips_restart(self): + """main writes hash and skips restart on initial load.""" + with tempfile.TemporaryDirectory() as tmpdir: + ca_file = Path(tmpdir) / "ca.crt" + ca_file.write_bytes(b"cert-data") + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print") as mock_print: + restart_on_ca_change.main() + mock_print.assert_called_once_with("Initial CA load, skipping restart") + # Hash file should have been created + hash_file = Path(tmpdir) / ".ca-hash-previous" + self.assertTrue(hash_file.exists()) + + def test_unchanged_hash_no_restart(self): + """main does nothing when hash has not changed.""" + with tempfile.TemporaryDirectory() as tmpdir: + ca_file = Path(tmpdir) / "ca.crt" + ca_file.write_bytes(b"cert-data") + # Pre-compute and write the hash + current_hash = restart_on_ca_change.hash_file(ca_file) + hash_file = Path(tmpdir) / ".ca-hash-previous" + hash_file.write_text(current_hash) + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print") as mock_print: + restart_on_ca_change.main() + mock_print.assert_not_called() + + @patch.object(restart_on_ca_change, "trigger_restart") + def test_changed_hash_triggers_restart(self, mock_restart): + """main triggers restart when hash has changed.""" + with tempfile.TemporaryDirectory() as tmpdir: + ca_file = Path(tmpdir) / "ca.crt" + ca_file.write_bytes(b"cert-data") + hash_file = Path(tmpdir) / ".ca-hash-previous" + hash_file.write_text("old-hash") + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print") as mock_print: + restart_on_ca_change.main() + + mock_restart.assert_called_once_with( + "metalk8s-monitoring", "oauth2-proxy-prometheus" + ) + self.assertIn( + "Rolling restart triggered", + mock_print.call_args[0][0], + ) + + @patch.object( + restart_on_ca_change, + "trigger_restart", + side_effect=requests.RequestException("refused"), + ) + def test_api_failure_exits_with_error(self, _mock_restart): + """main exits with code 1 when the API call fails.""" + with tempfile.TemporaryDirectory() as tmpdir: + ca_file = Path(tmpdir) / "ca.crt" + ca_file.write_bytes(b"cert-data") + hash_file = Path(tmpdir) / ".ca-hash-previous" + hash_file.write_text("old-hash") + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print"): + with self.assertRaises(SystemExit) as ctx: + restart_on_ca_change.main() + + self.assertEqual(ctx.exception.code, 1) + + @patch.object( + restart_on_ca_change, + "trigger_restart", + side_effect=requests.RequestException("refused"), + ) + def test_hash_not_persisted_on_api_failure(self, _mock_restart): + """Hash file is not updated when the API call fails.""" + with tempfile.TemporaryDirectory() as tmpdir: + ca_file = Path(tmpdir) / "ca.crt" + ca_file.write_bytes(b"cert-data") + hash_file = Path(tmpdir) / ".ca-hash-previous" + hash_file.write_text("old-hash") + env = {**ENV_VARS, "CA_DIR": tmpdir} + with patch.dict(os.environ, env), patch("builtins.print"): + try: + restart_on_ca_change.main() + except SystemExit: + pass + + # Hash file should still contain the old hash + self.assertEqual(hash_file.read_text(), "old-hash") + + +class TestTriggerRestart(TestCase): + """Tests for trigger_restart function.""" + + @patch.object(restart_on_ca_change.requests, "patch") + def test_sends_patch_request(self, mock_patch): + """trigger_restart sends a PATCH to the K8s API.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".token") as f: + f.write("fake-token") + f.flush() + with patch.object(restart_on_ca_change, "SA_TOKEN", Path(f.name)): + restart_on_ca_change.trigger_restart( + "metalk8s-monitoring", "oauth2-proxy-prometheus" + ) + + mock_patch.assert_called_once() + args, kwargs = mock_patch.call_args + self.assertIn( + "/namespaces/metalk8s-monitoring/deployments/oauth2-proxy-prometheus", + args[0], + ) + self.assertEqual( + kwargs["headers"]["Content-Type"], + "application/strategic-merge-patch+json", + ) + self.assertIn( + "Bearer fake-token", + kwargs["headers"]["Authorization"], + )