Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions buildchain/buildchain/salt_tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,14 @@ def _download_ui_operator_crds() -> str:
Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-chart.sls"),
Path("salt/metalk8s/addons/prometheus-operator/deployed/thanos-query-sd-files.sls"),
Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-rbac.sls"),
Path(
"salt/metalk8s/addons/prometheus-operator/deployed/",
"oidc-proxy-restart-script.sls",
),
Path(
"salt/metalk8s/addons/prometheus-operator/deployed/files/",
"restart-on-ca-change.py",
),
Path("salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-prometheus.sls"),
Path(
"salt/metalk8s/addons/prometheus-operator/deployed/oidc-proxy-alertmanager.sls"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ kind: Secret
metadata:
name: ingress-control-plane-default-certificate
namespace: metalk8s-ingress
labels:
metalk8s.scality.com/oidc-ca: "true"
type: Opaque
data:
tls.crt: "{{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
import hashlib
import os
import sys
from datetime import datetime, timezone
from pathlib import Path

import requests

HASH_FILE_NAME = ".ca-hash-previous"

SA_TOKEN = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
SA_CA = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
K8S_API = "https://kubernetes.default.svc"


def hash_file(file_path: Path) -> str:
h = hashlib.sha256()
h.update(file_path.read_bytes())
return h.hexdigest()


def trigger_restart(namespace: str, deployment: str) -> None:
token = SA_TOKEN.read_text()
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
body = {
"spec": {
"template": {
"metadata": {
"annotations": {"kubectl.kubernetes.io/restartedAt": timestamp}
}
}
}
}
url = f"{K8S_API}/apis/apps/v1/namespaces/{namespace}/deployments/{deployment}"
response = requests.patch(
url,
json=body,
headers={
"Authorization": f"Bearer {token}",
"Content-Type": "application/strategic-merge-patch+json",
},
verify=SA_CA,
)
response.raise_for_status()


def main() -> None:
ca_dir = Path(os.environ["CA_DIR"])
ca_file = ca_dir / os.environ["CA_FILE_NAME"]
hash_file_path = ca_dir / HASH_FILE_NAME

if not ca_file.exists():
print(f"CA file {ca_file} does not exist, skipping")
return

current_hash = hash_file(ca_file)

if not hash_file_path.exists():
hash_file_path.write_text(current_hash)
print("Initial CA load, skipping restart")
return

previous_hash = hash_file_path.read_text().strip()

if current_hash == previous_hash:
return

namespace = os.environ["DEPLOYMENT_NAMESPACE"]
deployment = os.environ["DEPLOYMENT_NAME"]

try:
trigger_restart(namespace, deployment)
except requests.RequestException as e:
print(
f"Failed to trigger restart for {deployment}: {e}",
file=sys.stderr,
)
sys.exit(1)

# Persist hash only after successful restart
hash_file_path.write_text(current_hash)
print(f"Rolling restart triggered for {deployment}")


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ include:
- .thanos-query-sd-files
- .thanos-chart
- .oidc-proxy-rbac
- .oidc-proxy-restart-script
- .oidc-proxy-prometheus
- .oidc-proxy-alertmanager
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,22 @@ Create oauth2-proxy-alertmanager Deployment:
value: secret
- name: UNIQUE_FILENAMES
value: "true"
- name: SCRIPT
value: /scripts/restart-on-ca-change.py
- name: DEPLOYMENT_NAMESPACE
value: metalk8s-monitoring
- name: DEPLOYMENT_NAME
value: oauth2-proxy-alertmanager
- name: CA_DIR
value: /tmp/secrets
- name: CA_FILE_NAME
value: {{ ca_file }}
volumeMounts:
- name: secrets-volume
mountPath: /tmp/secrets
- name: restart-script
mountPath: /scripts
readOnly: true
containers:
- name: oauth2-proxy
image: {{ build_image_name("oauth2-proxy") }}
Expand Down Expand Up @@ -93,6 +106,10 @@ Create oauth2-proxy-alertmanager Deployment:
volumes:
- name: secrets-volume
emptyDir: {}
- name: restart-script
configMap:
name: oidc-proxy-restart-script
defaultMode: "0555"
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same issue: defaultMode should be an integer, not a string. "0555" is interpreted as decimal 555 (octal 01053), not as octal 0555.

— Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same issue: defaultMode should be an integer, not a string.

— Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same defaultMode issue — should be an unquoted decimal integer 365 instead of the string "0555".

— Claude Code


Create oauth2-proxy-alertmanager Service:
metalk8s_kubernetes.object_present:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,22 @@ Create oauth2-proxy-prometheus Deployment:
value: secret
- name: UNIQUE_FILENAMES
value: "true"
- name: SCRIPT
value: /scripts/restart-on-ca-change.py
- name: DEPLOYMENT_NAMESPACE
value: metalk8s-monitoring
- name: DEPLOYMENT_NAME
value: oauth2-proxy-prometheus
- name: CA_DIR
value: /tmp/secrets
- name: CA_FILE_NAME
value: {{ ca_file }}
volumeMounts:
- name: secrets-volume
mountPath: /tmp/secrets
- name: restart-script
mountPath: /scripts
readOnly: true
containers:
- name: oauth2-proxy
image: {{ build_image_name("oauth2-proxy") }}
Expand Down Expand Up @@ -92,6 +105,10 @@ Create oauth2-proxy-prometheus Deployment:
volumes:
- name: secrets-volume
emptyDir: {}
- name: restart-script
configMap:
name: oidc-proxy-restart-script
defaultMode: "0555"
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

defaultMode should be an integer (0555), not a string ("0555"). Kubernetes accepts strings here but interprets them as decimal — "0555" becomes octal 01053 (decimal 555), which is not what you want. Use an unquoted integer instead.

— Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

defaultMode in Kubernetes expects an integer, not a string. 0555 quoted is a YAML string and may be passed as-is to the API, where it could be rejected or misinterpreted. Use the unquoted octal literal 0555 (YAML interprets this as decimal 365) or just 365. The existing thanos-chart.sls uses defaultMode: 420 (integer) as a reference.

— Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

defaultMode expects an integer, not a string. "0555" will be passed as the string "0555" to the Kubernetes API, which expects a decimal integer. Octal 0555 = decimal 365. The rest of the codebase uses unquoted decimal integers (e.g., defaultMode: 420 for 0644).

— Claude Code


Create oauth2-proxy-prometheus Service:
metalk8s_kubernetes.object_present:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,37 @@ Create oidc-proxy-prometheus-secret-reader-binding RoleBinding in {{ prometheus_
name: oidc-proxy-prometheus-secret-reader
apiGroup: rbac.authorization.k8s.io

Create oidc-proxy-prometheus-deployment-restarter Role:
metalk8s_kubernetes.object_present:
- manifest:
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: oidc-proxy-prometheus-deployment-restarter
namespace: metalk8s-monitoring
rules:
- apiGroups: ["apps"]
resources: ["deployments"]
resourceNames: ["oauth2-proxy-prometheus"]
verbs: ["get", "patch"]

Create oidc-proxy-prometheus-deployment-restarter-binding RoleBinding:
metalk8s_kubernetes.object_present:
- manifest:
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: oidc-proxy-prometheus-deployment-restarter-binding
namespace: metalk8s-monitoring
subjects:
- kind: ServiceAccount
name: oidc-proxy-prometheus
namespace: metalk8s-monitoring
roleRef:
kind: Role
name: oidc-proxy-prometheus-deployment-restarter
apiGroup: rbac.authorization.k8s.io

{%- else %}

Ensure oidc-proxy-prometheus ServiceAccount does not exist:
Expand All @@ -90,6 +121,20 @@ Ensure oidc-proxy-prometheus-secret-reader-binding RoleBinding does not exist in
- kind: RoleBinding
- apiVersion: rbac.authorization.k8s.io/v1

Ensure oidc-proxy-prometheus-deployment-restarter Role does not exist:
metalk8s_kubernetes.object_absent:
- name: oidc-proxy-prometheus-deployment-restarter
- namespace: metalk8s-monitoring
- kind: Role
- apiVersion: rbac.authorization.k8s.io/v1

Ensure oidc-proxy-prometheus-deployment-restarter-binding RoleBinding does not exist:
metalk8s_kubernetes.object_absent:
- name: oidc-proxy-prometheus-deployment-restarter-binding
- namespace: metalk8s-monitoring
- kind: RoleBinding
- apiVersion: rbac.authorization.k8s.io/v1

{%- endif %}

{%- if alertmanager_oidc_enabled %}
Expand Down Expand Up @@ -133,6 +178,37 @@ Create oidc-proxy-alertmanager-secret-reader-binding RoleBinding in {{ alertmana
name: oidc-proxy-alertmanager-secret-reader
apiGroup: rbac.authorization.k8s.io

Create oidc-proxy-alertmanager-deployment-restarter Role:
metalk8s_kubernetes.object_present:
- manifest:
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: oidc-proxy-alertmanager-deployment-restarter
namespace: metalk8s-monitoring
rules:
- apiGroups: ["apps"]
resources: ["deployments"]
resourceNames: ["oauth2-proxy-alertmanager"]
verbs: ["get", "patch"]

Create oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding:
metalk8s_kubernetes.object_present:
- manifest:
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: oidc-proxy-alertmanager-deployment-restarter-binding
namespace: metalk8s-monitoring
subjects:
- kind: ServiceAccount
name: oidc-proxy-alertmanager
namespace: metalk8s-monitoring
roleRef:
kind: Role
name: oidc-proxy-alertmanager-deployment-restarter
apiGroup: rbac.authorization.k8s.io

{%- else %}

Ensure oidc-proxy-alertmanager ServiceAccount does not exist:
Expand All @@ -156,4 +232,18 @@ Ensure oidc-proxy-alertmanager-secret-reader-binding RoleBinding does not exist
- kind: RoleBinding
- apiVersion: rbac.authorization.k8s.io/v1

Ensure oidc-proxy-alertmanager-deployment-restarter Role does not exist:
metalk8s_kubernetes.object_absent:
- name: oidc-proxy-alertmanager-deployment-restarter
- namespace: metalk8s-monitoring
- kind: Role
- apiVersion: rbac.authorization.k8s.io/v1

Ensure oidc-proxy-alertmanager-deployment-restarter-binding RoleBinding does not exist:
metalk8s_kubernetes.object_absent:
- name: oidc-proxy-alertmanager-deployment-restarter-binding
- namespace: metalk8s-monitoring
- kind: RoleBinding
- apiVersion: rbac.authorization.k8s.io/v1

{%- endif %}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{%- set prometheus_defaults = salt.slsutil.renderer(
'salt://metalk8s/addons/prometheus-operator/config/prometheus.yaml',
saltenv=saltenv
)
%}

{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf(
'metalk8s-monitoring', 'metalk8s-prometheus-config', prometheus_defaults
)
%}

{%- set alertmanager_defaults = salt.slsutil.renderer(
'salt://metalk8s/addons/prometheus-operator/config/alertmanager.yaml',
saltenv=saltenv
)
%}

{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf(
'metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults
)
%}

{%- set prometheus_oidc_enabled = prometheus.spec.get('config', {}).get('enable_oidc_authentication', False) %}
{%- set alertmanager_oidc_enabled = alertmanager.spec.get('config', {}).get('enable_oidc_authentication', False) %}

{%- if prometheus_oidc_enabled or alertmanager_oidc_enabled %}

{%- set script_content = salt['cp.get_file_str'](
'salt://metalk8s/addons/prometheus-operator/deployed/files/restart-on-ca-change.py',
saltenv=saltenv
)
%}

Create oidc-proxy-restart-script ConfigMap:
metalk8s_kubernetes.object_present:
- manifest:
apiVersion: v1
kind: ConfigMap
metadata:
name: oidc-proxy-restart-script
namespace: metalk8s-monitoring
labels:
app.kubernetes.io/managed-by: salt
app.kubernetes.io/part-of: metalk8s
heritage: metalk8s
data:
restart-on-ca-change.py: |-
{{ script_content | indent(12) }}

{%- else %}

Ensure oidc-proxy-restart-script ConfigMap does not exist:
metalk8s_kubernetes.object_absent:
- name: oidc-proxy-restart-script
- namespace: metalk8s-monitoring
- kind: ConfigMap
- apiVersion: v1

{%- endif %}
1 change: 1 addition & 0 deletions salt/tests/unit/formulas/fixtures/salt.py
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ def slsutil_renderer(salt_mock: SaltMock, source: str, **_kwargs: Any) -> Any:
register_basic("file.find")(MagicMock(return_value=[]))
register_basic("file.join")(lambda *args: "/".join(args))
register_basic("file.read")(MagicMock(return_value="<file contents>"))
register_basic("cp.get_file_str")(MagicMock(return_value="<file contents>"))
register_basic("hashutil.base64_b64decode")(lambda input_data: input_data)
register_basic("hashutil.base64_encodefile")(
MagicMock(return_value="<b64-encoded data>")
Expand Down
Loading
Loading