add ValidatingWebhook for preventing attachment of mantle-related PVs

.github/workflows/e2e-multiple-k8s-clusters.yaml

@@ -41,7 +41,7 @@ jobs:
             ginkgo-flags: --label-filter="various-transfer-part-size"
     #runs-on: "ubuntu-22.04"
     runs-on: mantle_large_runner_16core
-    timeout-minutes: 60
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5

charts/mantle/templates/_helpers.tpl

@@ -60,3 +60,25 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Generate certificates for webhook
+*/}}
+{{- define "mantle.webhookCerts" -}}
+{{- if eq .Values.webhook.certificate.managedBy "helm" }}
+{{- $serviceName := printf "%s-webhook" (include "mantle.fullname" .) -}}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace $serviceName -}}
+{{- if $secret -}}
+caCert: {{ index $secret.data "ca.crt" }}
+clientCert: {{ index $secret.data "tls.crt" }}
+clientKey: {{ index $secret.data "tls.key" }}
+{{- else -}}
+{{- $altNames := list (printf "%s.%s" $serviceName .Release.Namespace) (printf "%s.%s.svc" $serviceName .Release.Namespace) (printf "%s.%s.svc.%s" $serviceName .Release.Namespace .Values.webhook.certificate.dnsDomain) -}}
+{{- $ca := genCA "mantle-ca" 3650 -}}
+{{- $cert := genSignedCert $serviceName nil $altNames 3650 $ca -}}
+caCert: {{ $ca.Cert | b64enc }}
+clientCert: {{ $cert.Cert | b64enc }}
+clientKey: {{ $cert.Key | b64enc }}
+{{- end -}}
+{{- end -}}
+{{- end -}}

charts/mantle/templates/certificate.yaml

@@ -0,0 +1,32 @@
+{{- if eq .Values.controller.role "secondary" }}
+{{- if .Values.webhook.enabled }}
+{{- if eq .Values.webhook.certificate.managedBy "cert-manager" }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "mantle.fullname" . }}-webhook
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "mantle.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  secretName: {{ template "mantle.fullname" . }}-webhook
+  duration: 8760h # 1y
+  issuerRef:
+    {{- with .Values.webhook.certificate.existingCertManagerIssuer }}
+    {{- toYaml . | nindent 4 -}}
+    {{- end }}
+  dnsNames:
+    - {{ template "mantle.fullname" . }}-webhook
+    - {{ template "mantle.fullname" . }}-webhook.{{ .Release.Namespace }}
+    - {{ template "mantle.fullname" . }}-webhook.{{ .Release.Namespace }}.svc
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+    - client auth
+{{- end }}
+{{- end }}
+{{- end }}

charts/mantle/templates/deployment.yaml

@@ -61,12 +61,6 @@ spec:
             periodSeconds: 10
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
-          volumeMounts:
-            - mountPath: /etc/ceph
-              name: ceph-config
-            {{- with .Values.controller.volumeMounts }}
-            {{- toYaml . | nindent 12 }}
-            {{- end }}
           command:
             - /manager
           args:
@@ -112,23 +106,31 @@ spec:
             {{- with .Values.controller.backupTransferPartSize }}
             - --backup-transfer-part-size={{ . }}
             {{- end }}
-            {{- with .Values.controller.grpcTLSClientCertPath }}
-            - --grpc-tls-client-cert-path={{ . }}
-            {{- end }}
-            {{- with .Values.controller.grpcTLSClientKeyPath }}
-            - --grpc-tls-client-key-path={{ . }}
-            {{- end }}
-            {{- with .Values.controller.grpcTLSClientCAPath }}
-            - --grpc-tls-client-ca-path={{ . }}
-            {{- end }}
-            {{- with .Values.controller.grpcTLSServerCertPath }}
-            - --grpc-tls-server-cert-path={{ . }}
-            {{- end }}
-            {{- with .Values.controller.grpcTLSServerKeyPath }}
-            - --grpc-tls-server-key-path={{ . }}
+
+            {{- if eq .Values.controller.role "primary" }}
+            {{-   with .Values.controller.replicationTLSClientCertPath }}
+            - --replication-tls-client-cert-path={{ . }}
+            {{-   end }}
+            {{-   with .Values.controller.replicationTLSClientKeyPath }}
+            - --replication-tls-client-key-path={{ . }}
+            {{-   end }}
+            {{-   with .Values.controller.replicationTLSServerCAPath }}
+            - --replication-tls-server-ca-path={{ . }}
+            {{-   end }}
             {{- end }}
-            {{- with .Values.controller.grpcTLSServerCAPath }}
-            - --grpc-tls-server-ca-path={{ . }}
+
+            {{- if eq .Values.controller.role "secondary" }}
+            {{-   with .Values.controller.replicationTLSServerCertPath }}
+            - --replication-tls-server-cert-path={{ . }}
+            {{-   end }}
+            {{-   with .Values.controller.replicationTLSServerKeyPath }}
+            - --replication-tls-server-key-path={{ . }}
+            {{-   end }}
+            {{-   with .Values.controller.replicationTLSClientCAPath }}
+            - --replication-tls-client-ca-path={{ . }}
+            {{-   end }}
+            - --webhook-cert-path=/mnt/webhook-tls-cert/tls.crt
+            - --webhook-key-path=/mnt/webhook-tls-cert/tls.key
             {{- end }}
           env:
             - name: POD_NAME
@@ -146,6 +148,16 @@ spec:
             {{- end }}
           ports:
             {{- toYaml .Values.controller.ports | nindent 12 }}
+          volumeMounts:
+            - name: ceph-config
+              mountPath: /etc/ceph
+            {{- if eq .Values.controller.role "secondary" }}
+            - name: webhook-tls-cert
+              mountPath: /mnt/webhook-tls-cert
+            {{- end }}
+            {{- with .Values.controller.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
         - command:
             - /bin/bash
             - -c
@@ -250,6 +262,11 @@ spec:
           name: mon-endpoint-volume
         - emptyDir: {}
           name: ceph-config
+        {{- if eq .Values.controller.role "secondary" }}
+        - name: webhook-tls-cert
+          secret:
+            secretName: {{ template "mantle.fullname" . }}-webhook
+        {{- end }}
         {{- with .Values.controller.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/mantle/templates/service.yaml

@@ -2,15 +2,34 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "mantle.fullname" . }}
+  name: {{ include "mantle.fullname" . }}-replication
   namespace: {{ .Release.Namespace }}
 spec:
-  type: {{ .Values.secondaryService.type }}
+  type: {{ .Values.replicationService.type }}
   ports:
-  - port: {{ .Values.secondaryService.port }}
-    targetPort: {{ .Values.secondaryService.targetPort }}
+  - port: {{ .Values.replicationService.port }}
+    targetPort: {{ .Values.replicationService.targetPort }}
     protocol: TCP
-    name: grpc
+    name: replication
+    {{- with .Values.replicationService.nodePort }}
+    nodePort: {{ . }}
+    {{- end }}
   selector:
     {{- include "mantle.selectorLabels" . | nindent 4 }}
+{{- if .Values.webhook.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "mantle.fullname" . }}-webhook
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - port: {{ .Values.webhook.service.port }}
+    targetPort: {{ .Values.webhook.service.targetPort }}
+    protocol: TCP
+    name: webhook
+  selector:
+    {{- include "mantle.selectorLabels" . | nindent 4 }}
+{{ end }}
 {{ end }}

charts/mantle/templates/webhook.yaml

@@ -0,0 +1,49 @@
+{{- $tls := fromYaml ( include "mantle.webhookCerts" . ) }}
+---
+{{- if eq .Values.controller.role "secondary" }}
+{{- if .Values.webhook.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: {{ include "mantle.fullname" . }}-validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    {{- if eq .Values.webhook.certificate.managedBy "helm" }}
+    caBundle: {{ $tls.caCert }}
+    {{- end }}
+    service:
+      name: {{ include "mantle.fullname" . }}-webhook
+      namespace: {{ .Release.Namespace }}
+      path: /validate-storage-k8s-io-v1-volumeattachment
+      port: {{ .Values.webhook.service.port }}
+  failurePolicy: Fail
+  name: volumeattachment.mantle.cybozu.io
+  rules:
+  - apiGroups:
+    - storage.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - volumeattachments
+  sideEffects: None
+{{- if eq .Values.webhook.certificate.managedBy "helm" }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "mantle.fullname" . }}-webhook
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "mantle.labels" . | nindent 4 }}
+type: kubernetes.io/tls
+data:
+  ca.crt: {{ $tls.caCert }}
+  tls.crt: {{ $tls.clientCert }}
+  tls.key: {{ $tls.clientKey }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/mantle/values.yaml

@@ -29,7 +29,41 @@ controller:
   ports: []
   env: []
 
-secondaryService:
+  # controller.replicationTLSClientCertPath -- Path to the client certificate for the replication service. Only used when the role is primary.
+  replicationTLSClientCertPath: "/mnt/replication-tls-client-cert/tls.crt"
+  # controller.replicationTLSClientKeyPath -- Path to the client key for the replication service. Only used when the role is primary.
+  replicationTLSClientKeyPath: "/mnt/replication-tls-client-cert/tls.key"
+  # controller.replicationTLSServerCAPath -- Path to the CA certificate for the replication service. Only used when the role is primary.
+  replicationTLSServerCAPath: "/mnt/replication-tls-server-cert/ca.crt"
+
+  # controller.replicationTLSServerCertPath -- Path to the server certificate for the replication service. Only used when the role is secondary.
+  replicationTLSServerCertPath: "/mnt/replication-tls-server-cert/tls.crt"
+  # controller.replicationTLSServerKeyPath -- Path to the server key for the replication service. Only used when the role is secondary.
+  replicationTLSServerKeyPath: "/mnt/replication-tls-server-cert/tls.key"
+  # controller.replicationTLSClientCAPath -- Path to the CA certificate for the replication service. Only used when the role is secondary.
+  replicationTLSClientCAPath: "/mnt/replication-tls-client-cert/ca.crt"
+
+replicationService:
   # type:
   # port:
   # targetPort:
+
+webhook:
+  # webhook.enabled -- If true and role is secondary, Deploying a webhook to prevent Pods from mounting PVCs created by Mantle.
+  # When deploying multiple Mantle instances, Set the value to `true` on only a single instance.
+  enabled: true
+  service:
+    # webhook.service.port -- Port for the webhook service. 
+    port: 9443
+    # webhook.service.targetPort -- Target port for the webhook server pod.
+    targetPort: 9443
+  certificate:
+    # webhook.certificate.managedBy -- Can be "cert-manager" or "helm".
+    managedBy: cert-manager
+    # webhook.certificate.dnsDomain -- Only used when managedBy is "helm". Cluster DNS domain (required for requesting TLS certificates).
+    dnsDomain: cluster.local
+    # webhook.certificate.existingCertManagerIssuer -- Only used when managedBy is "cert-manager". Specify the cert-manager issuer to be used for AdmissionWebhook.
+    existingCertManagerIssuer: {}
+      # group: cert-manager.io
+      # kind: Issuer
+      # name: webhook-issuer