✨ add secure metrics support with secure endpoint instead of the RBAC proxy and improve helm values structure (#1458)

slntopp · web-flow · commit 6e4f78298083 · 2026-04-01T14:03:52.000+02:00
* ✨ add secure metrics support with TLS and RBAC

- Introduced `SecureMetrics` field in `MondooOperatorConfigSpec` to enable authentication and authorization on the metrics endpoint.
- Updated deployment configuration to serve metrics over HTTPS on port 8443 when `SecureMetrics` is enabled.
- Modified ServiceMonitor to handle both HTTP and HTTPS metrics endpoints based on the `SecureMetrics` setting.
- Added tests for metrics endpoint behavior under both secure and non-secure configurations.
- Updated Helm chart values to include `secureMetrics` option.
- Adjusted deployment and verification scripts to support the new secure metrics feature.

* ✨ refactor: update controller manager args and introduce extraArgs for flexibility

* ✨ feat: add secure metrics support with RBAC and update verification script
diff --git a/api/v1alpha2/mondoooperatorconfig_types.go b/api/v1alpha2/mondoooperatorconfig_types.go
@@ -55,6 +55,11 @@ type MondooOperatorConfigSpec struct {
 
 type Metrics struct {
 	Enable bool `json:"enable,omitempty"`
+	// SecureMetrics enables authentication and authorization on the metrics endpoint
+	// using controller-runtime's built-in TLS and RBAC-based auth.
+	// When enabled, metrics are served over HTTPS on port 8443 with token-based auth.
+	// When disabled, metrics are served over HTTP on port 8080 without auth.
+	SecureMetrics bool `json:"secureMetrics,omitempty"`
 	// ResourceLabels allows providing a list of extra labels to apply to the metrics-related
 	// resources (eg. ServiceMonitor)
 	ResourceLabels map[string]string `json:"resourceLabels,omitempty"`
diff --git a/charts/mondoo-operator/files/crds/k8s.mondoo.com_mondoooperatorconfigs.yaml b/charts/mondoo-operator/files/crds/k8s.mondoo.com_mondoooperatorconfigs.yaml
@@ -93,6 +93,13 @@ spec:
                       ResourceLabels allows providing a list of extra labels to apply to the metrics-related
                       resources (eg. ServiceMonitor)
                     type: object
+                  secureMetrics:
+                    description: |-
+                      SecureMetrics enables authentication and authorization on the metrics endpoint
+                      using controller-runtime's built-in TLS and RBAC-based auth.
+                      When enabled, metrics are served over HTTPS on port 8443 with token-based auth.
+                      When disabled, metrics are served over HTTP on port 8080 without auth.
+                    type: boolean
                 type: object
               noProxy:
                 description: NoProxy specifies a comma-separated list of hosts that
diff --git a/charts/mondoo-operator/templates/deployment.yaml b/charts/mondoo-operator/templates/deployment.yaml
@@ -17,7 +17,19 @@ spec:
         kubectl.kubernetes.io/default-container: manager
     spec:
       containers:
-      - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }}
+      - args:
+        - operator
+        - --health-probe-bind-address=:8081
+        - --leader-elect
+        {{- if .Values.controllerManager.manager.secureMetrics }}
+        - --metrics-bind-address=:8443
+        - --secure-metrics
+        {{- else }}
+        - --metrics-bind-address=:8080
+        {{- end }}
+        {{- with .Values.controllerManager.manager.extraArgs }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         command:
         - /mondoo-operator
         env:
@@ -59,9 +71,15 @@ spec:
           periodSeconds: 20
         name: manager
         ports:
+        {{- if .Values.controllerManager.manager.secureMetrics }}
+        - containerPort: 8443
+          name: https
+          protocol: TCP
+        {{- else }}
         - containerPort: 8080
           name: metrics
           protocol: TCP
+        {{- end }}
         readinessProbe:
           httpGet:
             path: /readyz
diff --git a/charts/mondoo-operator/templates/metrics-reader-rbac.yaml b/charts/mondoo-operator/templates/metrics-reader-rbac.yaml
@@ -11,4 +11,21 @@ rules:
 - nonResourceURLs:
   - /metrics
   verbs:
-  - get
+  - get
+{{- if .Values.controllerManager.manager.secureMetrics }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "mondoo-operator.fullname" . }}-metrics-reader-binding
+  labels:
+  {{- include "mondoo-operator.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "mondoo-operator.fullname" . }}-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ include "mondoo-operator.fullname" . }}-controller-manager
+  namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/mondoo-operator/templates/metrics-service.yaml b/charts/mondoo-operator/templates/metrics-service.yaml
@@ -9,4 +9,11 @@ spec:
   selector:
     {{- include "mondoo-operator.selectorLabels" . | nindent 4 }}
   ports:
+  {{- if .Values.controllerManager.manager.secureMetrics }}
+  - name: https
+    port: 8443
+    protocol: TCP
+    targetPort: https
+  {{- else }}
   {{- .Values.metricsService.ports | toYaml | nindent 2 }}
+  {{- end }}
diff --git a/charts/mondoo-operator/templates/mondoooperatorconfig.yaml b/charts/mondoo-operator/templates/mondoooperatorconfig.yaml
@@ -34,5 +34,9 @@ spec:
   {{- if .Values.operator.skipProxyForCnspec }}
   {{- $_ := set $spec "skipProxyForCnspec" .Values.operator.skipProxyForCnspec }}
   {{- end }}
+  {{- if .Values.controllerManager.manager.secureMetrics }}
+  {{- $metrics := dict "enable" true "secureMetrics" true }}
+  {{- $_ := set $spec "metrics" $metrics }}
+  {{- end }}
   {{- $spec | toYaml | nindent 2 }}
 {{- end }}
diff --git a/charts/mondoo-operator/values.yaml b/charts/mondoo-operator/values.yaml
@@ -3,12 +3,8 @@
 
 controllerManager:
   manager:
-    ## @param controllerManager.manager.args Command-line arguments passed to the operator manager container
-    args:
-    - operator
-    - --health-probe-bind-address=:8081
-    - --metrics-bind-address=:8080
-    - --leader-elect
+    ## @param controllerManager.manager.extraArgs Additional command-line arguments passed to the operator manager container
+    extraArgs: []
     ## @param controllerManager.manager.containerSecurityContext [object] Security context for the manager container
     containerSecurityContext:
       allowPrivilegeEscalation: false
@@ -22,6 +18,8 @@ controllerManager:
       repository: ghcr.io/mondoohq/mondoo-operator
       ## @param controllerManager.manager.image.tag Container image tag for the operator (defaults to .Chart.AppVersion)
       tag: ""
+    ## @param controllerManager.manager.secureMetrics Enable RBAC-authenticated HTTPS metrics (port 8443)
+    secureMetrics: false
     ## @param controllerManager.manager.imagePullPolicy Image pull policy for the operator container
     imagePullPolicy: IfNotPresent
     ## @param controllerManager.manager.resources [object] Resource requests and limits for the manager container
diff --git a/cmd/mondoo-operator/operator/cmd.go b/cmd/mondoo-operator/operator/cmd.go
@@ -4,12 +4,14 @@
 package operator
 
 import (
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -52,6 +54,8 @@ func init() {
 	probeAddr := Cmd.Flags().String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	enableLeaderElection := Cmd.Flags().Bool("leader-elect", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	secureMetrics := Cmd.Flags().Bool("secure-metrics", false,
+		"Enable authentication and authorization on the metrics endpoint via HTTPS and RBAC.")
 
 	Cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		// TODO: opts.BindFlags(flag.CommandLine) is not supported with cobra. If we want to support that we should manually
@@ -67,9 +71,18 @@ func init() {
 		utilruntime.Must(certmanagerv1.AddToScheme(scheme))
 		utilruntime.Must(monitoringv1.AddToScheme(scheme))
 
+		metricsOpts := metricsserver.Options{BindAddress: *metricsAddr}
+		if *secureMetrics {
+			metricsOpts.SecureServing = true
+			metricsOpts.FilterProvider = filters.WithAuthenticationAndAuthorization
+			metricsOpts.TLSOpts = []func(*tls.Config){
+				func(c *tls.Config) { c.MinVersion = tls.VersionTLS12 },
+			}
+		}
+
 		mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 			Scheme:                 scheme,
-			Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+			Metrics:                metricsOpts,
 			HealthProbeBindAddress: *probeAddr,
 			LeaderElection:         *enableLeaderElection,
 			LeaderElectionID:       "60679458.mondoo.com",
diff --git a/config/crd/bases/k8s.mondoo.com_mondoooperatorconfigs.yaml b/config/crd/bases/k8s.mondoo.com_mondoooperatorconfigs.yaml
@@ -93,6 +93,13 @@ spec:
                       ResourceLabels allows providing a list of extra labels to apply to the metrics-related
                       resources (eg. ServiceMonitor)
                     type: object
+                  secureMetrics:
+                    description: |-
+                      SecureMetrics enables authentication and authorization on the metrics endpoint
+                      using controller-runtime's built-in TLS and RBAC-based auth.
+                      When enabled, metrics are served over HTTPS on port 8443 with token-based auth.
+                      When disabled, metrics are served over HTTP on port 8080 without auth.
+                    type: boolean
                 type: object
               noProxy:
                 description: NoProxy specifies a comma-separated list of hosts that
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -30,9 +30,6 @@ bases:
 #- ../prometheus
 
 patchesStrategicMerge:
-# Uncomment the following line to protect the /metrics endpoint with kube-rbac-proxy.
-# - manager_auth_proxy_patch.yaml
-
 - manager_public_metrics_patch.yaml
 
 # Mount the controller config file for loading manager configurations
diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
diff --git a/config/rbac/auth_proxy_client_clusterrolebinding.yaml b/config/rbac/auth_proxy_client_clusterrolebinding.yaml
@@ -0,0 +1,15 @@
+# Copyright (c) Mondoo, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -12,11 +12,13 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Uncomment the following 4 lines to enable the kube-rbac-proxy auth proxy.
+# Uncomment the following 4 lines to enable RBAC-authenticated secure metrics.
+# Required when using --secure-metrics flag.
 #- auth_proxy_service.yaml
 #- auth_proxy_role.yaml
 #- auth_proxy_role_binding.yaml
 #- auth_proxy_client_clusterrole.yaml
+#- auth_proxy_client_clusterrolebinding.yaml
 # Comment the following lines to self manage roles for the workload
 - k8s_resources_scanning_service_account.yaml
 - k8s_resources_scanning_clusterrole.yaml
diff --git a/controllers/service_monitor.go b/controllers/service_monitor.go
@@ -88,12 +88,7 @@ func (s *ServiceMonitor) serviceMonitorForMondoo(m *mondoov1alpha2.MondooOperato
 		},
 		Spec: monitoringv1.ServiceMonitorSpec{
 			Endpoints: []monitoringv1.Endpoint{
-				{
-					Path: "/metrics",
-					// The named port exposing metrics from the mondoo-operator Deployment
-					Port:   "metrics",
-					Scheme: ptr.To(monitoringv1.SchemeHTTP),
-				},
+				s.metricsEndpoint(),
 			},
 			Selector: metav1.LabelSelector{
 				MatchLabels: map[string]string{
@@ -109,6 +104,31 @@ func (s *ServiceMonitor) serviceMonitorForMondoo(m *mondoov1alpha2.MondooOperato
 	return dep
 }
 
+func (s *ServiceMonitor) metricsEndpoint() monitoringv1.Endpoint {
+	if s.Config.Spec.Metrics.SecureMetrics {
+		ep := monitoringv1.Endpoint{
+			Path:   "/metrics",
+			Port:   "https",
+			Scheme: ptr.To(monitoringv1.SchemeHTTPS),
+		}
+		ep.BearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint:staticcheck // SA1019: no credentialsFile alternative available in SafeAuthorization
+		// InsecureSkipVerify is safe here because controller-runtime's SecureServing
+		// uses a self-signed certificate for the metrics endpoint. The connection is
+		// still encrypted (HTTPS); we just skip CA verification since there is no CA.
+		ep.TLSConfig = &monitoringv1.TLSConfig{
+			SafeTLSConfig: monitoringv1.SafeTLSConfig{
+				InsecureSkipVerify: ptr.To(true),
+			},
+		}
+		return ep
+	}
+	return monitoringv1.Endpoint{
+		Path:   "/metrics",
+		Port:   "metrics",
+		Scheme: ptr.To(monitoringv1.SchemeHTTP),
+	}
+}
+
 func (s *ServiceMonitor) Reconcile(ctx context.Context, clt client.Client, scheme *runtime.Scheme, req ctrl.Request) (ctrl.Result, error) {
 	log := ctrllog.FromContext(ctx)
 	found, err := k8s.VerifyAPI(monitoringv1.SchemeGroupVersion.Group, monitoringv1.SchemeGroupVersion.Version)
diff --git a/controllers/service_monitor_test.go b/controllers/service_monitor_test.go
diff --git a/go.mod b/go.mod
diff --git a/go.sum b/go.sum
diff --git a/tests/e2e/scripts/deploy-operator.sh b/tests/e2e/scripts/deploy-operator.sh
diff --git a/tests/e2e/scripts/verify.sh b/tests/e2e/scripts/verify.sh
