refactor: improve function signatures and error handling across multiple files

Author: cuisongliu

controllers/user/deploy/charts/user/templates/cert.yaml

@@ -23,3 +23,31 @@ spec:
     kind: Issuer
     name: {{ include "user.fullname" . }}-selfsigned-issuer
   secretName: webhook-server-cert
+---
+{{- if .Values.metrics.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    {{- include "user.labels" . | nindent 4 }}
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    {{- include "user.labels" . | nindent 4 }}
+  name: metrics-certs
+spec:
+  privateKey:
+    rotationPolicy: Always
+  dnsNames:
+    - {{ include "user.fullname" . }}-metrics.{{.Release.Namespace}}.svc
+    - {{ include "user.fullname" . }}-metrics.{{.Release.Namespace}}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: {{.Values.metrics.secretName}}
+{{- end }}

controllers/user/deploy/charts/user/templates/configmap.yaml

@@ -39,15 +39,21 @@ spec:
             - /manager
           args:
             - --health-probe-bind-address=:8081
-            - --metrics-bind-address=127.0.0.1:8080
             - --leader-elect
-            - --config-file-path=/config.yaml
+            {{- if .Values.metrics.enabled }}
+            - --metrics-secure=true
+            - --metrics-bind-address=:8443
+            {{- end }}
           env:
             - name: NAMESPACE_NAME
               valueFrom:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.namespace
+            - name: SEALOS_CLOUD_APISERVER_HOST
+              value: "{{.Values.cloudAPIServerDomain}}"
+            - name: SEALOS_CLOUD_APISERVER_PORT
+              value: "{{.Values.cloudAPIServerPort}}"
           ports:
             - containerPort: 9443
               name: webhook-server
@@ -63,28 +69,17 @@ spec:
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           volumeMounts:
+            {{- with .Values.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+            {{- if .Values.metrics.enabled }}
+            - name: metrics-certs
+              mountPath: {{.Values.metrics.certPath}}
+              readOnly: true
+            {{- end }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
-            - name: user-manager-volume
-              mountPath: /config.yaml
-              subPath: config.yaml
-        - name: kube-rbac-proxy
-          args:
-            - --secure-listen-address=0.0.0.0:8443
-            - --upstream=http://127.0.0.1:8080/
-            - --logtostderr=true
-            - --v=0
-          image: "{{ .Values.proxy.image }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          ports:
-            - containerPort: 8443
-              name: https
-              protocol: TCP
-          resources:
-            {{- toYaml .Values.proxy.resources | nindent 12 }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
       terminationGracePeriodSeconds: 10
       affinity:
         {{- if .Values.affinity }}
@@ -111,10 +106,23 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        {{- if .Values.metrics.enabled }}
+        - name: metrics-certs
+          secret:
+            secretName: {{.Values.metrics.secretName}}
+            optional: false
+            items:
+              - key: ca.crt
+                path: ca.crt
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.key
+        {{- end }}
+        {{- with .Values.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         - name: cert
           secret:
             defaultMode: 420
             secretName: webhook-server-cert
-        - name: user-manager-volume
-          configMap:
-            name: {{ include "user.fullname" . }}-manager-config

controllers/user/deploy/charts/user/templates/deployment.yaml

@@ -44,16 +44,6 @@ resources:
     cpu: 10m
     memory: 64Mi
 
-proxy:
-  image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
-  resources:
-    limits:
-      cpu: 1000m
-      memory: 1024Mi
-    requests:
-      cpu: 5m
-      memory: 64Mi
-
 cloudAPIServerDomain: 127.0.0.1.nip.io
 cloudAPIServerPort: "6443"
 
@@ -73,7 +63,7 @@ readinessProbe:
 
 metrics:
   # Enable metrics endpoint
-  enabled: false
+  enabled: true
   # Path to the metrics certificate
   certPath: /tmp/k8s-metrics-server/metrics-certs
   secretName: metrics-server-cert

controllers/user/deploy/charts/user/values.yaml

@@ -4,5 +4,7 @@ set -euo pipefail
 HELM_OPTS=${HELM_OPTS:-""}
 
 kubectl delete -f ./drop/ --ignore-not-found
-helm upgrade -i user -n user-system --create-namespace ./charts/user ${HELM_OPTS}
+SEALOS_CLOUD_DOMAIN=$(kubectl get configmap sealos-config -n sealos-system -o jsonpath='{.data.cloudDomain}')
+
+helm upgrade -i user -n user-system --create-namespace ./charts/user --set cloudAPIServerDomain=${SEALOS_CLOUD_DOMAIN} ${HELM_OPTS}
 helm show crds ./charts/user | kubectl apply -f - --server-side --force-conflicts

controllers/user/deploy/entrypoint.sh

@@ -18,13 +18,16 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"os"
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -38,7 +41,6 @@ import (
 	licensev1 "github.com/labring/sealos/controllers/license/api/v1"
 	userv1 "github.com/labring/sealos/controllers/user/api/v1"
 	"github.com/labring/sealos/controllers/user/controllers"
-	configpkg "github.com/labring/sealos/controllers/user/controllers/helper/config"
 	ratelimiter "github.com/labring/sealos/controllers/user/controllers/helper/ratelimiter"
 	//+kubebuilder:scaffold:imports
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -70,6 +72,9 @@ func main() {
 		operationReqExpirationTime time.Duration
 		restartPredicateDuration   time.Duration
 		operationReqRetentionTime  time.Duration
+		secureMetrics              bool
+		enableHTTP2                bool
+		tlsOpts                    []func(*tls.Config)
 	)
 	flag.StringVar(
 		&metricsAddr,
@@ -128,6 +133,14 @@ func main() {
 		"/config.yaml",
 		"The path to the configuration file.",
 	)
+	flag.BoolVar(
+		&secureMetrics,
+		"metrics-secure",
+		true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.",
+	)
+	flag.BoolVar(&enableHTTP2, "enable-http2", false,
+		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	rateLimiterOptions.BindFlags(flag.CommandLine)
 	opts := zap.Options{
 		Development: true,
@@ -137,11 +150,45 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// if the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	disableHTTP2 := func(c *tls.Config) {
+		setupLog.Info("disabling http/2")
+		c.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsOpts = append(tlsOpts, disableHTTP2)
+	}
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+		// generate self-signed certificates for the metrics server. While convenient for development and testing,
+		// this setup is not recommended for production.
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress: metricsAddr,
-		},
+		Scheme:  scheme,
+		Metrics: metricsServerOptions,
 		// WebhookServer: webhook.NewServer(webhook.Options{
 		//	Port: 9443,
 		// }),
@@ -171,19 +218,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Load the configuration file
-	config := &configpkg.Config{}
-	if err := configpkg.LoadConfig(configFilePath, config); err != nil {
-		setupLog.Error(err, "unable to load configuration file")
-		os.Exit(1)
-	}
-
-	// Set the configuration
-	if err := setConfigToEnv(*config); err != nil {
-		setupLog.Error(err, "unable to set configuration to environment variables")
-		os.Exit(1)
-	}
-
 	if err := controllers.SetupLicenseGate(mgr); err != nil {
 		setupLog.Error(err, "unable to set up license gate")
 		os.Exit(1)
@@ -243,10 +277,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-func setConfigToEnv(cfg configpkg.Config) error {
-	if err := os.Setenv("SEALOS_CLOUD_APISERVER_HOST", cfg.CloudAPIServerDomain); err != nil {
-		return err
-	}
-	return os.Setenv("SEALOS_CLOUD_APISERVER_PORT", cfg.CloudAPIServerPort)
-}