vmauth: JWT support

Author: AndrewChubatiuk

.golangci.yml

@@ -44,10 +44,12 @@ linters:
           alias: metav1
         - pkg: k8s.io/api/apps/v1
           alias: appsv1
-        - pkg: k8s.io/api/autoscaling/v2"
+        - pkg: k8s.io/api/autoscaling/v2
           alias: autoscalingv2
         - pkg: k8s.io/apimachinery/pkg/api/errors
           alias: k8serrors
+        - pkg: k8s.io/api/networking/v1
+          alias: networkingv1
 formatters:
   enable:
     - gofmt

api/operator/v1beta1/vmauth_types.go

@@ -7,7 +7,8 @@ import (
 	"regexp"
 	"strings"
 
-	v12 "k8s.io/api/networking/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/utils/ptr"
@@ -84,6 +85,9 @@ type VMAuthSpec struct {
 	// will be added after removal of VMUserConfigOptions
 	// currently it has collision with inlined fields
 	// IPFilters VMUserIPFilters `json:"ip_filters,omitempty"`
+	// OODC represents configuration section for OIDC authorization
+	// +optional
+	OIDC []*VMAuthOIDCRealm `json:"jwt,omitempty"`
 	// will be removed at v1.0 release
 	// +deprecated
 	// +kubebuilder:validation:Schemaless
@@ -125,6 +129,28 @@ type VMAuthSpec struct {
 	UseProxyProtocol bool `json:"useProxyProtocol,omitempty"`
 }
 
+// VMAuthOIDCRealm defines OIDC realm parameters
+type VMAuthOIDCRealm struct {
+	// EnforcePrefix requires JWT token to start with "Bearer: "
+	// +optional
+	EnforcePrefix bool `json:"enforce_prefix,omitempty"`
+	// IssuerURL is OpenID Connect issuer URL
+	// +optional
+	IssuerURL string `json:"issuer_url,omitempty"`
+	// JWKsURL is the OpenID Connect JWKS URL
+	// +optional
+	JWKsURL string `json:"jwks_url,omitempty"`
+	// SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints
+	// +optional
+	SkipDiscovery bool `json:"skip_discovery,omitempty"`
+	// PublicKeyFiles is a list of paths pointing to public key files in PEM format to use
+	// for verifying JWT tokens
+	PublicKeyFiles []string `json:"public_key_files,omitempty"`
+	// PublicKeySecrets is a list of k8s Secret selectors pointing to public key files in PEM format to use
+	// for verifying JWT tokens
+	PublicKeySecrets []*corev1.SecretKeySelector `json:"public_key_secrets,omitempty"`
+}
+
 // VMAuthUnauthorizedUserAccessSpec defines unauthorized_user section configuration for vmauth
 type VMAuthUnauthorizedUserAccessSpec struct {
 	// URLPrefix defines prefix prefix for destination
@@ -425,7 +451,9 @@ func (cr *VMAuth) Validate() error {
 			return fmt.Errorf("incorrect cr.spec UnauthorizedAccessConfig options: %w", err)
 		}
 	}
-
+	if len(cr.Spec.OIDC) > 0 && !cr.Spec.License.IsProvided() {
+		return fmt.Errorf("spec.jwt is only allowed in enterprise mode, but no license provided")
+	}
 	if cr.Spec.UnauthorizedUserAccessSpec != nil {
 		if err := cr.Spec.UnauthorizedUserAccessSpec.Validate(); err != nil {
 			return fmt.Errorf("incorrect cr.spec.UnauthorizedUserAccess syntax: %w", err)
@@ -461,11 +489,11 @@ type EmbeddedIngress struct {
 	// ExtraRules - additional rules for ingress,
 	// must be checked for correctness by user.
 	// +optional
-	ExtraRules []v12.IngressRule `json:"extraRules,omitempty" yaml:"extraRules,omitempty"`
+	ExtraRules []networkingv1.IngressRule `json:"extraRules,omitempty" yaml:"extraRules,omitempty"`
 	// ExtraTLS - additional TLS configuration for ingress
 	// must be checked for correctness by user.
 	// +optional
-	ExtraTLS []v12.IngressTLS `json:"extraTls,omitempty" yaml:"extraTls,omitempty"`
+	ExtraTLS []networkingv1.IngressTLS `json:"extraTls,omitempty" yaml:"extraTls,omitempty"`
 	// Host defines ingress host parameter for default rule
 	// It will be used, only if TlsHosts is empty
 	// +optional

api/operator/v1beta1/vmuser_types.go

@@ -15,6 +15,9 @@ type VMUserSpec struct {
 	// Name of the VMUser object.
 	// +optional
 	Name *string `json:"name,omitempty"`
+	// ClientID extracted from JWT token
+	// +optional
+	ClientID *string `json:"client_id,omitempty"`
 	// UserName basic auth user name for accessing protected endpoint,
 	// will be replaced with metadata.name of VMUser if omitted.
 	// +optional

api/operator/v1beta1/zz_generated.deepcopy.go

@@ -23651,6 +23651,69 @@ spec:
                   and v1.111.0 vmauth version
                   related doc https://docs.victoriametrics.com/victoriametrics/vmauth/#security
                 type: string
+              jwt:
+                description: |-
+                  IPFilters global access ip filters
+                  supported only with enterprise version of [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/#ip-filters)
+                  will be added after removal of VMUserConfigOptions
+                  currently it has collision with inlined fields
+                  IPFilters VMUserIPFilters `json:"ip_filters,omitempty"`
+                  OODC represents configuration section for OIDC authorization
+                items:
+                  description: VMAuthOIDCRealm defines OIDC realm parameters
+                  properties:
+                    enforce_prefix:
+                      description: 'EnforcePrefix requires JWT token to start with
+                        "Bearer: "'
+                      type: boolean
+                    issuer_url:
+                      description: IssuerURL is OpenID Connect issuer URL
+                      type: string
+                    jwks_url:
+                      description: JWKsURL is the OpenID Connect JWKS URL
+                      type: string
+                    public_key_files:
+                      description: |-
+                        PublicKeyFiles is a list of paths pointing to public key files in PEM format to use
+                        for verifying JWT tokens
+                      items:
+                        type: string
+                      type: array
+                    public_key_secrets:
+                      description: |-
+                        PublicKeySecrets is a list of k8s Secret selectors pointing to public key files in PEM format to use
+                        for verifying JWT tokens
+                      items:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            default: ""
+                            description: |-
+                              Name of the referent.
+                              This field is effectively required, but due to backwards compatibility is
+                              allowed to be empty. Instances of this type with an empty value here are
+                              almost certainly wrong.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      type: array
+                    skip_discovery:
+                      description: SkipDiscovery allows to skip OIDC discovery and
+                        use manually supplied Endpoints
+                      type: boolean
+                  type: object
+                type: array
               license:
                 description: |-
                   License allows to configure license key to be used for enterprise features.
@@ -38811,6 +38874,9 @@ spec:
                 description: BearerToken Authorization header value for accessing
                   protected endpoint.
                 type: string
+              client_id:
+                description: ClientID extracted from JWT token
+                type: string
               default_url:
                 description: |-
                   DefaultURLs backend url for non-matching paths filter

config/crd/overlay/crd.yaml

@@ -3547,6 +3547,24 @@ Appears in: [VMAuthLoadBalancer](#vmauthloadbalancer)
 | volumes<a href="#vmauthloadbalancerspec-volumes" id="vmauthloadbalancerspec-volumes">#</a><br/>_[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | _(Required)_<br/>Volumes allows configuration of additional volumes on the output Deployment/StatefulSet definition.<br />Volumes specified will be appended to other volumes that are generated.<br />/ +optional |
 
 
+#### VMAuthOIDCRealm
+
+
+
+VMAuthOIDCRealm defines OIDC realm parameters
+
+Appears in: [VMAuthSpec](#vmauthspec)
+
+| Field | Description |
+| --- | --- |
+| enforce_prefix<a href="#vmauthoidcrealm-enforce_prefix" id="vmauthoidcrealm-enforce_prefix">#</a><br/>_boolean_ | _(Optional)_<br/>EnforcePrefix requires JWT token to start with "Bearer: " |
+| issuer_url<a href="#vmauthoidcrealm-issuer_url" id="vmauthoidcrealm-issuer_url">#</a><br/>_string_ | _(Optional)_<br/>IssuerURL is OpenID Connect issuer URL |
+| jwks_url<a href="#vmauthoidcrealm-jwks_url" id="vmauthoidcrealm-jwks_url">#</a><br/>_string_ | _(Optional)_<br/>JWKsURL is the OpenID Connect JWKS URL |
+| public_key_files<a href="#vmauthoidcrealm-public_key_files" id="vmauthoidcrealm-public_key_files">#</a><br/>_string array_ | _(Required)_<br/>PublicKeyFiles is a list of paths pointing to public key files in PEM format to use<br />for verifying JWT tokens |
+| public_key_secrets<a href="#vmauthoidcrealm-public_key_secrets" id="vmauthoidcrealm-public_key_secrets">#</a><br/>_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#secretkeyselector-v1-core) array_ | _(Required)_<br/>PublicKeySecrets is a list of k8s Secret selectors pointing to public key files in PEM format to use<br />for verifying JWT tokens |
+| skip_discovery<a href="#vmauthoidcrealm-skip_discovery" id="vmauthoidcrealm-skip_discovery">#</a><br/>_boolean_ | _(Optional)_<br/>SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints |
+
+
 #### VMAuthSpec
 
 
@@ -3587,6 +3605,7 @@ Appears in: [VMAuth](#vmauth)
 | initContainers<a href="#vmauthspec-initcontainers" id="vmauthspec-initcontainers">#</a><br/>_[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#container-v1-core) array_ | _(Optional)_<br/>InitContainers allows adding initContainers to the pod definition.<br />Any errors during the execution of an initContainer will lead to a restart of the Pod.<br />More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
 | internalListenPort<a href="#vmauthspec-internallistenport" id="vmauthspec-internallistenport">#</a><br/>_string_ | _(Optional)_<br/>InternalListenPort instructs vmauth to serve internal routes at given port<br />available from v0.56.0 operator<br />and v1.111.0 vmauth version<br />related doc https://docs.victoriametrics.com/victoriametrics/vmauth/#security |
 | ip_filters<a href="#vmauthspec-ip_filters" id="vmauthspec-ip_filters">#</a><br/>_[VMUserIPFilters](#vmuseripfilters)_ | _(Optional)_<br/>IPFilters defines per target src ip filters<br />supported only with enterprise version of [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/#ip-filters) |
+| jwt<a href="#vmauthspec-jwt" id="vmauthspec-jwt">#</a><br/>_[VMAuthOIDCRealm](#vmauthoidcrealm) array_ | _(Optional)_<br/>IPFilters global access ip filters<br />supported only with enterprise version of [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/#ip-filters)<br />will be added after removal of VMUserConfigOptions<br />currently it has collision with inlined fields<br />IPFilters VMUserIPFilters `json:"ip_filters,omitempty"`<br />OODC represents configuration section for OIDC authorization |
 | license<a href="#vmauthspec-license" id="vmauthspec-license">#</a><br/>_[License](#license)_ | _(Optional)_<br/>License allows to configure license key to be used for enterprise features.<br />Using license key is supported starting from VictoriaMetrics v1.94.0.<br />See [here](https://docs.victoriametrics.com/victoriametrics/enterprise/) |
 | load_balancing_policy<a href="#vmauthspec-load_balancing_policy" id="vmauthspec-load_balancing_policy">#</a><br/>_string_ | _(Optional)_<br/>LoadBalancingPolicy defines load balancing policy to use for backend urls.<br />Supported policies: least_loaded, first_available.<br />See [here](https://docs.victoriametrics.com/victoriametrics/vmauth#load-balancing) for more details (default "least_loaded") |
 | logFormat<a href="#vmauthspec-logformat" id="vmauthspec-logformat">#</a><br/>_string_ | _(Optional)_<br/>LogFormat for VMAuth to be configured with. |
@@ -4476,6 +4495,7 @@ Appears in: [VMUser](#vmuser)
 | Field | Description |
 | --- | --- |
 | bearerToken<a href="#vmuserspec-bearertoken" id="vmuserspec-bearertoken">#</a><br/>_string_ | _(Optional)_<br/>BearerToken Authorization header value for accessing protected endpoint. |
+| client_id<a href="#vmuserspec-client_id" id="vmuserspec-client_id">#</a><br/>_string_ | _(Optional)_<br/>ClientID extracted from JWT token |
 | default_url<a href="#vmuserspec-default_url" id="vmuserspec-default_url">#</a><br/>_string array_ | _(Required)_<br/>DefaultURLs backend url for non-matching paths filter<br />usually used for default backend with error message |
 | disable_secret_creation<a href="#vmuserspec-disable_secret_creation" id="vmuserspec-disable_secret_creation">#</a><br/>_boolean_ | _(Required)_<br/>DisableSecretCreation skips related secret creation for vmuser |
 | discover_backend_ips<a href="#vmuserspec-discover_backend_ips" id="vmuserspec-discover_backend_ips">#</a><br/>_boolean_ | _(Required)_<br/>DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS. |

docs/api.md

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	v2 "k8s.io/api/autoscaling/v2"
+	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	"k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -16,9 +16,9 @@ import (
 )
 
 // HPA creates or update horizontalPodAutoscaler object
-func HPA(ctx context.Context, rclient client.Client, newHPA, prevHPA *v2.HorizontalPodAutoscaler) error {
+func HPA(ctx context.Context, rclient client.Client, newHPA, prevHPA *autoscalingv2.HorizontalPodAutoscaler) error {
 	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		var currentHPA v2.HorizontalPodAutoscaler
+		var currentHPA autoscalingv2.HorizontalPodAutoscaler
 		if err := rclient.Get(ctx, types.NamespacedName{Name: newHPA.GetName(), Namespace: newHPA.GetNamespace()}, &currentHPA); err != nil {
 			if k8serrors.IsNotFound(err) {
 				logger.WithContext(ctx).Info(fmt.Sprintf("creating HPA %s configuration", newHPA.Name))

internal/controller/operator/factory/reconcile/hpa.go

@@ -424,6 +424,42 @@ func generateVMAuthConfig(cr *vmv1beta1.VMAuth, sus *skipableVMUsers, crdCache m
 		return nil, fmt.Errorf("cannot build unauthorized_user config section: %w", err)
 	}
 
+	var oidcCfg []yaml.MapSlice
+	for _, realm := range cr.Spec.OIDC {
+		if realm == nil {
+			continue
+		}
+		var oidcItem yaml.MapSlice
+		if realm.SkipDiscovery {
+			if len(realm.JWKsURL) > 0 {
+				oidcItem = append(oidcItem, yaml.MapItem{Key: "jwks_url", Value: realm.JWKsURL})
+			}
+			var publicKeyFiles []string
+			if len(realm.PublicKeyFiles) > 0 {
+				publicKeyFiles = append(publicKeyFiles, realm.PublicKeyFiles...)
+			}
+			for _, ref := range realm.PublicKeySecrets {
+				file, err := ac.LoadPathFromSecret(build.TLSAssetsResourceKind, cr.Namespace, ref)
+				if err != nil {
+					return nil, fmt.Errorf("cannot build jwt config section: %w", err)
+				}
+				publicKeyFiles = append(publicKeyFiles, file)
+			}
+			if len(publicKeyFiles) > 0 {
+				oidcItem = append(oidcItem, yaml.MapItem{Key: "public_key_files", Value: publicKeyFiles})
+			}
+		} else {
+			oidcItem = append(oidcItem, yaml.MapItem{Key: "issuer_url", Value: realm.IssuerURL})
+		}
+		if realm.EnforcePrefix {
+			oidcItem = append(oidcItem, yaml.MapItem{Key: "enforce_prefix", Value: realm.EnforcePrefix})
+		}
+		oidcCfg = append(oidcCfg, oidcItem)
+	}
+	if len(oidcCfg) > 0 {
+		cfg = append(cfg, yaml.MapItem{Key: "oidc", Value: oidcCfg})
+	}
+
 	if len(unAuthorizedAccessValue) > 0 {
 		cfg = append(cfg, yaml.MapItem{Key: "unauthorized_user", Value: unAuthorizedAccessValue})
 	}
@@ -789,7 +825,7 @@ func genUserCfg(user *vmv1beta1.VMUser, crdURLCache map[string]string, cr *vmv1b
 	}
 
 	// generate user access config.
-	var name, username, password, token string
+	var name, username, password, token, clientID string
 	if user.Spec.Name != nil {
 		name = *user.Spec.Name
 	}
@@ -807,6 +843,9 @@ func genUserCfg(user *vmv1beta1.VMUser, crdURLCache map[string]string, cr *vmv1b
 		password = *user.Spec.Password
 	}
 
+	if cr.Spec.License.IsProvided() && user.Spec.ClientID != nil {
+		clientID = *user.Spec.ClientID
+	}
 	if user.Spec.BearerToken != nil {
 		token = *user.Spec.BearerToken
 	}
@@ -829,6 +868,15 @@ func genUserCfg(user *vmv1beta1.VMUser, crdURLCache map[string]string, cr *vmv1b
 		})
 		return r, nil
 	}
+
+	if clientID != "" {
+		r = append(r, yaml.MapItem{
+			Key:   "client_id",
+			Value: clientID,
+		})
+		return r, nil
+	}
+
 	// mutate vmuser
 	if username == "" {
 		username = user.Name