Support for configurable tls in remote jwks store

api/v1alpha1/agentgateway_policy_types.go

@@ -514,22 +514,20 @@ type AgentJWKS struct {
 	Inline *string `json:"inline,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=jwksUri;backendRef
 type AgentRemoteJWKS struct {
 	// IdP jwks endpoint. Default tls settings are used to connect to this url.
 	// +kubebuilder:validation:Pattern=`^(https|http):\/\/[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*(:\d+)?\/.*$`
-	// +optional
+	// +required
 	JwksUri string `json:"jwksUri,omitempty"`
 	// +optional
 	// +kubebuilder:validation:XValidation:rule="matches(self, '^([0-9]{1,5}(h|m|s|ms)){1,4}$')",message="invalid duration value"
 	// +kubebuilder:validation:XValidation:rule="duration(self) >= duration('5m')",message="cacheDuration must be at least 5m."
 	// +kubebuilder:default="5m"
 	CacheDuration *metav1.Duration `json:"cacheDuration,omitempty"`
 	// backendRef references the remote JWKS server to reach.
-	// Not implemented yet, only jwksUri is currently supported.
 	// Supported types: Service and Backend.
 	// +optional
-	BackendRef gwv1.BackendObjectReference `json:"backendRef,omitempty"`
+	BackendRef *gwv1.BackendObjectReference `json:"backendRef,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Strict;Optional

install/helm/kgateway-crds/templates/gateway.kgateway.dev_agentgatewaypolicies.yaml

@@ -2853,7 +2853,6 @@ spec:
                                     backendRef:
                                       description: |-
                                         backendRef references the remote JWKS server to reach.
-                                        Not implemented yet, only jwksUri is currently supported.
                                         Supported types: Service and Backend.
                                       properties:
                                         group:
@@ -2936,12 +2935,9 @@ spec:
                                         settings are used to connect to this url.
                                       pattern: ^(https|http):\/\/[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*(:\d+)?\/.*$
                                       type: string
+                                  required:
+                                  - jwksUri
                                   type: object
-                                  x-kubernetes-validations:
-                                  - message: exactly one of the fields in [jwksUri
-                                      backendRef] must be set
-                                    rule: '[has(self.jwksUri),has(self.backendRef)].filter(x,x==true).size()
-                                      == 1'
                               type: object
                               x-kubernetes-validations:
                               - message: exactly one of the fields in [remote inline]

internal/kgateway/agentjwksstore/cm_controller.go

@@ -0,0 +1,154 @@
+package agentjwksstore
+
+import (
+	"context"
+	"math"
+	"time"
+
+	"golang.org/x/time/rate"
+	"istio.io/istio/pkg/kube/controllers"
+	"istio.io/istio/pkg/kube/kclient"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/jwks"
+	"github.com/kgateway-dev/kgateway/v2/pkg/apiclient"
+	"github.com/kgateway-dev/kgateway/v2/pkg/logging"
+)
+
+var cmLogger = logging.New("jwks_store_config_map_controller")
+
+const JwksStoreConfigMapName = "jwks-store"
+
+type JwksStoreConfigMapsController struct {
+	apiClient           apiclient.Client
+	cmClient            kclient.Client[*corev1.ConfigMap]
+	eventQueue          controllers.Queue
+	jwksUpdates         chan map[string]string
+	jwksStore           *jwks.JwksStore
+	deploymentNamespace string
+	waitForSync         []cache.InformerSynced
+}
+
+var (
+	rateLimiter = workqueue.NewTypedMaxOfRateLimiter(
+		workqueue.NewTypedItemExponentialFailureRateLimiter[any](500*time.Millisecond, 10*time.Second),
+		// 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
+		&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+	)
+)
+
+func NewJWKSStoreConfigMapsController(apiClient apiclient.Client, deploymentNamespace string, jwksStore *jwks.JwksStore) *JwksStoreConfigMapsController {
+	cmLogger.Info("creating jwks store ConfigMap controller")
+	return &JwksStoreConfigMapsController{
+		apiClient:           apiClient,
+		deploymentNamespace: deploymentNamespace,
+		jwksStore:           jwksStore,
+	}
+}
+
+func (jcm *JwksStoreConfigMapsController) Init(ctx context.Context) {
+	jcm.cmClient = kclient.NewFiltered[*corev1.ConfigMap](jcm.apiClient,
+		kclient.Filter{
+			ObjectFilter:  jcm.apiClient.ObjectFilter(),
+			Namespace:     jcm.deploymentNamespace,
+			LabelSelector: jwks.JwksStoreLabelString})
+
+	jcm.waitForSync = []cache.InformerSynced{
+		jcm.cmClient.HasSynced,
+	}
+
+	jcm.jwksUpdates = jcm.jwksStore.SubscribeToUpdates()
+	jcm.eventQueue = controllers.NewQueue("JwksStoreController", controllers.WithReconciler(jcm.Reconcile), controllers.WithMaxAttempts(math.MaxInt), controllers.WithRateLimiter(rateLimiter))
+}
+
+func (jcm *JwksStoreConfigMapsController) Start(ctx context.Context) error {
+	cmLogger.Info("waiting for cache to sync")
+	jcm.apiClient.Core().WaitForCacheSync(
+		"kube jwks store ConfigMap syncer",
+		ctx.Done(),
+		jcm.waitForSync...,
+	)
+
+	cmLogger.Info("starting jwks store ConfigMap controller")
+	jcm.cmClient.AddEventHandler(
+		controllers.FromEventHandler(
+			func(o controllers.Event) {
+				jcm.eventQueue.AddObject(o.Latest())
+			}))
+
+	go func() {
+		for {
+			select {
+			case u := <-jcm.jwksUpdates:
+				for uri := range u {
+					jcm.eventQueue.AddObject(jcm.newJwksStoreConfigMap(jwks.JwksConfigMapName(uri)))
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	go jcm.eventQueue.Run(ctx.Done())
+
+	<-ctx.Done()
+	return nil
+}
+
+func (jcm *JwksStoreConfigMapsController) Reconcile(req types.NamespacedName) error {
+	cmLogger.Debug("syncing jwks store to ConfigMap(s)")
+	ctx := context.Background()
+
+	uri, storedJwks, ok := jcm.jwksStore.JwksByConfigMapName(req.Name)
+	if !ok {
+		cmLogger.Debug("deleting ConfigMap", "name", req.Name)
+		return client.IgnoreNotFound(jcm.apiClient.Kube().CoreV1().ConfigMaps(req.Namespace).Delete(ctx, req.Name, metav1.DeleteOptions{}))
+	}
+
+	existingCm := jcm.cmClient.Get(req.Name, req.Namespace)
+	if existingCm == nil {
+		newCm := jcm.newJwksStoreConfigMap(jwks.JwksConfigMapName(uri))
+		if err := jwks.SetJwksInConfigMap(newCm, uri, storedJwks); err != nil {
+			cmLogger.Error("error updating ConfigMap", "error", err)
+			return err // no retries?
+		}
+
+		_, err := jcm.apiClient.Kube().CoreV1().ConfigMaps(req.Namespace).Create(ctx, newCm, metav1.CreateOptions{})
+		if err != nil {
+			cmLogger.Error("error creating ConfigMap", "error", err)
+			return err
+		}
+	} else {
+		if err := jwks.SetJwksInConfigMap(existingCm, uri, storedJwks); err != nil {
+			cmLogger.Error("error updating ConfigMap", "error", err)
+			return err // no retries?
+		}
+		_, err := jcm.apiClient.Kube().CoreV1().ConfigMaps(req.Namespace).Update(ctx, existingCm, metav1.UpdateOptions{})
+		if err != nil {
+			cmLogger.Error("error updating jwks ConfigMap", "error", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// runs on the leader only
+func (jcm *JwksStoreConfigMapsController) NeedLeaderElection() bool {
+	return true
+}
+
+func (jcm *JwksStoreConfigMapsController) newJwksStoreConfigMap(name string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: jcm.deploymentNamespace,
+			Labels:    jwks.JwksStoreLabelMap,
+		},
+		Data: make(map[string]string),
+	}
+}