Merge pull request #2524 from zimnx/remote-configmaps

Mirror ConfigMaps and Secrets referenced in ScyllaDBCluster into remote datacenters

Author: scylla-operator-bot[bot]

deploy/operator.yaml

@@ -443,6 +443,8 @@ rules:
   - endpoints
   - namespaces
   - services
+  - secrets
+  - configmaps
   verbs:
   - create
   - delete

deploy/operator/00_operator_remote.clusterrole_def.yaml

@@ -47,6 +47,8 @@ rules:
   - endpoints
   - namespaces
   - services
+  - secrets
+  - configmaps
   verbs:
   - create
   - delete

helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml

@@ -47,6 +47,8 @@ rules:
   - endpoints
   - namespaces
   - services
+  - secrets
+  - configmaps
   verbs:
   - create
   - delete

pkg/cmd/operator/operator.go

@@ -325,6 +325,18 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS
 		),
 	)
 
+	remoteOperatorManagedResourcesOnlyInformer := remoteinformers.NewSharedInformerFactoryWithOptions[kubernetes.Interface](
+		&o.clusterKubeClient,
+		resyncPeriod,
+		remoteinformers.WithTweakListOptions[kubernetes.Interface](
+			func(options *metav1.ListOptions) {
+				options.LabelSelector = labels.SelectorFromSet(map[string]string{
+					naming.KubernetesManagedByLabel: naming.RemoteOperatorAppNameWithDomain,
+				}).String()
+			},
+		),
+	)
+
 	scyllaOperatorConfigInformers := scyllainformers.NewSharedInformerFactoryWithOptions(o.scyllaClient, resyncPeriod, scyllainformers.WithTweakListOptions(
 		func(options *metav1.ListOptions) {
 			options.FieldSelector = fields.OneTermEqualSelector("metadata.name", naming.SingletonName).String()
@@ -476,6 +488,8 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS
 		&o.clusterScyllaClient,
 		scyllaInformers.Scylla().V1alpha1().ScyllaDBClusters(),
 		scyllaInformers.Scylla().V1alpha1().ScyllaOperatorConfigs(),
+		kubeInformers.Core().V1().ConfigMaps(),
+		kubeInformers.Core().V1().Secrets(),
 		remoteScyllaInformer.ForResource(&scyllav1alpha1.RemoteOwner{}, remoteinformers.ClusterListWatch[scyllaversionedclient.Interface]{
 			ListFunc: func(client remoteclient.ClusterClientInterface[scyllaversionedclient.Interface], cluster, ns string) cache.ListFunc {
 				return func(options metav1.ListOptions) (runtime.Object, error) {
@@ -616,6 +630,46 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS
 				}
 			},
 		}),
+		remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.ConfigMap{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{
+			ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc {
+				return func(options metav1.ListOptions) (runtime.Object, error) {
+					clusterClient, err := client.Cluster(cluster)
+					if err != nil {
+						return nil, err
+					}
+					return clusterClient.CoreV1().ConfigMaps(ns).List(ctx, options)
+				}
+			},
+			WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc {
+				return func(options metav1.ListOptions) (watch.Interface, error) {
+					clusterClient, err := client.Cluster(cluster)
+					if err != nil {
+						return nil, err
+					}
+					return clusterClient.CoreV1().ConfigMaps(ns).Watch(ctx, options)
+				}
+			},
+		}),
+		remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.Secret{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{
+			ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc {
+				return func(options metav1.ListOptions) (runtime.Object, error) {
+					clusterClient, err := client.Cluster(cluster)
+					if err != nil {
+						return nil, err
+					}
+					return clusterClient.CoreV1().Secrets(ns).List(ctx, options)
+				}
+			},
+			WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc {
+				return func(options metav1.ListOptions) (watch.Interface, error) {
+					clusterClient, err := client.Cluster(cluster)
+					if err != nil {
+						return nil, err
+					}
+					return clusterClient.CoreV1().Secrets(ns).Watch(ctx, options)
+				}
+			},
+		}),
 	)
 	if err != nil {
 		return fmt.Errorf("can't create ScyllaDBCluster controller: %w", err)
@@ -672,6 +726,12 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS
 		remoteScyllaPodInformer.Start(ctx.Done())
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		remoteOperatorManagedResourcesOnlyInformer.Start(ctx.Done())
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()

pkg/controller/scylladbcluster/conditions.go

@@ -15,4 +15,8 @@ const (
 	remoteEndpointsControllerDegradedCondition             = "RemoteEndpointsControllerDegraded"
 	scyllaDBClusterFinalizerProgressingCondition           = "ScyllaDBClusterFinalizerProgressing"
 	scyllaDBClusterFinalizerDegradedCondition              = "ScyllaDBClusterFinalizerDegraded"
+	remoteConfigMapControllerProgressingCondition          = "RemoteConfigMapControllerProgressing"
+	remoteConfigMapControllerDegradedCondition             = "RemoteConfigMapControllerDegraded"
+	remoteSecretControllerProgressingCondition             = "RemoteSecretControllerProgressing"
+	remoteSecretControllerDegradedCondition                = "RemoteSecretControllerDegraded"
 )

pkg/controller/scylladbcluster/controller.go

@@ -6,6 +6,8 @@ import (
 	"sync"
 	"time"
 
+	corev1informers "k8s.io/client-go/informers/core/v1"
+
 	scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1"
 	scyllaclient "github.com/scylladb/scylla-operator/pkg/client/scylla/clientset/versioned"
 	scyllav1alpha1informers "github.com/scylladb/scylla-operator/pkg/client/scylla/informers/externalversions/scylla/v1alpha1"
@@ -54,6 +56,8 @@ type Controller struct {
 
 	scyllaDBClusterLister      scyllav1alpha1listers.ScyllaDBClusterLister
 	scyllaOperatorConfigLister scyllav1alpha1listers.ScyllaOperatorConfigLister
+	configMapLister            corev1listers.ConfigMapLister
+	secretLister               corev1listers.SecretLister
 
 	remoteRemoteOwnerLister        remotelister.GenericClusterLister[scyllav1alpha1listers.RemoteOwnerLister]
 	remoteScyllaDBDatacenterLister remotelister.GenericClusterLister[scyllav1alpha1listers.ScyllaDBDatacenterLister]
@@ -62,6 +66,8 @@ type Controller struct {
 	remoteEndpointSliceLister      remotelister.GenericClusterLister[discoveryv1listers.EndpointSliceLister]
 	remoteEndpointsLister          remotelister.GenericClusterLister[corev1listers.EndpointsLister]
 	remotePodLister                remotelister.GenericClusterLister[corev1listers.PodLister]
+	remoteConfigMapLister          remotelister.GenericClusterLister[corev1listers.ConfigMapLister]
+	remoteSecretLister             remotelister.GenericClusterLister[corev1listers.SecretLister]
 
 	cachesToSync []cache.InformerSynced
 
@@ -78,13 +84,17 @@ func NewController(
 	scyllaRemoteClient remoteclient.ClusterClientInterface[scyllaclient.Interface],
 	scyllaDBClusterInformer scyllav1alpha1informers.ScyllaDBClusterInformer,
 	scyllaOperatorConfigInformer scyllav1alpha1informers.ScyllaOperatorConfigInformer,
+	configMapInformer corev1informers.ConfigMapInformer,
+	secretInformer corev1informers.SecretInformer,
 	remoteRemoteOwnerInformer remoteinformers.GenericClusterInformer,
 	remoteScyllaDBDatacenterInformer remoteinformers.GenericClusterInformer,
 	remoteNamespaceInformer remoteinformers.GenericClusterInformer,
 	remoteServiceInformer remoteinformers.GenericClusterInformer,
 	remoteEndpointSliceInformer remoteinformers.GenericClusterInformer,
 	remoteEndpointsInformer remoteinformers.GenericClusterInformer,
 	remotePodInformer remoteinformers.GenericClusterInformer,
+	remoteConfigMapInformer remoteinformers.GenericClusterInformer,
+	remoteSecretInformer remoteinformers.GenericClusterInformer,
 ) (*Controller, error) {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartStructuredLogging(0)
@@ -98,6 +108,8 @@ func NewController(
 
 		scyllaDBClusterLister:      scyllaDBClusterInformer.Lister(),
 		scyllaOperatorConfigLister: scyllaOperatorConfigInformer.Lister(),
+		configMapLister:            configMapInformer.Lister(),
+		secretLister:               secretInformer.Lister(),
 
 		remoteRemoteOwnerLister:        remotelister.NewClusterLister(scyllav1alpha1listers.NewRemoteOwnerLister, remoteRemoteOwnerInformer.Indexer().Cluster),
 		remoteScyllaDBDatacenterLister: remotelister.NewClusterLister(scyllav1alpha1listers.NewScyllaDBDatacenterLister, remoteScyllaDBDatacenterInformer.Indexer().Cluster),
@@ -106,16 +118,23 @@ func NewController(
 		remoteEndpointSliceLister:      remotelister.NewClusterLister(discoveryv1listers.NewEndpointSliceLister, remoteEndpointSliceInformer.Indexer().Cluster),
 		remoteEndpointsLister:          remotelister.NewClusterLister(corev1listers.NewEndpointsLister, remoteEndpointsInformer.Indexer().Cluster),
 		remotePodLister:                remotelister.NewClusterLister(corev1listers.NewPodLister, remotePodInformer.Indexer().Cluster),
+		remoteConfigMapLister:          remotelister.NewClusterLister(corev1listers.NewConfigMapLister, remoteConfigMapInformer.Indexer().Cluster),
+		remoteSecretLister:             remotelister.NewClusterLister(corev1listers.NewSecretLister, remoteSecretInformer.Indexer().Cluster),
 
 		cachesToSync: []cache.InformerSynced{
 			scyllaDBClusterInformer.Informer().HasSynced,
+			scyllaOperatorConfigInformer.Informer().HasSynced,
+			configMapInformer.Informer().HasSynced,
+			secretInformer.Informer().HasSynced,
 			remoteRemoteOwnerInformer.Informer().HasSynced,
 			remoteScyllaDBDatacenterInformer.Informer().HasSynced,
 			remoteNamespaceInformer.Informer().HasSynced,
 			remoteServiceInformer.Informer().HasSynced,
 			remoteEndpointSliceInformer.Informer().HasSynced,
 			remoteEndpointsInformer.Informer().HasSynced,
 			remotePodInformer.Informer().HasSynced,
+			remoteConfigMapInformer.Informer().HasSynced,
+			remoteSecretInformer.Informer().HasSynced,
 		},
 
 		eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "scylladbcluster-controller"}),
@@ -154,6 +173,11 @@ func NewController(
 		errs = append(errs, fmt.Errorf("can't register to ScyllaDBCluster events: %w", err))
 	}
 
+	// Local ConfigMap and Secret handlers are skipped to optimize number of syncs which doesn't do anything.
+	// Applying configuration change requires rolling restart of ScyllaDBCluster, so these resources will be synced upon
+	// ScyllaDBCluster update.
+	// These could be added once ConfigMaps and Secrets would require immediate sync.
+
 	// TODO: add error handling once these start returning errors
 	remoteRemoteOwnerInformer.Informer().AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
@@ -211,6 +235,22 @@ func NewController(
 		},
 	)
 
+	remoteConfigMapInformer.Informer().AddEventHandler(
+		cache.ResourceEventHandlerFuncs{
+			AddFunc:    scc.addRemoteConfigMap,
+			UpdateFunc: scc.updateRemoteConfigMap,
+			DeleteFunc: scc.deleteRemoteConfigMap,
+		},
+	)
+
+	remoteSecretInformer.Informer().AddEventHandler(
+		cache.ResourceEventHandlerFuncs{
+			AddFunc:    scc.addRemoteSecret,
+			UpdateFunc: scc.updateRemoteSecret,
+			DeleteFunc: scc.deleteRemoteSecret,
+		},
+	)
+
 	err = utilerrors.NewAggregate(errs)
 	if err != nil {
 		return nil, fmt.Errorf("can't register event handlers: %w", err)
@@ -540,3 +580,49 @@ func (scc *Controller) deleteRemotePod(obj interface{}) {
 		scc.enqueueThroughParentLabel,
 	)
 }
+
+func (scc *Controller) addRemoteConfigMap(obj interface{}) {
+	scc.handlers.HandleAdd(
+		obj.(*corev1.ConfigMap),
+		scc.enqueueThroughParentLabel,
+	)
+}
+
+func (scc *Controller) updateRemoteConfigMap(old, cur interface{}) {
+	scc.handlers.HandleUpdate(
+		old.(*corev1.ConfigMap),
+		cur.(*corev1.ConfigMap),
+		scc.enqueueThroughParentLabel,
+		scc.deleteRemoteConfigMap,
+	)
+}
+
+func (scc *Controller) deleteRemoteConfigMap(obj interface{}) {
+	scc.handlers.HandleDelete(
+		obj,
+		scc.enqueueThroughParentLabel,
+	)
+}
+
+func (scc *Controller) addRemoteSecret(obj interface{}) {
+	scc.handlers.HandleAdd(
+		obj.(*corev1.Secret),
+		scc.enqueueThroughParentLabel,
+	)
+}
+
+func (scc *Controller) updateRemoteSecret(old, cur interface{}) {
+	scc.handlers.HandleUpdate(
+		old.(*corev1.Secret),
+		cur.(*corev1.Secret),
+		scc.enqueueThroughParentLabel,
+		scc.deleteRemoteSecret,
+	)
+}
+
+func (scc *Controller) deleteRemoteSecret(obj interface{}) {
+	scc.handlers.HandleDelete(
+		obj,
+		scc.enqueueThroughParentLabel,
+	)
+}