From abe378066e4261754e995358ccd83f0aa22f3e7f Mon Sep 17 00:00:00 2001 From: Maciej Zimnoch Date: Wed, 26 Feb 2025 17:50:39 +0100 Subject: [PATCH 1/3] Add label to all remotely managed resources Added `app.kubernetes.io/managed-by: remote.scylla-operator.scylladb.com` to all remotely managed resources. It helps with setting up remote Informers caching only resources created by Operator to limit memory footprint. --- pkg/controller/scylladbcluster/resource_test.go | 13 +++++++++++++ pkg/naming/constants.go | 7 ++++++- pkg/naming/labels.go | 11 ++++++----- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pkg/controller/scylladbcluster/resource_test.go b/pkg/controller/scylladbcluster/resource_test.go index 54bc5681c69..b1746ac6d4c 100644 --- a/pkg/controller/scylladbcluster/resource_test.go +++ b/pkg/controller/scylladbcluster/resource_test.go @@ -82,6 +82,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -97,6 +98,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -112,6 +114,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -210,6 +213,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -221,6 +225,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -232,6 +237,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -475,6 +481,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -497,6 +504,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -521,6 +529,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -543,6 +552,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -567,6 +577,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -589,6 +600,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -838,6 +850,7 @@ func TestMakeScyllaDBDatacenters(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-name": "cluster", "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, Annotations: map[string]string{}, OwnerReferences: newBasicOwnerReference(namespace), diff --git a/pkg/naming/constants.go b/pkg/naming/constants.go index 4d07b43d98a..4f317d0a6e7 100644 --- a/pkg/naming/constants.go +++ b/pkg/naming/constants.go @@ -231,10 +231,15 @@ const ( ) const ( - OperatorAppNameWithDomain = "scylla-operator.scylladb.com" + OperatorAppNameWithDomain = "scylla-operator.scylladb.com" + RemoteOperatorAppNameWithDomain = "remote.scylla-operator.scylladb.com" ) const ( RemoteKubernetesClusterFinalizer = "scylla-operator.scylladb.com/remotekubernetescluster-protection" ScyllaDBClusterFinalizer = "scylla-operator.scylladb.com/scylladbcluster-protection" ) + +const ( + KubernetesManagedByLabel = "app.kubernetes.io/managed-by" +) diff --git a/pkg/naming/labels.go b/pkg/naming/labels.go index d87129be604..e588041d5e0 100644 --- a/pkg/naming/labels.go +++ b/pkg/naming/labels.go @@ -91,9 +91,10 @@ func mergeLabels(l1, l2 map[string]string) map[string]string { return res } -func ManagedResourcesLabels(managingClusterDomain string) map[string]string { +func RemoteManagedResourcesLabels(managingClusterDomain string) map[string]string { return map[string]string{ - ManagedByClusterLabel: managingClusterDomain, + KubernetesManagedByLabel: RemoteOperatorAppNameWithDomain, + ManagedByClusterLabel: managingClusterDomain, } } @@ -113,7 +114,7 @@ func ScyllaDBClusterDatacenterLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scy if dc.Metadata != nil { maps.Copy(dcLabels, dc.Metadata.Labels) } - maps.Copy(dcLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(dcLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(dcLabels, ScyllaDBClusterSelectorLabels(sc)) dcLabels[ParentClusterDatacenterNameLabel] = dc.Name return dcLabels @@ -155,7 +156,7 @@ func ScyllaDBClusterDatacenterEndpointsLabels(sc *scyllav1alpha1.ScyllaDBCluster if dc.Metadata != nil { maps.Copy(dcLabels, dc.Metadata.Labels) } - maps.Copy(dcLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(dcLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(dcLabels, ScyllaDBClusterEndpointsSelectorLabels(sc)) dcLabels[ParentClusterDatacenterNameLabel] = dc.Name return dcLabels @@ -200,7 +201,7 @@ func RemoteOwnerSelectorLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scyllav1a func RemoteOwnerLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scyllav1alpha1.ScyllaDBClusterDatacenter, managingClusterDomain string) map[string]string { remoteOwnerLabels := make(map[string]string) - maps.Copy(remoteOwnerLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(remoteOwnerLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(remoteOwnerLabels, RemoteOwnerSelectorLabels(sc, dc)) return remoteOwnerLabels From baa868d0dfc793671340194cd4721ff3d9b3b2c1 Mon Sep 17 00:00:00 2001 From: Maciej Zimnoch Date: Wed, 26 Feb 2025 17:50:23 +0100 Subject: [PATCH 2/3] Mirror ConfigMaps and Secrets referenced in ScyllaDBCluster into remote datacenters The ScyllaDBCluster controller has been extended with a new capability of mirroring the ConfigMaps and Secrets referenced by the ScyllaDBCluster, into remote datacenters. This improvement provides users with the convenience of managing the configuration of an entire cluster from a single centralized location. Note: any modifications made to the ConfigMaps or Secrets do not automatically initiate a rollout to apply those changes. A manual trigger is required to implement the updates - for example using `forceRedeploymentReason`. --- .../operator_remote.clusterrole_def.yaml | 2 + pkg/cmd/operator/operator.go | 60 +++++ pkg/controller/scylladbcluster/conditions.go | 4 + pkg/controller/scylladbcluster/controller.go | 86 +++++++ pkg/controller/scylladbcluster/resource.go | 191 ++++++++++++++++ pkg/controller/scylladbcluster/sync.go | 72 ++++++ .../scylladbcluster/sync_configmap.go | 88 ++++++++ .../scylladbcluster/sync_secrets.go | 88 ++++++++ pkg/scyllaclient/config_client.go | 9 + .../multidatacenter_scylladbcluster_config.go | 212 ++++++++++++++++++ .../verification/scylladbcluster/verify.go | 16 ++ 11 files changed, 828 insertions(+) create mode 100644 pkg/controller/scylladbcluster/sync_configmap.go create mode 100644 pkg/controller/scylladbcluster/sync_secrets.go create mode 100644 test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go diff --git a/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml b/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml index 4651a461987..e1022ec430b 100644 --- a/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml +++ b/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml @@ -47,6 +47,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete diff --git a/pkg/cmd/operator/operator.go b/pkg/cmd/operator/operator.go index 1e84519bca7..e568bbd2fd4 100644 --- a/pkg/cmd/operator/operator.go +++ b/pkg/cmd/operator/operator.go @@ -325,6 +325,18 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS ), ) + remoteOperatorManagedResourcesOnlyInformer := remoteinformers.NewSharedInformerFactoryWithOptions[kubernetes.Interface]( + &o.clusterKubeClient, + resyncPeriod, + remoteinformers.WithTweakListOptions[kubernetes.Interface]( + func(options *metav1.ListOptions) { + options.LabelSelector = labels.SelectorFromSet(map[string]string{ + naming.KubernetesManagedByLabel: naming.RemoteOperatorAppNameWithDomain, + }).String() + }, + ), + ) + scyllaOperatorConfigInformers := scyllainformers.NewSharedInformerFactoryWithOptions(o.scyllaClient, resyncPeriod, scyllainformers.WithTweakListOptions( func(options *metav1.ListOptions) { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", naming.SingletonName).String() @@ -476,6 +488,8 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS &o.clusterScyllaClient, scyllaInformers.Scylla().V1alpha1().ScyllaDBClusters(), scyllaInformers.Scylla().V1alpha1().ScyllaOperatorConfigs(), + kubeInformers.Core().V1().ConfigMaps(), + kubeInformers.Core().V1().Secrets(), remoteScyllaInformer.ForResource(&scyllav1alpha1.RemoteOwner{}, remoteinformers.ClusterListWatch[scyllaversionedclient.Interface]{ ListFunc: func(client remoteclient.ClusterClientInterface[scyllaversionedclient.Interface], cluster, ns string) cache.ListFunc { return func(options metav1.ListOptions) (runtime.Object, error) { @@ -616,6 +630,46 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS } }, }), + remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.ConfigMap{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{ + ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc { + return func(options metav1.ListOptions) (runtime.Object, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().ConfigMaps(ns).List(ctx, options) + } + }, + WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc { + return func(options metav1.ListOptions) (watch.Interface, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().ConfigMaps(ns).Watch(ctx, options) + } + }, + }), + remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.Secret{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{ + ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc { + return func(options metav1.ListOptions) (runtime.Object, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().Secrets(ns).List(ctx, options) + } + }, + WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc { + return func(options metav1.ListOptions) (watch.Interface, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().Secrets(ns).Watch(ctx, options) + } + }, + }), ) if err != nil { return fmt.Errorf("can't create ScyllaDBCluster controller: %w", err) @@ -672,6 +726,12 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS remoteScyllaPodInformer.Start(ctx.Done()) }() + wg.Add(1) + go func() { + defer wg.Done() + remoteOperatorManagedResourcesOnlyInformer.Start(ctx.Done()) + }() + wg.Add(1) go func() { defer wg.Done() diff --git a/pkg/controller/scylladbcluster/conditions.go b/pkg/controller/scylladbcluster/conditions.go index 68880131601..7f1e455ee4d 100644 --- a/pkg/controller/scylladbcluster/conditions.go +++ b/pkg/controller/scylladbcluster/conditions.go @@ -15,4 +15,8 @@ const ( remoteEndpointsControllerDegradedCondition = "RemoteEndpointsControllerDegraded" scyllaDBClusterFinalizerProgressingCondition = "ScyllaDBClusterFinalizerProgressing" scyllaDBClusterFinalizerDegradedCondition = "ScyllaDBClusterFinalizerDegraded" + remoteConfigMapControllerProgressingCondition = "RemoteConfigMapControllerProgressing" + remoteConfigMapControllerDegradedCondition = "RemoteConfigMapControllerDegraded" + remoteSecretControllerProgressingCondition = "RemoteSecretControllerProgressing" + remoteSecretControllerDegradedCondition = "RemoteSecretControllerDegraded" ) diff --git a/pkg/controller/scylladbcluster/controller.go b/pkg/controller/scylladbcluster/controller.go index 2caada2bb38..0ba4f19c8d8 100644 --- a/pkg/controller/scylladbcluster/controller.go +++ b/pkg/controller/scylladbcluster/controller.go @@ -6,6 +6,8 @@ import ( "sync" "time" + corev1informers "k8s.io/client-go/informers/core/v1" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" scyllaclient "github.com/scylladb/scylla-operator/pkg/client/scylla/clientset/versioned" scyllav1alpha1informers "github.com/scylladb/scylla-operator/pkg/client/scylla/informers/externalversions/scylla/v1alpha1" @@ -54,6 +56,8 @@ type Controller struct { scyllaDBClusterLister scyllav1alpha1listers.ScyllaDBClusterLister scyllaOperatorConfigLister scyllav1alpha1listers.ScyllaOperatorConfigLister + configMapLister corev1listers.ConfigMapLister + secretLister corev1listers.SecretLister remoteRemoteOwnerLister remotelister.GenericClusterLister[scyllav1alpha1listers.RemoteOwnerLister] remoteScyllaDBDatacenterLister remotelister.GenericClusterLister[scyllav1alpha1listers.ScyllaDBDatacenterLister] @@ -62,6 +66,8 @@ type Controller struct { remoteEndpointSliceLister remotelister.GenericClusterLister[discoveryv1listers.EndpointSliceLister] remoteEndpointsLister remotelister.GenericClusterLister[corev1listers.EndpointsLister] remotePodLister remotelister.GenericClusterLister[corev1listers.PodLister] + remoteConfigMapLister remotelister.GenericClusterLister[corev1listers.ConfigMapLister] + remoteSecretLister remotelister.GenericClusterLister[corev1listers.SecretLister] cachesToSync []cache.InformerSynced @@ -78,6 +84,8 @@ func NewController( scyllaRemoteClient remoteclient.ClusterClientInterface[scyllaclient.Interface], scyllaDBClusterInformer scyllav1alpha1informers.ScyllaDBClusterInformer, scyllaOperatorConfigInformer scyllav1alpha1informers.ScyllaOperatorConfigInformer, + configMapInformer corev1informers.ConfigMapInformer, + secretInformer corev1informers.SecretInformer, remoteRemoteOwnerInformer remoteinformers.GenericClusterInformer, remoteScyllaDBDatacenterInformer remoteinformers.GenericClusterInformer, remoteNamespaceInformer remoteinformers.GenericClusterInformer, @@ -85,6 +93,8 @@ func NewController( remoteEndpointSliceInformer remoteinformers.GenericClusterInformer, remoteEndpointsInformer remoteinformers.GenericClusterInformer, remotePodInformer remoteinformers.GenericClusterInformer, + remoteConfigMapInformer remoteinformers.GenericClusterInformer, + remoteSecretInformer remoteinformers.GenericClusterInformer, ) (*Controller, error) { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartStructuredLogging(0) @@ -98,6 +108,8 @@ func NewController( scyllaDBClusterLister: scyllaDBClusterInformer.Lister(), scyllaOperatorConfigLister: scyllaOperatorConfigInformer.Lister(), + configMapLister: configMapInformer.Lister(), + secretLister: secretInformer.Lister(), remoteRemoteOwnerLister: remotelister.NewClusterLister(scyllav1alpha1listers.NewRemoteOwnerLister, remoteRemoteOwnerInformer.Indexer().Cluster), remoteScyllaDBDatacenterLister: remotelister.NewClusterLister(scyllav1alpha1listers.NewScyllaDBDatacenterLister, remoteScyllaDBDatacenterInformer.Indexer().Cluster), @@ -106,9 +118,14 @@ func NewController( remoteEndpointSliceLister: remotelister.NewClusterLister(discoveryv1listers.NewEndpointSliceLister, remoteEndpointSliceInformer.Indexer().Cluster), remoteEndpointsLister: remotelister.NewClusterLister(corev1listers.NewEndpointsLister, remoteEndpointsInformer.Indexer().Cluster), remotePodLister: remotelister.NewClusterLister(corev1listers.NewPodLister, remotePodInformer.Indexer().Cluster), + remoteConfigMapLister: remotelister.NewClusterLister(corev1listers.NewConfigMapLister, remoteConfigMapInformer.Indexer().Cluster), + remoteSecretLister: remotelister.NewClusterLister(corev1listers.NewSecretLister, remoteSecretInformer.Indexer().Cluster), cachesToSync: []cache.InformerSynced{ scyllaDBClusterInformer.Informer().HasSynced, + scyllaOperatorConfigInformer.Informer().HasSynced, + configMapInformer.Informer().HasSynced, + secretInformer.Informer().HasSynced, remoteRemoteOwnerInformer.Informer().HasSynced, remoteScyllaDBDatacenterInformer.Informer().HasSynced, remoteNamespaceInformer.Informer().HasSynced, @@ -116,6 +133,8 @@ func NewController( remoteEndpointSliceInformer.Informer().HasSynced, remoteEndpointsInformer.Informer().HasSynced, remotePodInformer.Informer().HasSynced, + remoteConfigMapInformer.Informer().HasSynced, + remoteSecretInformer.Informer().HasSynced, }, eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "scylladbcluster-controller"}), @@ -154,6 +173,11 @@ func NewController( errs = append(errs, fmt.Errorf("can't register to ScyllaDBCluster events: %w", err)) } + // Local ConfigMap and Secret handlers are skipped to optimize number of syncs which doesn't do anything. + // Applying configuration change requires rolling restart of ScyllaDBCluster, so these resources will be synced upon + // ScyllaDBCluster update. + // These could be added once ConfigMaps and Secrets would require immediate sync. + // TODO: add error handling once these start returning errors remoteRemoteOwnerInformer.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -211,6 +235,22 @@ func NewController( }, ) + remoteConfigMapInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: scc.addRemoteConfigMap, + UpdateFunc: scc.updateRemoteConfigMap, + DeleteFunc: scc.deleteRemoteConfigMap, + }, + ) + + remoteSecretInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: scc.addRemoteSecret, + UpdateFunc: scc.updateRemoteSecret, + DeleteFunc: scc.deleteRemoteSecret, + }, + ) + err = utilerrors.NewAggregate(errs) if err != nil { return nil, fmt.Errorf("can't register event handlers: %w", err) @@ -540,3 +580,49 @@ func (scc *Controller) deleteRemotePod(obj interface{}) { scc.enqueueThroughParentLabel, ) } + +func (scc *Controller) addRemoteConfigMap(obj interface{}) { + scc.handlers.HandleAdd( + obj.(*corev1.ConfigMap), + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) updateRemoteConfigMap(old, cur interface{}) { + scc.handlers.HandleUpdate( + old.(*corev1.ConfigMap), + cur.(*corev1.ConfigMap), + scc.enqueueThroughParentLabel, + scc.deleteRemoteConfigMap, + ) +} + +func (scc *Controller) deleteRemoteConfigMap(obj interface{}) { + scc.handlers.HandleDelete( + obj, + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) addRemoteSecret(obj interface{}) { + scc.handlers.HandleAdd( + obj.(*corev1.Secret), + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) updateRemoteSecret(old, cur interface{}) { + scc.handlers.HandleUpdate( + old.(*corev1.Secret), + cur.(*corev1.Secret), + scc.enqueueThroughParentLabel, + scc.deleteRemoteSecret, + ) +} + +func (scc *Controller) deleteRemoteSecret(obj interface{}) { + scc.handlers.HandleDelete( + obj, + scc.enqueueThroughParentLabel, + ) +} diff --git a/pkg/controller/scylladbcluster/resource.go b/pkg/controller/scylladbcluster/resource.go index d73f3f359b6..39f42fdef18 100644 --- a/pkg/controller/scylladbcluster/resource.go +++ b/pkg/controller/scylladbcluster/resource.go @@ -2,6 +2,8 @@ package scylladbcluster import ( "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/errors" "maps" "sort" "strings" @@ -911,3 +913,192 @@ func getRemoteNamespaceAndController(controllerProgressingType string, sc *scyll return progressingConditions, remoteNamespace, remoteController } + +func MakeRemoteConfigMaps(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localConfigMapLister corev1listers.ConfigMapLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.ConfigMap, error) { + requiredRemoteConfigMaps := make(map[string][]*corev1.ConfigMap) + + progressingConditions, mirroredConfigMaps, err := makeMirroredRemoteConfigMaps(sc, remoteNamespaces, remoteControllers, localConfigMapLister, managingClusterDomain) + if err != nil { + return progressingConditions, nil, fmt.Errorf("can't make mirrored configmaps: %w", err) + } + + for k, v := range mirroredConfigMaps { + requiredRemoteConfigMaps[k] = append(requiredRemoteConfigMaps[k], v...) + } + + return progressingConditions, requiredRemoteConfigMaps, nil +} + +func makeMirroredRemoteConfigMaps(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localConfigMapLister corev1listers.ConfigMapLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.ConfigMap, error) { + var errs []error + var progressingConditions []metav1.Condition + requiredRemoteConfigMaps := make(map[string][]*corev1.ConfigMap, len(sc.Spec.Datacenters)) + + for _, dc := range sc.Spec.Datacenters { + dcProgressingConditions, remoteNamespace, remoteController := getRemoteNamespaceAndController( + remoteConfigMapControllerProgressingCondition, + sc, + dc.RemoteKubernetesClusterName, + remoteNamespaces, + remoteControllers, + ) + if len(dcProgressingConditions) > 0 { + progressingConditions = append(progressingConditions, dcProgressingConditions...) + continue + } + + var configMapsToMirror []string + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.ScyllaDB != nil && sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef) + } + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB.CustomConfigMapRef) + } + + if dc.ScyllaDB != nil && dc.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *dc.ScyllaDB.CustomConfigMapRef) + } + + if dc.RackTemplate != nil && dc.RackTemplate.ScyllaDB != nil && dc.RackTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *dc.RackTemplate.ScyllaDB.CustomConfigMapRef) + } + + for _, rack := range dc.Racks { + if rack.ScyllaDB != nil && rack.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *rack.ScyllaDB.CustomConfigMapRef) + } + } + + for _, cmName := range configMapsToMirror { + localConfigMap, err := localConfigMapLister.ConfigMaps(sc.Namespace).Get(cmName) + if err != nil { + if apierrors.IsNotFound(err) { + progressingConditions = append(progressingConditions, metav1.Condition{ + Type: remoteConfigMapControllerProgressingCondition, + Status: metav1.ConditionTrue, + Reason: "WaitingForConfigMap", + Message: fmt.Sprintf("Waiting for ConfigMap %q to exist.", naming.ManualRef(sc.Namespace, cmName)), + ObservedGeneration: sc.Generation, + }) + continue + } + errs = append(errs, fmt.Errorf("can't get %q ConfigMap: %w", naming.ManualRef(sc.Namespace, cmName), err)) + continue + } + + requiredRemoteConfigMaps[dc.RemoteKubernetesClusterName] = append(requiredRemoteConfigMaps[dc.RemoteKubernetesClusterName], &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: remoteNamespace.Name, + Labels: naming.ScyllaDBClusterDatacenterLabels(sc, &dc, managingClusterDomain), + Annotations: naming.ScyllaDBClusterDatacenterAnnotations(sc, &dc), + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(remoteController, remoteControllerGVK)}, + }, + Immutable: localConfigMap.Immutable, + Data: maps.Clone(localConfigMap.Data), + BinaryData: maps.Clone(localConfigMap.BinaryData), + }) + } + } + + err := errors.NewAggregate(errs) + if err != nil { + return progressingConditions, requiredRemoteConfigMaps, fmt.Errorf("can't make mirrored ConfigMaps: %w", err) + } + + return progressingConditions, requiredRemoteConfigMaps, nil +} + +func MakeRemoteSecrets(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localSecretLister corev1listers.SecretLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.Secret, error) { + requiredRemoteSecrets := make(map[string][]*corev1.Secret) + + progressingConditions, mirroredSecrets, err := makeMirroredRemoteSecrets(sc, remoteNamespaces, remoteControllers, localSecretLister, managingClusterDomain) + if err != nil { + return progressingConditions, nil, fmt.Errorf("can't make mirrored secrets: %w", err) + } + + for k, v := range mirroredSecrets { + requiredRemoteSecrets[k] = append(requiredRemoteSecrets[k], v...) + } + + return progressingConditions, requiredRemoteSecrets, nil +} + +func makeMirroredRemoteSecrets(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localSecretLister corev1listers.SecretLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.Secret, error) { + var progressingConditions []metav1.Condition + requiredRemoteSecrets := make(map[string][]*corev1.Secret, len(sc.Spec.Datacenters)) + + var errs []error + for _, dc := range sc.Spec.Datacenters { + dcProgressingConditions, remoteNamespace, remoteController := getRemoteNamespaceAndController( + remoteConfigMapControllerProgressingCondition, + sc, + dc.RemoteKubernetesClusterName, + remoteNamespaces, + remoteControllers, + ) + if len(dcProgressingConditions) > 0 { + progressingConditions = append(progressingConditions, dcProgressingConditions...) + continue + } + + var secretsToMirror []string + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent != nil && sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if dc.ScyllaDBManagerAgent != nil && dc.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *dc.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if dc.RackTemplate != nil && dc.RackTemplate.ScyllaDBManagerAgent != nil && dc.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *dc.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + for _, rack := range dc.Racks { + if rack.ScyllaDBManagerAgent != nil && rack.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *rack.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + } + + for _, secretName := range secretsToMirror { + localSecret, err := localSecretLister.Secrets(sc.Namespace).Get(secretName) + if err != nil { + if apierrors.IsNotFound(err) { + progressingConditions = append(progressingConditions, metav1.Condition{ + Type: remoteSecretControllerProgressingCondition, + Status: metav1.ConditionTrue, + Reason: "WaitingForSecret", + Message: fmt.Sprintf("Waiting for Secret %q to exist.", naming.ManualRef(sc.Namespace, secretName)), + ObservedGeneration: sc.Generation, + }) + continue + } + errs = append(errs, fmt.Errorf("can't get %q Secret: %w", naming.ManualRef(sc.Namespace, secretName), err)) + continue + } + + requiredRemoteSecrets[dc.RemoteKubernetesClusterName] = append(requiredRemoteSecrets[dc.RemoteKubernetesClusterName], &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: remoteNamespace.Name, + Labels: naming.ScyllaDBClusterDatacenterLabels(sc, &dc, managingClusterDomain), + Annotations: naming.ScyllaDBClusterDatacenterAnnotations(sc, &dc), + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(remoteController, remoteControllerGVK)}, + }, + Immutable: localSecret.Immutable, + Data: maps.Clone(localSecret.Data), + Type: localSecret.Type, + }) + } + } + + return progressingConditions, requiredRemoteSecrets, nil +} diff --git a/pkg/controller/scylladbcluster/sync.go b/pkg/controller/scylladbcluster/sync.go index 49f29493e16..b508803b08e 100644 --- a/pkg/controller/scylladbcluster/sync.go +++ b/pkg/controller/scylladbcluster/sync.go @@ -158,6 +158,52 @@ func (scc *Controller) sync(ctx context.Context, key string) error { objectErrs = append(objectErrs, fmt.Errorf("can't get remote endpoints: %w", err)) } + remoteConfigMapMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *corev1.ConfigMap](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *corev1.ConfigMap]{ + ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *corev1.ConfigMap], error) { + ns, ok := remoteNamespaces[clusterName] + if !ok { + return nil, nil + } + + kubeClusterClient, scyllaClusterClient, err := scc.getClusterClients(clusterName) + if err != nil { + return nil, fmt.Errorf("can't get cluster %q clients: %w", clusterName, err) + } + + return &controllerhelpers.ControlleeManagerGetObjectsFuncs[remoteCT, *corev1.ConfigMap]{ + GetControllerUncachedFunc: scyllaClusterClient.ScyllaV1alpha1().RemoteOwners(ns.Name).Get, + ListObjectsFunc: scc.remoteConfigMapLister.Cluster(clusterName).ConfigMaps(ns.Name).List, + PatchObjectFunc: kubeClusterClient.CoreV1().ConfigMaps(ns.Name).Patch, + }, nil + }, + }) + if err != nil { + objectErrs = append(objectErrs, fmt.Errorf("can't get remote configmaps: %w", err)) + } + + remoteSecretMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *corev1.Secret](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *corev1.Secret]{ + ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *corev1.Secret], error) { + ns, ok := remoteNamespaces[clusterName] + if !ok { + return nil, nil + } + + kubeClusterClient, scyllaClusterClient, err := scc.getClusterClients(clusterName) + if err != nil { + return nil, fmt.Errorf("can't get cluster %q clients: %w", clusterName, err) + } + + return &controllerhelpers.ControlleeManagerGetObjectsFuncs[remoteCT, *corev1.Secret]{ + GetControllerUncachedFunc: scyllaClusterClient.ScyllaV1alpha1().RemoteOwners(ns.Name).Get, + ListObjectsFunc: scc.remoteSecretLister.Cluster(clusterName).Secrets(ns.Name).List, + PatchObjectFunc: kubeClusterClient.CoreV1().Secrets(ns.Name).Patch, + }, nil + }, + }) + if err != nil { + objectErrs = append(objectErrs, fmt.Errorf("can't get remote configmaps: %w", err)) + } + remoteScyllaDBDatacenterMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter]{ ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter], error) { ns, ok := remoteNamespaces[clusterName] @@ -285,6 +331,32 @@ func (scc *Controller) sync(ctx context.Context, key string) error { errs = append(errs, fmt.Errorf("can't sync remote endpoints: %w", err)) } + err = controllerhelpers.RunSync( + &status.Conditions, + remoteConfigMapControllerProgressingCondition, + remoteConfigMapControllerDegradedCondition, + sc.Generation, + func() ([]metav1.Condition, error) { + return scc.syncConfigMaps(ctx, sc, remoteNamespaces, remoteControllers, remoteConfigMapMap, managingClusterDomain) + }, + ) + if err != nil { + errs = append(errs, fmt.Errorf("can't sync remote configmaps: %w", err)) + } + + err = controllerhelpers.RunSync( + &status.Conditions, + remoteSecretControllerProgressingCondition, + remoteSecretControllerDegradedCondition, + sc.Generation, + func() ([]metav1.Condition, error) { + return scc.syncSecrets(ctx, sc, remoteNamespaces, remoteControllers, remoteSecretMap, managingClusterDomain) + }, + ) + if err != nil { + errs = append(errs, fmt.Errorf("can't sync remote secrets: %w", err)) + } + err = controllerhelpers.RunSync( &status.Conditions, remoteScyllaDBDatacenterControllerProgressingCondition, diff --git a/pkg/controller/scylladbcluster/sync_configmap.go b/pkg/controller/scylladbcluster/sync_configmap.go new file mode 100644 index 00000000000..e2364b6d637 --- /dev/null +++ b/pkg/controller/scylladbcluster/sync_configmap.go @@ -0,0 +1,88 @@ +package scylladbcluster + +import ( + "context" + "fmt" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/resourceapply" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +func (scc *Controller) syncConfigMaps( + ctx context.Context, + sc *scyllav1alpha1.ScyllaDBCluster, + remoteNamespaces map[string]*corev1.Namespace, + remoteControllers map[string]metav1.Object, + remoteConfigMaps map[string]map[string]*corev1.ConfigMap, + managingClusterDomain string, +) ([]metav1.Condition, error) { + progressingConditions, requiredConfigMaps, err := MakeRemoteConfigMaps(sc, remoteNamespaces, remoteControllers, scc.configMapLister, managingClusterDomain) + if err != nil { + return progressingConditions, fmt.Errorf("can't make required configmaps: %w", err) + } + + if len(progressingConditions) > 0 { + return progressingConditions, nil + } + + // Delete has to be the first action to avoid getting stuck on quota. + var deletionErrors []error + for _, dc := range sc.Spec.Datacenters { + ns, ok := remoteNamespaces[dc.RemoteKubernetesClusterName] + if !ok { + continue + } + + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + return nil, fmt.Errorf("can't get client to %q cluster: %w", dc.RemoteKubernetesClusterName, err) + } + + err = controllerhelpers.Prune(ctx, + requiredConfigMaps[dc.RemoteKubernetesClusterName], + remoteConfigMaps[dc.RemoteKubernetesClusterName], + &controllerhelpers.PruneControlFuncs{ + DeleteFunc: clusterClient.CoreV1().ConfigMaps(ns.Name).Delete, + }, + scc.eventRecorder, + ) + if err != nil { + return progressingConditions, fmt.Errorf("can't prune configmap(s) in %q Datacenter of %q ScyllaDBCluster: %w", dc.Name, naming.ObjRef(sc), err) + } + } + + if err := utilerrors.NewAggregate(deletionErrors); err != nil { + return nil, fmt.Errorf("can't prune remote configmap(s): %w", err) + } + + var errs []error + for _, dc := range sc.Spec.Datacenters { + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + errs = append(errs, fmt.Errorf("can't get client to %q cluster: %w", dc.Name, err)) + continue + } + + for _, cm := range requiredConfigMaps[dc.RemoteKubernetesClusterName] { + _, changed, err := resourceapply.ApplyConfigMap(ctx, clusterClient.CoreV1(), scc.remoteConfigMapLister.Cluster(dc.RemoteKubernetesClusterName), scc.eventRecorder, cm, resourceapply.ApplyOptions{}) + if changed { + controllerhelpers.AddGenericProgressingStatusCondition(&progressingConditions, remoteConfigMapControllerProgressingCondition, cm, "apply", sc.Generation) + } + if err != nil { + errs = append(errs, fmt.Errorf("can't apply configmap: %w", err)) + continue + } + } + } + + err = utilerrors.NewAggregate(errs) + if err != nil { + return progressingConditions, fmt.Errorf("can't apply configmap(s): %w", err) + } + + return progressingConditions, nil +} diff --git a/pkg/controller/scylladbcluster/sync_secrets.go b/pkg/controller/scylladbcluster/sync_secrets.go new file mode 100644 index 00000000000..4da26c36a5f --- /dev/null +++ b/pkg/controller/scylladbcluster/sync_secrets.go @@ -0,0 +1,88 @@ +package scylladbcluster + +import ( + "context" + "fmt" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/resourceapply" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +func (scc *Controller) syncSecrets( + ctx context.Context, + sc *scyllav1alpha1.ScyllaDBCluster, + remoteNamespaces map[string]*corev1.Namespace, + remoteControllers map[string]metav1.Object, + remoteSecrets map[string]map[string]*corev1.Secret, + managingClusterDomain string, +) ([]metav1.Condition, error) { + progressingConditions, requiredRemoteSecrets, err := MakeRemoteSecrets(sc, remoteNamespaces, remoteControllers, scc.secretLister, managingClusterDomain) + if err != nil { + return progressingConditions, fmt.Errorf("can't make remote secrets: %w", err) + } + + if len(progressingConditions) > 0 { + return progressingConditions, nil + } + + // Delete has to be the first action to avoid getting stuck on quota. + var deletionErrors []error + for _, dc := range sc.Spec.Datacenters { + ns, ok := remoteNamespaces[dc.RemoteKubernetesClusterName] + if !ok { + continue + } + + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + return nil, fmt.Errorf("can't get client to %q cluster: %w", dc.RemoteKubernetesClusterName, err) + } + + err = controllerhelpers.Prune(ctx, + requiredRemoteSecrets[dc.RemoteKubernetesClusterName], + remoteSecrets[dc.RemoteKubernetesClusterName], + &controllerhelpers.PruneControlFuncs{ + DeleteFunc: clusterClient.CoreV1().Secrets(ns.Name).Delete, + }, + scc.eventRecorder, + ) + if err != nil { + return progressingConditions, fmt.Errorf("can't prune secret(s) in %q Datacenter of %q ScyllaDBCluster: %w", dc.Name, naming.ObjRef(sc), err) + } + } + + if err := utilerrors.NewAggregate(deletionErrors); err != nil { + return nil, fmt.Errorf("can't prune remote secret(s): %w", err) + } + + var errs []error + for _, dc := range sc.Spec.Datacenters { + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + errs = append(errs, fmt.Errorf("can't get client to %q cluster: %w", dc.Name, err)) + continue + } + + for _, rs := range requiredRemoteSecrets[dc.RemoteKubernetesClusterName] { + _, changed, err := resourceapply.ApplySecret(ctx, clusterClient.CoreV1(), scc.remoteSecretLister.Cluster(dc.RemoteKubernetesClusterName), scc.eventRecorder, rs, resourceapply.ApplyOptions{}) + if changed { + controllerhelpers.AddGenericProgressingStatusCondition(&progressingConditions, remoteSecretControllerProgressingCondition, rs, "apply", sc.Generation) + } + if err != nil { + errs = append(errs, fmt.Errorf("can't apply secret: %w", err)) + continue + } + } + } + + err = utilerrors.NewAggregate(errs) + if err != nil { + return progressingConditions, fmt.Errorf("can't apply secret(s): %w", err) + } + + return progressingConditions, nil +} diff --git a/pkg/scyllaclient/config_client.go b/pkg/scyllaclient/config_client.go index 205119bc1a1..f613a09184c 100644 --- a/pkg/scyllaclient/config_client.go +++ b/pkg/scyllaclient/config_client.go @@ -79,3 +79,12 @@ func (c *ConfigClient) ReplaceNodeFirstBoot(ctx context.Context) (string, error) } return resp.Payload, nil } + +// ReadRequestTimeoutInMs returns value of "read_request_timeout_in_ms" config parameter. +func (c *ConfigClient) ReadRequestTimeoutInMs(ctx context.Context) (int64, error) { + resp, err := c.client.Config.FindConfigReadRequestTimeoutInMs(config.NewFindConfigReadRequestTimeoutInMsParamsWithContext(ctx)) + if err != nil { + return 0, fmt.Errorf("can't get read_request_timeout_in_ms: %w", err) + } + return resp.Payload, nil +} diff --git a/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go b/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go new file mode 100644 index 00000000000..4401d44bca5 --- /dev/null +++ b/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go @@ -0,0 +1,212 @@ +package multidatacenter + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/pointer" + "github.com/scylladb/scylla-operator/pkg/scyllaclient" + "github.com/scylladb/scylla-operator/test/e2e/framework" + "github.com/scylladb/scylla-operator/test/e2e/utils" + scylladbclusterverification "github.com/scylladb/scylla-operator/test/e2e/utils/verification/scylladbcluster" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = g.Describe("Multi datacenter ScyllaDBCluster", framework.MultiDatacenter, func() { + f := framework.NewFramework("scylladbcluster") + + g.It("should mirror ConfigMaps and Secrets referenced by ScyllaDBCluster into remote datacenters", func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + rkcs, rkcClusterMap, err := utils.SetUpRemoteKubernetesClustersFromRestConfigs(ctx, framework.TestContext.RestConfigs, f) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Creating configs for ScyllaDB and ScyllaDB Manager Agent") + + metaCluster := f.Cluster(0) + userNS, userClient, ok := metaCluster.DefaultNamespaceIfAny() + o.Expect(ok).To(o.BeTrue()) + + makeScyllaDBConfigMap := func(name, configValue string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: userNS.Name, + }, + Data: map[string]string{ + "scylla.yaml": fmt.Sprintf(`read_request_timeout_in_ms: %s`, configValue), + }, + } + } + + makeScyllaDBManagerAgentSecret := func(name, configValue string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: userNS.Name, + }, + StringData: map[string]string{ + "scylla-manager-agent.yaml": fmt.Sprintf(`auth_token: %s`, configValue), + }, + } + } + + scyllaDBDatacenterTemplateConfigMap := makeScyllaDBConfigMap("datacenter-template", "1111") + scyllaDBDatacenterTemplateRackTemplateConfigMap := makeScyllaDBConfigMap("datacenter-template-rack-template", "2222") + scyllaDBDatacenterRackTemplateConfigMap := makeScyllaDBConfigMap("datacenter-rack-template", "3333") + scyllaDBDatacenterDatacenterConfigMap := makeScyllaDBConfigMap("datacenter", "4444") + const scyllaDBRackReadRequestTimeoutInMs = 5555 + scyllaDBDatacenterRackConfigMap := makeScyllaDBConfigMap("rack", fmt.Sprintf("%d", scyllaDBRackReadRequestTimeoutInMs)) + + scyllaDBConfigMaps := []*corev1.ConfigMap{ + scyllaDBDatacenterTemplateConfigMap, + scyllaDBDatacenterTemplateRackTemplateConfigMap, + scyllaDBDatacenterRackTemplateConfigMap, + scyllaDBDatacenterDatacenterConfigMap, + scyllaDBDatacenterRackConfigMap, + } + + scyllaDBManagerAgentDatacenterTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-template", "aaaa") + scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-template-rack-template", "bbbb") + scyllaDBManagerAgentDatacenterRackTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-rack-template", "cccc") + scyllaDBManagerAgentDatacenterSecret := makeScyllaDBManagerAgentSecret("datacenter", "dddd") + const scyllaDBManagerAgentRackAuthToken = "eeee" + scyllaDBManagerAgentDatacenterRackSecret := makeScyllaDBManagerAgentSecret("rack", scyllaDBManagerAgentRackAuthToken) + + scyllaDBManagerAgentSecrets := []*corev1.Secret{ + scyllaDBManagerAgentDatacenterTemplateSecret, + scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret, + scyllaDBManagerAgentDatacenterRackTemplateSecret, + scyllaDBManagerAgentDatacenterSecret, + scyllaDBManagerAgentDatacenterRackSecret, + } + + for _, cm := range scyllaDBConfigMaps { + _, err := userClient.KubeClient().CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + for _, secret := range scyllaDBManagerAgentSecrets { + _, err := userClient.KubeClient().CoreV1().Secrets(secret.Namespace).Create(ctx, secret, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + framework.By("Creating ScyllaDBCluster") + sc := f.GetDefaultScyllaDBCluster(rkcs) + + sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef = pointer.Ptr(scyllaDBDatacenterTemplateConfigMap.Name) + sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterTemplateSecret.Name), + } + sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB = &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterTemplateRackTemplateConfigMap.Name), + } + sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret.Name), + } + o.Expect(len(sc.Spec.Datacenters)).To(o.BeNumerically(">", 0)) + for idx := range sc.Spec.Datacenters { + sc.Spec.Datacenters[idx].ScyllaDB = &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterDatacenterConfigMap.Name), + } + sc.Spec.Datacenters[idx].ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterSecret.Name), + } + sc.Spec.Datacenters[idx].RackTemplate = &scyllav1alpha1.RackTemplate{ + ScyllaDB: &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterRackTemplateConfigMap.Name), + }, + ScyllaDBManagerAgent: &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterRackTemplateSecret.Name), + }, + } + sc.Spec.Datacenters[idx].Racks = []scyllav1alpha1.RackSpec{ + { + Name: "a", + RackTemplate: scyllav1alpha1.RackTemplate{ + ScyllaDB: &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterRackConfigMap.Name), + }, + ScyllaDBManagerAgent: &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterRackSecret.Name), + }, + }, + }, + } + } + + sc, err = metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(userNS.Name).Create(ctx, sc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the ScyllaDBCluster %q roll out (RV=%s)", sc.Name, sc.ResourceVersion) + waitCtx2, waitCtx2Cancel := utils.ContextForMultiDatacenterScyllaDBClusterRollout(ctx, sc) + defer waitCtx2Cancel() + sc, err = controllerhelpers.WaitForScyllaDBClusterState(waitCtx2, metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(sc.Namespace), sc.Name, controllerhelpers.WaitForStateOptions{}, utils.IsScyllaDBClusterRolledOut) + o.Expect(err).NotTo(o.HaveOccurred()) + + scylladbclusterverification.Verify(ctx, sc, rkcClusterMap) + err = scylladbclusterverification.WaitForFullQuorum(ctx, rkcClusterMap, sc) + o.Expect(err).NotTo(o.HaveOccurred()) + + for _, dc := range sc.Spec.Datacenters { + framework.By("Verifying if ConfigMaps and Secrets referenced by ScyllaDBCluster %q are mirrored in %q Datacenter", sc.Name, dc.Name) + + clusterClient := rkcClusterMap[dc.RemoteKubernetesClusterName] + + dcStatus, _, ok := slices.Find(sc.Status.Datacenters, func(dcStatus scyllav1alpha1.ScyllaDBClusterDatacenterStatus) bool { + return dc.Name == dcStatus.Name + }) + o.Expect(ok).To(o.BeTrue()) + o.Expect(dcStatus.RemoteNamespaceName).ToNot(o.BeNil()) + + for _, cm := range scyllaDBConfigMaps { + mirroredCM, err := clusterClient.KubeAdminClient().CoreV1().ConfigMaps(*dcStatus.RemoteNamespaceName).Get(ctx, cm.Name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(mirroredCM.Data).To(o.Equal(cm.Data)) + } + + for _, secret := range scyllaDBManagerAgentSecrets { + mirroredSecret, err := clusterClient.KubeAdminClient().CoreV1().Secrets(*dcStatus.RemoteNamespaceName).Get(ctx, secret.Name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + for k, v := range secret.Data { + decodedValue, err := io.ReadAll(base64.NewDecoder(base64.StdEncoding, bytes.NewReader(mirroredSecret.Data[k]))) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(mirroredSecret.Data).To(o.HaveKey(k)) + o.Expect(decodedValue).To(o.Equal(v)) + } + } + + framework.By("Verifying if custom auth token set via configuration of ScyllaDB Manager Agent is being used by %q datacenter", dc.Name) + sdc, err := clusterClient.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBDatacenters(*dcStatus.RemoteNamespaceName).Get(ctx, naming.ScyllaDBDatacenterName(sc, &dc), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + svc, err := clusterClient.KubeAdminClient().CoreV1().Services(*dcStatus.RemoteNamespaceName).Get(ctx, naming.MemberServiceName(sdc.Spec.Racks[0], sdc, 0), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + pod, err := clusterClient.KubeAdminClient().CoreV1().Pods(*dcStatus.RemoteNamespaceName).Get(ctx, naming.PodNameFromService(svc), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + host, err := controllerhelpers.GetScyllaHost(sdc, svc, pod) + o.Expect(err).NotTo(o.HaveOccurred()) + + scyllaConfigClient := scyllaclient.NewConfigClient(host, scyllaDBManagerAgentRackAuthToken) + + framework.By("Verifying if configuration of ScyllaDB is being used by %q datacenter", dc.Name) + gotReadRequestTimeoutInMs, err := scyllaConfigClient.ReadRequestTimeoutInMs(ctx) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(gotReadRequestTimeoutInMs).To(o.Equal(int64(scyllaDBRackReadRequestTimeoutInMs))) + } + }) +}) diff --git a/test/e2e/utils/verification/scylladbcluster/verify.go b/test/e2e/utils/verification/scylladbcluster/verify.go index c4ccd190a06..674313c151d 100644 --- a/test/e2e/utils/verification/scylladbcluster/verify.go +++ b/test/e2e/utils/verification/scylladbcluster/verify.go @@ -104,6 +104,22 @@ func Verify(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster, rkcClusterM condType: "RemoteRemoteOwnerControllerDegraded", status: metav1.ConditionFalse, }, + { + condType: "RemoteConfigMapControllerProgressing", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteConfigMapControllerDegraded", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteSecretControllerProgressing", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteSecretControllerDegraded", + status: metav1.ConditionFalse, + }, } expectedConditions := make([]interface{}, 0, len(condList)) From 48b523d28da9e8338e0b06da75f940c8d749d136 Mon Sep 17 00:00:00 2001 From: Maciej Zimnoch Date: Wed, 26 Feb 2025 17:51:28 +0100 Subject: [PATCH 3/3] Update autogenerated --- deploy/operator.yaml | 2 ++ deploy/operator/00_operator_remote.clusterrole_def.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 9febb86740c..1ef0b76850f 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -443,6 +443,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete diff --git a/deploy/operator/00_operator_remote.clusterrole_def.yaml b/deploy/operator/00_operator_remote.clusterrole_def.yaml index 4651a461987..e1022ec430b 100644 --- a/deploy/operator/00_operator_remote.clusterrole_def.yaml +++ b/deploy/operator/00_operator_remote.clusterrole_def.yaml @@ -47,6 +47,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete