diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 9febb86740c..1ef0b76850f 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -443,6 +443,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete diff --git a/deploy/operator/00_operator_remote.clusterrole_def.yaml b/deploy/operator/00_operator_remote.clusterrole_def.yaml index 4651a461987..e1022ec430b 100644 --- a/deploy/operator/00_operator_remote.clusterrole_def.yaml +++ b/deploy/operator/00_operator_remote.clusterrole_def.yaml @@ -47,6 +47,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete diff --git a/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml b/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml index 4651a461987..e1022ec430b 100644 --- a/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml +++ b/helm/scylla-operator/templates/operator_remote.clusterrole_def.yaml @@ -47,6 +47,8 @@ rules: - endpoints - namespaces - services + - secrets + - configmaps verbs: - create - delete diff --git a/pkg/cmd/operator/operator.go b/pkg/cmd/operator/operator.go index 1e84519bca7..e568bbd2fd4 100644 --- a/pkg/cmd/operator/operator.go +++ b/pkg/cmd/operator/operator.go @@ -325,6 +325,18 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS ), ) + remoteOperatorManagedResourcesOnlyInformer := remoteinformers.NewSharedInformerFactoryWithOptions[kubernetes.Interface]( + &o.clusterKubeClient, + resyncPeriod, + remoteinformers.WithTweakListOptions[kubernetes.Interface]( + func(options *metav1.ListOptions) { + options.LabelSelector = labels.SelectorFromSet(map[string]string{ + naming.KubernetesManagedByLabel: naming.RemoteOperatorAppNameWithDomain, + }).String() + }, + ), + ) + scyllaOperatorConfigInformers := scyllainformers.NewSharedInformerFactoryWithOptions(o.scyllaClient, resyncPeriod, scyllainformers.WithTweakListOptions( func(options *metav1.ListOptions) { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", naming.SingletonName).String() @@ -476,6 +488,8 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS &o.clusterScyllaClient, scyllaInformers.Scylla().V1alpha1().ScyllaDBClusters(), scyllaInformers.Scylla().V1alpha1().ScyllaOperatorConfigs(), + kubeInformers.Core().V1().ConfigMaps(), + kubeInformers.Core().V1().Secrets(), remoteScyllaInformer.ForResource(&scyllav1alpha1.RemoteOwner{}, remoteinformers.ClusterListWatch[scyllaversionedclient.Interface]{ ListFunc: func(client remoteclient.ClusterClientInterface[scyllaversionedclient.Interface], cluster, ns string) cache.ListFunc { return func(options metav1.ListOptions) (runtime.Object, error) { @@ -616,6 +630,46 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS } }, }), + remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.ConfigMap{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{ + ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc { + return func(options metav1.ListOptions) (runtime.Object, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().ConfigMaps(ns).List(ctx, options) + } + }, + WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc { + return func(options metav1.ListOptions) (watch.Interface, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().ConfigMaps(ns).Watch(ctx, options) + } + }, + }), + remoteOperatorManagedResourcesOnlyInformer.ForResource(&corev1.Secret{}, remoteinformers.ClusterListWatch[kubernetes.Interface]{ + ListFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.ListFunc { + return func(options metav1.ListOptions) (runtime.Object, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().Secrets(ns).List(ctx, options) + } + }, + WatchFunc: func(client remoteclient.ClusterClientInterface[kubernetes.Interface], cluster, ns string) cache.WatchFunc { + return func(options metav1.ListOptions) (watch.Interface, error) { + clusterClient, err := client.Cluster(cluster) + if err != nil { + return nil, err + } + return clusterClient.CoreV1().Secrets(ns).Watch(ctx, options) + } + }, + }), ) if err != nil { return fmt.Errorf("can't create ScyllaDBCluster controller: %w", err) @@ -672,6 +726,12 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS remoteScyllaPodInformer.Start(ctx.Done()) }() + wg.Add(1) + go func() { + defer wg.Done() + remoteOperatorManagedResourcesOnlyInformer.Start(ctx.Done()) + }() + wg.Add(1) go func() { defer wg.Done() diff --git a/pkg/controller/scylladbcluster/conditions.go b/pkg/controller/scylladbcluster/conditions.go index 68880131601..7f1e455ee4d 100644 --- a/pkg/controller/scylladbcluster/conditions.go +++ b/pkg/controller/scylladbcluster/conditions.go @@ -15,4 +15,8 @@ const ( remoteEndpointsControllerDegradedCondition = "RemoteEndpointsControllerDegraded" scyllaDBClusterFinalizerProgressingCondition = "ScyllaDBClusterFinalizerProgressing" scyllaDBClusterFinalizerDegradedCondition = "ScyllaDBClusterFinalizerDegraded" + remoteConfigMapControllerProgressingCondition = "RemoteConfigMapControllerProgressing" + remoteConfigMapControllerDegradedCondition = "RemoteConfigMapControllerDegraded" + remoteSecretControllerProgressingCondition = "RemoteSecretControllerProgressing" + remoteSecretControllerDegradedCondition = "RemoteSecretControllerDegraded" ) diff --git a/pkg/controller/scylladbcluster/controller.go b/pkg/controller/scylladbcluster/controller.go index 2caada2bb38..0ba4f19c8d8 100644 --- a/pkg/controller/scylladbcluster/controller.go +++ b/pkg/controller/scylladbcluster/controller.go @@ -6,6 +6,8 @@ import ( "sync" "time" + corev1informers "k8s.io/client-go/informers/core/v1" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" scyllaclient "github.com/scylladb/scylla-operator/pkg/client/scylla/clientset/versioned" scyllav1alpha1informers "github.com/scylladb/scylla-operator/pkg/client/scylla/informers/externalversions/scylla/v1alpha1" @@ -54,6 +56,8 @@ type Controller struct { scyllaDBClusterLister scyllav1alpha1listers.ScyllaDBClusterLister scyllaOperatorConfigLister scyllav1alpha1listers.ScyllaOperatorConfigLister + configMapLister corev1listers.ConfigMapLister + secretLister corev1listers.SecretLister remoteRemoteOwnerLister remotelister.GenericClusterLister[scyllav1alpha1listers.RemoteOwnerLister] remoteScyllaDBDatacenterLister remotelister.GenericClusterLister[scyllav1alpha1listers.ScyllaDBDatacenterLister] @@ -62,6 +66,8 @@ type Controller struct { remoteEndpointSliceLister remotelister.GenericClusterLister[discoveryv1listers.EndpointSliceLister] remoteEndpointsLister remotelister.GenericClusterLister[corev1listers.EndpointsLister] remotePodLister remotelister.GenericClusterLister[corev1listers.PodLister] + remoteConfigMapLister remotelister.GenericClusterLister[corev1listers.ConfigMapLister] + remoteSecretLister remotelister.GenericClusterLister[corev1listers.SecretLister] cachesToSync []cache.InformerSynced @@ -78,6 +84,8 @@ func NewController( scyllaRemoteClient remoteclient.ClusterClientInterface[scyllaclient.Interface], scyllaDBClusterInformer scyllav1alpha1informers.ScyllaDBClusterInformer, scyllaOperatorConfigInformer scyllav1alpha1informers.ScyllaOperatorConfigInformer, + configMapInformer corev1informers.ConfigMapInformer, + secretInformer corev1informers.SecretInformer, remoteRemoteOwnerInformer remoteinformers.GenericClusterInformer, remoteScyllaDBDatacenterInformer remoteinformers.GenericClusterInformer, remoteNamespaceInformer remoteinformers.GenericClusterInformer, @@ -85,6 +93,8 @@ func NewController( remoteEndpointSliceInformer remoteinformers.GenericClusterInformer, remoteEndpointsInformer remoteinformers.GenericClusterInformer, remotePodInformer remoteinformers.GenericClusterInformer, + remoteConfigMapInformer remoteinformers.GenericClusterInformer, + remoteSecretInformer remoteinformers.GenericClusterInformer, ) (*Controller, error) { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartStructuredLogging(0) @@ -98,6 +108,8 @@ func NewController( scyllaDBClusterLister: scyllaDBClusterInformer.Lister(), scyllaOperatorConfigLister: scyllaOperatorConfigInformer.Lister(), + configMapLister: configMapInformer.Lister(), + secretLister: secretInformer.Lister(), remoteRemoteOwnerLister: remotelister.NewClusterLister(scyllav1alpha1listers.NewRemoteOwnerLister, remoteRemoteOwnerInformer.Indexer().Cluster), remoteScyllaDBDatacenterLister: remotelister.NewClusterLister(scyllav1alpha1listers.NewScyllaDBDatacenterLister, remoteScyllaDBDatacenterInformer.Indexer().Cluster), @@ -106,9 +118,14 @@ func NewController( remoteEndpointSliceLister: remotelister.NewClusterLister(discoveryv1listers.NewEndpointSliceLister, remoteEndpointSliceInformer.Indexer().Cluster), remoteEndpointsLister: remotelister.NewClusterLister(corev1listers.NewEndpointsLister, remoteEndpointsInformer.Indexer().Cluster), remotePodLister: remotelister.NewClusterLister(corev1listers.NewPodLister, remotePodInformer.Indexer().Cluster), + remoteConfigMapLister: remotelister.NewClusterLister(corev1listers.NewConfigMapLister, remoteConfigMapInformer.Indexer().Cluster), + remoteSecretLister: remotelister.NewClusterLister(corev1listers.NewSecretLister, remoteSecretInformer.Indexer().Cluster), cachesToSync: []cache.InformerSynced{ scyllaDBClusterInformer.Informer().HasSynced, + scyllaOperatorConfigInformer.Informer().HasSynced, + configMapInformer.Informer().HasSynced, + secretInformer.Informer().HasSynced, remoteRemoteOwnerInformer.Informer().HasSynced, remoteScyllaDBDatacenterInformer.Informer().HasSynced, remoteNamespaceInformer.Informer().HasSynced, @@ -116,6 +133,8 @@ func NewController( remoteEndpointSliceInformer.Informer().HasSynced, remoteEndpointsInformer.Informer().HasSynced, remotePodInformer.Informer().HasSynced, + remoteConfigMapInformer.Informer().HasSynced, + remoteSecretInformer.Informer().HasSynced, }, eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "scylladbcluster-controller"}), @@ -154,6 +173,11 @@ func NewController( errs = append(errs, fmt.Errorf("can't register to ScyllaDBCluster events: %w", err)) } + // Local ConfigMap and Secret handlers are skipped to optimize number of syncs which doesn't do anything. + // Applying configuration change requires rolling restart of ScyllaDBCluster, so these resources will be synced upon + // ScyllaDBCluster update. + // These could be added once ConfigMaps and Secrets would require immediate sync. + // TODO: add error handling once these start returning errors remoteRemoteOwnerInformer.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -211,6 +235,22 @@ func NewController( }, ) + remoteConfigMapInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: scc.addRemoteConfigMap, + UpdateFunc: scc.updateRemoteConfigMap, + DeleteFunc: scc.deleteRemoteConfigMap, + }, + ) + + remoteSecretInformer.Informer().AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: scc.addRemoteSecret, + UpdateFunc: scc.updateRemoteSecret, + DeleteFunc: scc.deleteRemoteSecret, + }, + ) + err = utilerrors.NewAggregate(errs) if err != nil { return nil, fmt.Errorf("can't register event handlers: %w", err) @@ -540,3 +580,49 @@ func (scc *Controller) deleteRemotePod(obj interface{}) { scc.enqueueThroughParentLabel, ) } + +func (scc *Controller) addRemoteConfigMap(obj interface{}) { + scc.handlers.HandleAdd( + obj.(*corev1.ConfigMap), + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) updateRemoteConfigMap(old, cur interface{}) { + scc.handlers.HandleUpdate( + old.(*corev1.ConfigMap), + cur.(*corev1.ConfigMap), + scc.enqueueThroughParentLabel, + scc.deleteRemoteConfigMap, + ) +} + +func (scc *Controller) deleteRemoteConfigMap(obj interface{}) { + scc.handlers.HandleDelete( + obj, + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) addRemoteSecret(obj interface{}) { + scc.handlers.HandleAdd( + obj.(*corev1.Secret), + scc.enqueueThroughParentLabel, + ) +} + +func (scc *Controller) updateRemoteSecret(old, cur interface{}) { + scc.handlers.HandleUpdate( + old.(*corev1.Secret), + cur.(*corev1.Secret), + scc.enqueueThroughParentLabel, + scc.deleteRemoteSecret, + ) +} + +func (scc *Controller) deleteRemoteSecret(obj interface{}) { + scc.handlers.HandleDelete( + obj, + scc.enqueueThroughParentLabel, + ) +} diff --git a/pkg/controller/scylladbcluster/resource.go b/pkg/controller/scylladbcluster/resource.go index d73f3f359b6..39f42fdef18 100644 --- a/pkg/controller/scylladbcluster/resource.go +++ b/pkg/controller/scylladbcluster/resource.go @@ -2,6 +2,8 @@ package scylladbcluster import ( "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/errors" "maps" "sort" "strings" @@ -911,3 +913,192 @@ func getRemoteNamespaceAndController(controllerProgressingType string, sc *scyll return progressingConditions, remoteNamespace, remoteController } + +func MakeRemoteConfigMaps(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localConfigMapLister corev1listers.ConfigMapLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.ConfigMap, error) { + requiredRemoteConfigMaps := make(map[string][]*corev1.ConfigMap) + + progressingConditions, mirroredConfigMaps, err := makeMirroredRemoteConfigMaps(sc, remoteNamespaces, remoteControllers, localConfigMapLister, managingClusterDomain) + if err != nil { + return progressingConditions, nil, fmt.Errorf("can't make mirrored configmaps: %w", err) + } + + for k, v := range mirroredConfigMaps { + requiredRemoteConfigMaps[k] = append(requiredRemoteConfigMaps[k], v...) + } + + return progressingConditions, requiredRemoteConfigMaps, nil +} + +func makeMirroredRemoteConfigMaps(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localConfigMapLister corev1listers.ConfigMapLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.ConfigMap, error) { + var errs []error + var progressingConditions []metav1.Condition + requiredRemoteConfigMaps := make(map[string][]*corev1.ConfigMap, len(sc.Spec.Datacenters)) + + for _, dc := range sc.Spec.Datacenters { + dcProgressingConditions, remoteNamespace, remoteController := getRemoteNamespaceAndController( + remoteConfigMapControllerProgressingCondition, + sc, + dc.RemoteKubernetesClusterName, + remoteNamespaces, + remoteControllers, + ) + if len(dcProgressingConditions) > 0 { + progressingConditions = append(progressingConditions, dcProgressingConditions...) + continue + } + + var configMapsToMirror []string + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.ScyllaDB != nil && sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef) + } + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB.CustomConfigMapRef) + } + + if dc.ScyllaDB != nil && dc.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *dc.ScyllaDB.CustomConfigMapRef) + } + + if dc.RackTemplate != nil && dc.RackTemplate.ScyllaDB != nil && dc.RackTemplate.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *dc.RackTemplate.ScyllaDB.CustomConfigMapRef) + } + + for _, rack := range dc.Racks { + if rack.ScyllaDB != nil && rack.ScyllaDB.CustomConfigMapRef != nil { + configMapsToMirror = append(configMapsToMirror, *rack.ScyllaDB.CustomConfigMapRef) + } + } + + for _, cmName := range configMapsToMirror { + localConfigMap, err := localConfigMapLister.ConfigMaps(sc.Namespace).Get(cmName) + if err != nil { + if apierrors.IsNotFound(err) { + progressingConditions = append(progressingConditions, metav1.Condition{ + Type: remoteConfigMapControllerProgressingCondition, + Status: metav1.ConditionTrue, + Reason: "WaitingForConfigMap", + Message: fmt.Sprintf("Waiting for ConfigMap %q to exist.", naming.ManualRef(sc.Namespace, cmName)), + ObservedGeneration: sc.Generation, + }) + continue + } + errs = append(errs, fmt.Errorf("can't get %q ConfigMap: %w", naming.ManualRef(sc.Namespace, cmName), err)) + continue + } + + requiredRemoteConfigMaps[dc.RemoteKubernetesClusterName] = append(requiredRemoteConfigMaps[dc.RemoteKubernetesClusterName], &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: remoteNamespace.Name, + Labels: naming.ScyllaDBClusterDatacenterLabels(sc, &dc, managingClusterDomain), + Annotations: naming.ScyllaDBClusterDatacenterAnnotations(sc, &dc), + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(remoteController, remoteControllerGVK)}, + }, + Immutable: localConfigMap.Immutable, + Data: maps.Clone(localConfigMap.Data), + BinaryData: maps.Clone(localConfigMap.BinaryData), + }) + } + } + + err := errors.NewAggregate(errs) + if err != nil { + return progressingConditions, requiredRemoteConfigMaps, fmt.Errorf("can't make mirrored ConfigMaps: %w", err) + } + + return progressingConditions, requiredRemoteConfigMaps, nil +} + +func MakeRemoteSecrets(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localSecretLister corev1listers.SecretLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.Secret, error) { + requiredRemoteSecrets := make(map[string][]*corev1.Secret) + + progressingConditions, mirroredSecrets, err := makeMirroredRemoteSecrets(sc, remoteNamespaces, remoteControllers, localSecretLister, managingClusterDomain) + if err != nil { + return progressingConditions, nil, fmt.Errorf("can't make mirrored secrets: %w", err) + } + + for k, v := range mirroredSecrets { + requiredRemoteSecrets[k] = append(requiredRemoteSecrets[k], v...) + } + + return progressingConditions, requiredRemoteSecrets, nil +} + +func makeMirroredRemoteSecrets(sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace, remoteControllers map[string]metav1.Object, localSecretLister corev1listers.SecretLister, managingClusterDomain string) ([]metav1.Condition, map[string][]*corev1.Secret, error) { + var progressingConditions []metav1.Condition + requiredRemoteSecrets := make(map[string][]*corev1.Secret, len(sc.Spec.Datacenters)) + + var errs []error + for _, dc := range sc.Spec.Datacenters { + dcProgressingConditions, remoteNamespace, remoteController := getRemoteNamespaceAndController( + remoteConfigMapControllerProgressingCondition, + sc, + dc.RemoteKubernetesClusterName, + remoteNamespaces, + remoteControllers, + ) + if len(dcProgressingConditions) > 0 { + progressingConditions = append(progressingConditions, dcProgressingConditions...) + continue + } + + var secretsToMirror []string + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent != nil && sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if sc.Spec.DatacenterTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent != nil && sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if dc.ScyllaDBManagerAgent != nil && dc.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *dc.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + if dc.RackTemplate != nil && dc.RackTemplate.ScyllaDBManagerAgent != nil && dc.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *dc.RackTemplate.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + + for _, rack := range dc.Racks { + if rack.ScyllaDBManagerAgent != nil && rack.ScyllaDBManagerAgent.CustomConfigSecretRef != nil { + secretsToMirror = append(secretsToMirror, *rack.ScyllaDBManagerAgent.CustomConfigSecretRef) + } + } + + for _, secretName := range secretsToMirror { + localSecret, err := localSecretLister.Secrets(sc.Namespace).Get(secretName) + if err != nil { + if apierrors.IsNotFound(err) { + progressingConditions = append(progressingConditions, metav1.Condition{ + Type: remoteSecretControllerProgressingCondition, + Status: metav1.ConditionTrue, + Reason: "WaitingForSecret", + Message: fmt.Sprintf("Waiting for Secret %q to exist.", naming.ManualRef(sc.Namespace, secretName)), + ObservedGeneration: sc.Generation, + }) + continue + } + errs = append(errs, fmt.Errorf("can't get %q Secret: %w", naming.ManualRef(sc.Namespace, secretName), err)) + continue + } + + requiredRemoteSecrets[dc.RemoteKubernetesClusterName] = append(requiredRemoteSecrets[dc.RemoteKubernetesClusterName], &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: remoteNamespace.Name, + Labels: naming.ScyllaDBClusterDatacenterLabels(sc, &dc, managingClusterDomain), + Annotations: naming.ScyllaDBClusterDatacenterAnnotations(sc, &dc), + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(remoteController, remoteControllerGVK)}, + }, + Immutable: localSecret.Immutable, + Data: maps.Clone(localSecret.Data), + Type: localSecret.Type, + }) + } + } + + return progressingConditions, requiredRemoteSecrets, nil +} diff --git a/pkg/controller/scylladbcluster/resource_test.go b/pkg/controller/scylladbcluster/resource_test.go index 54bc5681c69..b1746ac6d4c 100644 --- a/pkg/controller/scylladbcluster/resource_test.go +++ b/pkg/controller/scylladbcluster/resource_test.go @@ -82,6 +82,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -97,6 +98,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -112,6 +114,7 @@ func TestMakeRemoteOwners(t *testing.T) { "internal.scylla-operator.scylladb.com/remote-owner-name": "cluster", "internal.scylla-operator.scylladb.com/remote-owner-gvr": "scylla.scylladb.com-v1alpha1-scylladbclusters", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }, @@ -210,6 +213,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -221,6 +225,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -232,6 +237,7 @@ func TestMakeNamespaces(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, }, }}, @@ -475,6 +481,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -497,6 +504,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc1", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -521,6 +529,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -543,6 +552,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc2", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -567,6 +577,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -589,6 +600,7 @@ func TestMakeServices(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/parent-scylladbcluster-datacenter-name": "dc3", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&scyllav1alpha1.RemoteOwner{ ObjectMeta: metav1.ObjectMeta{ @@ -838,6 +850,7 @@ func TestMakeScyllaDBDatacenters(t *testing.T) { "scylla-operator.scylladb.com/parent-scylladbcluster-name": "cluster", "scylla-operator.scylladb.com/parent-scylladbcluster-namespace": "scylla", "scylla-operator.scylladb.com/managed-by-cluster": "test-cluster.local", + "app.kubernetes.io/managed-by": "remote.scylla-operator.scylladb.com", }, Annotations: map[string]string{}, OwnerReferences: newBasicOwnerReference(namespace), diff --git a/pkg/controller/scylladbcluster/sync.go b/pkg/controller/scylladbcluster/sync.go index 49f29493e16..b508803b08e 100644 --- a/pkg/controller/scylladbcluster/sync.go +++ b/pkg/controller/scylladbcluster/sync.go @@ -158,6 +158,52 @@ func (scc *Controller) sync(ctx context.Context, key string) error { objectErrs = append(objectErrs, fmt.Errorf("can't get remote endpoints: %w", err)) } + remoteConfigMapMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *corev1.ConfigMap](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *corev1.ConfigMap]{ + ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *corev1.ConfigMap], error) { + ns, ok := remoteNamespaces[clusterName] + if !ok { + return nil, nil + } + + kubeClusterClient, scyllaClusterClient, err := scc.getClusterClients(clusterName) + if err != nil { + return nil, fmt.Errorf("can't get cluster %q clients: %w", clusterName, err) + } + + return &controllerhelpers.ControlleeManagerGetObjectsFuncs[remoteCT, *corev1.ConfigMap]{ + GetControllerUncachedFunc: scyllaClusterClient.ScyllaV1alpha1().RemoteOwners(ns.Name).Get, + ListObjectsFunc: scc.remoteConfigMapLister.Cluster(clusterName).ConfigMaps(ns.Name).List, + PatchObjectFunc: kubeClusterClient.CoreV1().ConfigMaps(ns.Name).Patch, + }, nil + }, + }) + if err != nil { + objectErrs = append(objectErrs, fmt.Errorf("can't get remote configmaps: %w", err)) + } + + remoteSecretMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *corev1.Secret](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *corev1.Secret]{ + ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *corev1.Secret], error) { + ns, ok := remoteNamespaces[clusterName] + if !ok { + return nil, nil + } + + kubeClusterClient, scyllaClusterClient, err := scc.getClusterClients(clusterName) + if err != nil { + return nil, fmt.Errorf("can't get cluster %q clients: %w", clusterName, err) + } + + return &controllerhelpers.ControlleeManagerGetObjectsFuncs[remoteCT, *corev1.Secret]{ + GetControllerUncachedFunc: scyllaClusterClient.ScyllaV1alpha1().RemoteOwners(ns.Name).Get, + ListObjectsFunc: scc.remoteSecretLister.Cluster(clusterName).Secrets(ns.Name).List, + PatchObjectFunc: kubeClusterClient.CoreV1().Secrets(ns.Name).Patch, + }, nil + }, + }) + if err != nil { + objectErrs = append(objectErrs, fmt.Errorf("can't get remote configmaps: %w", err)) + } + remoteScyllaDBDatacenterMap, err := controllerhelpers.GetRemoteObjects[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter](ctx, remoteClusterNames, remoteControllers, remoteControllerGVK, scRemoteSelector, &controllerhelpers.ClusterControlleeManagerGetObjectsFuncs[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter]{ ClusterFunc: func(clusterName string) (controllerhelpers.ControlleeManagerGetObjectsInterface[remoteCT, *scyllav1alpha1.ScyllaDBDatacenter], error) { ns, ok := remoteNamespaces[clusterName] @@ -285,6 +331,32 @@ func (scc *Controller) sync(ctx context.Context, key string) error { errs = append(errs, fmt.Errorf("can't sync remote endpoints: %w", err)) } + err = controllerhelpers.RunSync( + &status.Conditions, + remoteConfigMapControllerProgressingCondition, + remoteConfigMapControllerDegradedCondition, + sc.Generation, + func() ([]metav1.Condition, error) { + return scc.syncConfigMaps(ctx, sc, remoteNamespaces, remoteControllers, remoteConfigMapMap, managingClusterDomain) + }, + ) + if err != nil { + errs = append(errs, fmt.Errorf("can't sync remote configmaps: %w", err)) + } + + err = controllerhelpers.RunSync( + &status.Conditions, + remoteSecretControllerProgressingCondition, + remoteSecretControllerDegradedCondition, + sc.Generation, + func() ([]metav1.Condition, error) { + return scc.syncSecrets(ctx, sc, remoteNamespaces, remoteControllers, remoteSecretMap, managingClusterDomain) + }, + ) + if err != nil { + errs = append(errs, fmt.Errorf("can't sync remote secrets: %w", err)) + } + err = controllerhelpers.RunSync( &status.Conditions, remoteScyllaDBDatacenterControllerProgressingCondition, diff --git a/pkg/controller/scylladbcluster/sync_configmap.go b/pkg/controller/scylladbcluster/sync_configmap.go new file mode 100644 index 00000000000..e2364b6d637 --- /dev/null +++ b/pkg/controller/scylladbcluster/sync_configmap.go @@ -0,0 +1,88 @@ +package scylladbcluster + +import ( + "context" + "fmt" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/resourceapply" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +func (scc *Controller) syncConfigMaps( + ctx context.Context, + sc *scyllav1alpha1.ScyllaDBCluster, + remoteNamespaces map[string]*corev1.Namespace, + remoteControllers map[string]metav1.Object, + remoteConfigMaps map[string]map[string]*corev1.ConfigMap, + managingClusterDomain string, +) ([]metav1.Condition, error) { + progressingConditions, requiredConfigMaps, err := MakeRemoteConfigMaps(sc, remoteNamespaces, remoteControllers, scc.configMapLister, managingClusterDomain) + if err != nil { + return progressingConditions, fmt.Errorf("can't make required configmaps: %w", err) + } + + if len(progressingConditions) > 0 { + return progressingConditions, nil + } + + // Delete has to be the first action to avoid getting stuck on quota. + var deletionErrors []error + for _, dc := range sc.Spec.Datacenters { + ns, ok := remoteNamespaces[dc.RemoteKubernetesClusterName] + if !ok { + continue + } + + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + return nil, fmt.Errorf("can't get client to %q cluster: %w", dc.RemoteKubernetesClusterName, err) + } + + err = controllerhelpers.Prune(ctx, + requiredConfigMaps[dc.RemoteKubernetesClusterName], + remoteConfigMaps[dc.RemoteKubernetesClusterName], + &controllerhelpers.PruneControlFuncs{ + DeleteFunc: clusterClient.CoreV1().ConfigMaps(ns.Name).Delete, + }, + scc.eventRecorder, + ) + if err != nil { + return progressingConditions, fmt.Errorf("can't prune configmap(s) in %q Datacenter of %q ScyllaDBCluster: %w", dc.Name, naming.ObjRef(sc), err) + } + } + + if err := utilerrors.NewAggregate(deletionErrors); err != nil { + return nil, fmt.Errorf("can't prune remote configmap(s): %w", err) + } + + var errs []error + for _, dc := range sc.Spec.Datacenters { + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + errs = append(errs, fmt.Errorf("can't get client to %q cluster: %w", dc.Name, err)) + continue + } + + for _, cm := range requiredConfigMaps[dc.RemoteKubernetesClusterName] { + _, changed, err := resourceapply.ApplyConfigMap(ctx, clusterClient.CoreV1(), scc.remoteConfigMapLister.Cluster(dc.RemoteKubernetesClusterName), scc.eventRecorder, cm, resourceapply.ApplyOptions{}) + if changed { + controllerhelpers.AddGenericProgressingStatusCondition(&progressingConditions, remoteConfigMapControllerProgressingCondition, cm, "apply", sc.Generation) + } + if err != nil { + errs = append(errs, fmt.Errorf("can't apply configmap: %w", err)) + continue + } + } + } + + err = utilerrors.NewAggregate(errs) + if err != nil { + return progressingConditions, fmt.Errorf("can't apply configmap(s): %w", err) + } + + return progressingConditions, nil +} diff --git a/pkg/controller/scylladbcluster/sync_secrets.go b/pkg/controller/scylladbcluster/sync_secrets.go new file mode 100644 index 00000000000..4da26c36a5f --- /dev/null +++ b/pkg/controller/scylladbcluster/sync_secrets.go @@ -0,0 +1,88 @@ +package scylladbcluster + +import ( + "context" + "fmt" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/resourceapply" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +func (scc *Controller) syncSecrets( + ctx context.Context, + sc *scyllav1alpha1.ScyllaDBCluster, + remoteNamespaces map[string]*corev1.Namespace, + remoteControllers map[string]metav1.Object, + remoteSecrets map[string]map[string]*corev1.Secret, + managingClusterDomain string, +) ([]metav1.Condition, error) { + progressingConditions, requiredRemoteSecrets, err := MakeRemoteSecrets(sc, remoteNamespaces, remoteControllers, scc.secretLister, managingClusterDomain) + if err != nil { + return progressingConditions, fmt.Errorf("can't make remote secrets: %w", err) + } + + if len(progressingConditions) > 0 { + return progressingConditions, nil + } + + // Delete has to be the first action to avoid getting stuck on quota. + var deletionErrors []error + for _, dc := range sc.Spec.Datacenters { + ns, ok := remoteNamespaces[dc.RemoteKubernetesClusterName] + if !ok { + continue + } + + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + return nil, fmt.Errorf("can't get client to %q cluster: %w", dc.RemoteKubernetesClusterName, err) + } + + err = controllerhelpers.Prune(ctx, + requiredRemoteSecrets[dc.RemoteKubernetesClusterName], + remoteSecrets[dc.RemoteKubernetesClusterName], + &controllerhelpers.PruneControlFuncs{ + DeleteFunc: clusterClient.CoreV1().Secrets(ns.Name).Delete, + }, + scc.eventRecorder, + ) + if err != nil { + return progressingConditions, fmt.Errorf("can't prune secret(s) in %q Datacenter of %q ScyllaDBCluster: %w", dc.Name, naming.ObjRef(sc), err) + } + } + + if err := utilerrors.NewAggregate(deletionErrors); err != nil { + return nil, fmt.Errorf("can't prune remote secret(s): %w", err) + } + + var errs []error + for _, dc := range sc.Spec.Datacenters { + clusterClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + errs = append(errs, fmt.Errorf("can't get client to %q cluster: %w", dc.Name, err)) + continue + } + + for _, rs := range requiredRemoteSecrets[dc.RemoteKubernetesClusterName] { + _, changed, err := resourceapply.ApplySecret(ctx, clusterClient.CoreV1(), scc.remoteSecretLister.Cluster(dc.RemoteKubernetesClusterName), scc.eventRecorder, rs, resourceapply.ApplyOptions{}) + if changed { + controllerhelpers.AddGenericProgressingStatusCondition(&progressingConditions, remoteSecretControllerProgressingCondition, rs, "apply", sc.Generation) + } + if err != nil { + errs = append(errs, fmt.Errorf("can't apply secret: %w", err)) + continue + } + } + } + + err = utilerrors.NewAggregate(errs) + if err != nil { + return progressingConditions, fmt.Errorf("can't apply secret(s): %w", err) + } + + return progressingConditions, nil +} diff --git a/pkg/naming/constants.go b/pkg/naming/constants.go index 4d07b43d98a..4f317d0a6e7 100644 --- a/pkg/naming/constants.go +++ b/pkg/naming/constants.go @@ -231,10 +231,15 @@ const ( ) const ( - OperatorAppNameWithDomain = "scylla-operator.scylladb.com" + OperatorAppNameWithDomain = "scylla-operator.scylladb.com" + RemoteOperatorAppNameWithDomain = "remote.scylla-operator.scylladb.com" ) const ( RemoteKubernetesClusterFinalizer = "scylla-operator.scylladb.com/remotekubernetescluster-protection" ScyllaDBClusterFinalizer = "scylla-operator.scylladb.com/scylladbcluster-protection" ) + +const ( + KubernetesManagedByLabel = "app.kubernetes.io/managed-by" +) diff --git a/pkg/naming/labels.go b/pkg/naming/labels.go index d87129be604..e588041d5e0 100644 --- a/pkg/naming/labels.go +++ b/pkg/naming/labels.go @@ -91,9 +91,10 @@ func mergeLabels(l1, l2 map[string]string) map[string]string { return res } -func ManagedResourcesLabels(managingClusterDomain string) map[string]string { +func RemoteManagedResourcesLabels(managingClusterDomain string) map[string]string { return map[string]string{ - ManagedByClusterLabel: managingClusterDomain, + KubernetesManagedByLabel: RemoteOperatorAppNameWithDomain, + ManagedByClusterLabel: managingClusterDomain, } } @@ -113,7 +114,7 @@ func ScyllaDBClusterDatacenterLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scy if dc.Metadata != nil { maps.Copy(dcLabels, dc.Metadata.Labels) } - maps.Copy(dcLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(dcLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(dcLabels, ScyllaDBClusterSelectorLabels(sc)) dcLabels[ParentClusterDatacenterNameLabel] = dc.Name return dcLabels @@ -155,7 +156,7 @@ func ScyllaDBClusterDatacenterEndpointsLabels(sc *scyllav1alpha1.ScyllaDBCluster if dc.Metadata != nil { maps.Copy(dcLabels, dc.Metadata.Labels) } - maps.Copy(dcLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(dcLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(dcLabels, ScyllaDBClusterEndpointsSelectorLabels(sc)) dcLabels[ParentClusterDatacenterNameLabel] = dc.Name return dcLabels @@ -200,7 +201,7 @@ func RemoteOwnerSelectorLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scyllav1a func RemoteOwnerLabels(sc *scyllav1alpha1.ScyllaDBCluster, dc *scyllav1alpha1.ScyllaDBClusterDatacenter, managingClusterDomain string) map[string]string { remoteOwnerLabels := make(map[string]string) - maps.Copy(remoteOwnerLabels, ManagedResourcesLabels(managingClusterDomain)) + maps.Copy(remoteOwnerLabels, RemoteManagedResourcesLabels(managingClusterDomain)) maps.Copy(remoteOwnerLabels, RemoteOwnerSelectorLabels(sc, dc)) return remoteOwnerLabels diff --git a/pkg/scyllaclient/config_client.go b/pkg/scyllaclient/config_client.go index 205119bc1a1..f613a09184c 100644 --- a/pkg/scyllaclient/config_client.go +++ b/pkg/scyllaclient/config_client.go @@ -79,3 +79,12 @@ func (c *ConfigClient) ReplaceNodeFirstBoot(ctx context.Context) (string, error) } return resp.Payload, nil } + +// ReadRequestTimeoutInMs returns value of "read_request_timeout_in_ms" config parameter. +func (c *ConfigClient) ReadRequestTimeoutInMs(ctx context.Context) (int64, error) { + resp, err := c.client.Config.FindConfigReadRequestTimeoutInMs(config.NewFindConfigReadRequestTimeoutInMsParamsWithContext(ctx)) + if err != nil { + return 0, fmt.Errorf("can't get read_request_timeout_in_ms: %w", err) + } + return resp.Payload, nil +} diff --git a/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go b/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go new file mode 100644 index 00000000000..4401d44bca5 --- /dev/null +++ b/test/e2e/set/scylladbcluster/multidatacenter/multidatacenter_scylladbcluster_config.go @@ -0,0 +1,212 @@ +package multidatacenter + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/pointer" + "github.com/scylladb/scylla-operator/pkg/scyllaclient" + "github.com/scylladb/scylla-operator/test/e2e/framework" + "github.com/scylladb/scylla-operator/test/e2e/utils" + scylladbclusterverification "github.com/scylladb/scylla-operator/test/e2e/utils/verification/scylladbcluster" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = g.Describe("Multi datacenter ScyllaDBCluster", framework.MultiDatacenter, func() { + f := framework.NewFramework("scylladbcluster") + + g.It("should mirror ConfigMaps and Secrets referenced by ScyllaDBCluster into remote datacenters", func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + rkcs, rkcClusterMap, err := utils.SetUpRemoteKubernetesClustersFromRestConfigs(ctx, framework.TestContext.RestConfigs, f) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Creating configs for ScyllaDB and ScyllaDB Manager Agent") + + metaCluster := f.Cluster(0) + userNS, userClient, ok := metaCluster.DefaultNamespaceIfAny() + o.Expect(ok).To(o.BeTrue()) + + makeScyllaDBConfigMap := func(name, configValue string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: userNS.Name, + }, + Data: map[string]string{ + "scylla.yaml": fmt.Sprintf(`read_request_timeout_in_ms: %s`, configValue), + }, + } + } + + makeScyllaDBManagerAgentSecret := func(name, configValue string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: userNS.Name, + }, + StringData: map[string]string{ + "scylla-manager-agent.yaml": fmt.Sprintf(`auth_token: %s`, configValue), + }, + } + } + + scyllaDBDatacenterTemplateConfigMap := makeScyllaDBConfigMap("datacenter-template", "1111") + scyllaDBDatacenterTemplateRackTemplateConfigMap := makeScyllaDBConfigMap("datacenter-template-rack-template", "2222") + scyllaDBDatacenterRackTemplateConfigMap := makeScyllaDBConfigMap("datacenter-rack-template", "3333") + scyllaDBDatacenterDatacenterConfigMap := makeScyllaDBConfigMap("datacenter", "4444") + const scyllaDBRackReadRequestTimeoutInMs = 5555 + scyllaDBDatacenterRackConfigMap := makeScyllaDBConfigMap("rack", fmt.Sprintf("%d", scyllaDBRackReadRequestTimeoutInMs)) + + scyllaDBConfigMaps := []*corev1.ConfigMap{ + scyllaDBDatacenterTemplateConfigMap, + scyllaDBDatacenterTemplateRackTemplateConfigMap, + scyllaDBDatacenterRackTemplateConfigMap, + scyllaDBDatacenterDatacenterConfigMap, + scyllaDBDatacenterRackConfigMap, + } + + scyllaDBManagerAgentDatacenterTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-template", "aaaa") + scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-template-rack-template", "bbbb") + scyllaDBManagerAgentDatacenterRackTemplateSecret := makeScyllaDBManagerAgentSecret("datacenter-rack-template", "cccc") + scyllaDBManagerAgentDatacenterSecret := makeScyllaDBManagerAgentSecret("datacenter", "dddd") + const scyllaDBManagerAgentRackAuthToken = "eeee" + scyllaDBManagerAgentDatacenterRackSecret := makeScyllaDBManagerAgentSecret("rack", scyllaDBManagerAgentRackAuthToken) + + scyllaDBManagerAgentSecrets := []*corev1.Secret{ + scyllaDBManagerAgentDatacenterTemplateSecret, + scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret, + scyllaDBManagerAgentDatacenterRackTemplateSecret, + scyllaDBManagerAgentDatacenterSecret, + scyllaDBManagerAgentDatacenterRackSecret, + } + + for _, cm := range scyllaDBConfigMaps { + _, err := userClient.KubeClient().CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + for _, secret := range scyllaDBManagerAgentSecrets { + _, err := userClient.KubeClient().CoreV1().Secrets(secret.Namespace).Create(ctx, secret, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + framework.By("Creating ScyllaDBCluster") + sc := f.GetDefaultScyllaDBCluster(rkcs) + + sc.Spec.DatacenterTemplate.ScyllaDB.CustomConfigMapRef = pointer.Ptr(scyllaDBDatacenterTemplateConfigMap.Name) + sc.Spec.DatacenterTemplate.ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterTemplateSecret.Name), + } + sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDB = &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterTemplateRackTemplateConfigMap.Name), + } + sc.Spec.DatacenterTemplate.RackTemplate.ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterTemplateRackTemplateSecret.Name), + } + o.Expect(len(sc.Spec.Datacenters)).To(o.BeNumerically(">", 0)) + for idx := range sc.Spec.Datacenters { + sc.Spec.Datacenters[idx].ScyllaDB = &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterDatacenterConfigMap.Name), + } + sc.Spec.Datacenters[idx].ScyllaDBManagerAgent = &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterSecret.Name), + } + sc.Spec.Datacenters[idx].RackTemplate = &scyllav1alpha1.RackTemplate{ + ScyllaDB: &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterRackTemplateConfigMap.Name), + }, + ScyllaDBManagerAgent: &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterRackTemplateSecret.Name), + }, + } + sc.Spec.Datacenters[idx].Racks = []scyllav1alpha1.RackSpec{ + { + Name: "a", + RackTemplate: scyllav1alpha1.RackTemplate{ + ScyllaDB: &scyllav1alpha1.ScyllaDBTemplate{ + CustomConfigMapRef: pointer.Ptr(scyllaDBDatacenterRackConfigMap.Name), + }, + ScyllaDBManagerAgent: &scyllav1alpha1.ScyllaDBManagerAgentTemplate{ + CustomConfigSecretRef: pointer.Ptr(scyllaDBManagerAgentDatacenterRackSecret.Name), + }, + }, + }, + } + } + + sc, err = metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(userNS.Name).Create(ctx, sc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the ScyllaDBCluster %q roll out (RV=%s)", sc.Name, sc.ResourceVersion) + waitCtx2, waitCtx2Cancel := utils.ContextForMultiDatacenterScyllaDBClusterRollout(ctx, sc) + defer waitCtx2Cancel() + sc, err = controllerhelpers.WaitForScyllaDBClusterState(waitCtx2, metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(sc.Namespace), sc.Name, controllerhelpers.WaitForStateOptions{}, utils.IsScyllaDBClusterRolledOut) + o.Expect(err).NotTo(o.HaveOccurred()) + + scylladbclusterverification.Verify(ctx, sc, rkcClusterMap) + err = scylladbclusterverification.WaitForFullQuorum(ctx, rkcClusterMap, sc) + o.Expect(err).NotTo(o.HaveOccurred()) + + for _, dc := range sc.Spec.Datacenters { + framework.By("Verifying if ConfigMaps and Secrets referenced by ScyllaDBCluster %q are mirrored in %q Datacenter", sc.Name, dc.Name) + + clusterClient := rkcClusterMap[dc.RemoteKubernetesClusterName] + + dcStatus, _, ok := slices.Find(sc.Status.Datacenters, func(dcStatus scyllav1alpha1.ScyllaDBClusterDatacenterStatus) bool { + return dc.Name == dcStatus.Name + }) + o.Expect(ok).To(o.BeTrue()) + o.Expect(dcStatus.RemoteNamespaceName).ToNot(o.BeNil()) + + for _, cm := range scyllaDBConfigMaps { + mirroredCM, err := clusterClient.KubeAdminClient().CoreV1().ConfigMaps(*dcStatus.RemoteNamespaceName).Get(ctx, cm.Name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(mirroredCM.Data).To(o.Equal(cm.Data)) + } + + for _, secret := range scyllaDBManagerAgentSecrets { + mirroredSecret, err := clusterClient.KubeAdminClient().CoreV1().Secrets(*dcStatus.RemoteNamespaceName).Get(ctx, secret.Name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + for k, v := range secret.Data { + decodedValue, err := io.ReadAll(base64.NewDecoder(base64.StdEncoding, bytes.NewReader(mirroredSecret.Data[k]))) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(mirroredSecret.Data).To(o.HaveKey(k)) + o.Expect(decodedValue).To(o.Equal(v)) + } + } + + framework.By("Verifying if custom auth token set via configuration of ScyllaDB Manager Agent is being used by %q datacenter", dc.Name) + sdc, err := clusterClient.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBDatacenters(*dcStatus.RemoteNamespaceName).Get(ctx, naming.ScyllaDBDatacenterName(sc, &dc), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + svc, err := clusterClient.KubeAdminClient().CoreV1().Services(*dcStatus.RemoteNamespaceName).Get(ctx, naming.MemberServiceName(sdc.Spec.Racks[0], sdc, 0), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + pod, err := clusterClient.KubeAdminClient().CoreV1().Pods(*dcStatus.RemoteNamespaceName).Get(ctx, naming.PodNameFromService(svc), metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + host, err := controllerhelpers.GetScyllaHost(sdc, svc, pod) + o.Expect(err).NotTo(o.HaveOccurred()) + + scyllaConfigClient := scyllaclient.NewConfigClient(host, scyllaDBManagerAgentRackAuthToken) + + framework.By("Verifying if configuration of ScyllaDB is being used by %q datacenter", dc.Name) + gotReadRequestTimeoutInMs, err := scyllaConfigClient.ReadRequestTimeoutInMs(ctx) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(gotReadRequestTimeoutInMs).To(o.Equal(int64(scyllaDBRackReadRequestTimeoutInMs))) + } + }) +}) diff --git a/test/e2e/utils/verification/scylladbcluster/verify.go b/test/e2e/utils/verification/scylladbcluster/verify.go index c4ccd190a06..674313c151d 100644 --- a/test/e2e/utils/verification/scylladbcluster/verify.go +++ b/test/e2e/utils/verification/scylladbcluster/verify.go @@ -104,6 +104,22 @@ func Verify(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster, rkcClusterM condType: "RemoteRemoteOwnerControllerDegraded", status: metav1.ConditionFalse, }, + { + condType: "RemoteConfigMapControllerProgressing", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteConfigMapControllerDegraded", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteSecretControllerProgressing", + status: metav1.ConditionFalse, + }, + { + condType: "RemoteSecretControllerDegraded", + status: metav1.ConditionFalse, + }, } expectedConditions := make([]interface{}, 0, len(condList))