Merge pull request #2245 from tnozicka/unblock-ds-controller

Speed up NodeConfig optimization test

Author: scylla-operator-bot[bot]

pkg/controllerhelpers/wait.go

@@ -11,6 +11,7 @@ import (
 	scyllav1client "github.com/scylladb/scylla-operator/pkg/client/scylla/clientset/versioned/typed/scylla/v1"
 	scyllav1alpha1client "github.com/scylladb/scylla-operator/pkg/client/scylla/clientset/versioned/typed/scylla/v1alpha1"
 	"github.com/scylladb/scylla-operator/pkg/helpers"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -19,6 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
+	appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1"
 	"k8s.io/client-go/tools/cache"
@@ -182,3 +184,7 @@ func WaitForSecretState(ctx context.Context, client corev1client.SecretInterface
 func WaitForServiceState(ctx context.Context, client corev1client.ServiceInterface, name string, options WaitForStateOptions, condition func(*corev1.Service) (bool, error), additionalConditions ...func(*corev1.Service) (bool, error)) (*corev1.Service, error) {
 	return WaitForObjectState[*corev1.Service, *corev1.ServiceList](ctx, client, name, options, condition, additionalConditions...)
 }
+
+func WaitForDaemonSetState(ctx context.Context, client appsv1client.DaemonSetInterface, name string, options WaitForStateOptions, condition func(*appsv1.DaemonSet) (bool, error), additionalConditions ...func(set *appsv1.DaemonSet) (bool, error)) (*appsv1.DaemonSet, error) {
+	return WaitForObjectState[*appsv1.DaemonSet, *appsv1.DaemonSetList](ctx, client, name, options, condition, additionalConditions...)
+}

test/e2e/set/nodeconfig/nodeconfig_optimizations.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 const (
@@ -349,6 +350,33 @@ var _ = g.Describe("NodeConfig Optimizations", framework.Serial, func() {
 		scyllaContainerID, err := controllerhelpers.GetScyllaContainerID(pod)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
+		framework.By("Bumping tuning DaemonSet sync and waiting for it to become healthy")
+
+		dsList, err := f.KubeAdminClient().AppsV1().DaemonSets(naming.ScyllaOperatorNodeTuningNamespace).List(ctx, metav1.ListOptions{
+			LabelSelector: labels.Set{
+				"app.kubernetes.io/name": naming.NodeConfigAppName,
+			}.AsSelector().String(),
+		})
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(dsList.Items).To(o.HaveLen(1), "there should be exactly 1 matching NodeConfig in this test")
+		ds := &dsList.Items[0]
+
+		// At this point the DaemonSet controller in kube-controller-manager is notably rate-limited
+		// because of resource quota failures. We have to trigger an event to reset its queue rate limiter,
+		// or there will be a large delay.
+		ds, err = f.KubeAdminClient().AppsV1().DaemonSets(naming.ScyllaOperatorNodeTuningNamespace).Patch(
+			ctx,
+			ds.Name,
+			types.JSONPatchType,
+			[]byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/annotations/e2e-requeue", "value": %q}]`, time.Now())),
+			metav1.PatchOptions{},
+		)
+
+		ctxTuningDS, ctxTuningDSCancel := utils.ContextForRollout(ctx, sc)
+		defer ctxTuningDSCancel()
+		ds, err = controllerhelpers.WaitForDaemonSetState(ctxTuningDS, f.KubeAdminClient().AppsV1().DaemonSets(ds.Namespace), ds.Name, controllerhelpers.WaitForStateOptions{}, controllerhelpers.IsDaemonSetRolledOut)
+		o.Expect(err).NotTo(o.HaveOccurred())
+
 		framework.By("Waiting for the NodeConfig to deploy")
 		ctx3, ctx3Cancel := context.WithTimeout(ctx, nodeConfigRolloutTimeout)
 		defer ctx3Cancel()