refactor(e2e): move watch-selector onto the harness

226 lines to 159, and the first use of WithOperatorArgs. The Logging literal
stays: fixture.WithFluentbit would add Network.Keepalive and WithFluentd would
add Resources, BufferVolumeMetrics and Workers, none of which this suite has.

Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com>

Author: vyncint

e2e/watch-selector/watch_selector_test.go

@@ -15,197 +15,130 @@
 package watch_selector
 
 import (
-	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
-	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/e2e-framework/third_party/helm"
 
 	"github.com/kube-logging/logging-operator/e2e/common"
-	"github.com/kube-logging/logging-operator/e2e/common/setup"
+	"github.com/kube-logging/logging-operator/e2e/internal/harness"
 	"github.com/kube-logging/logging-operator/e2e/internal/wait"
 	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
 )
 
-var TestTempDir string
+const (
+	ns      = "test"
+	release = "e2e"
 
-func init() {
-	var ok bool
-	TestTempDir, ok = os.LookupEnv("PROJECT_DIR")
-	if !ok {
-		TestTempDir = "../.."
-	}
-	TestTempDir = filepath.Join(TestTempDir, "build/_test")
-	err := os.MkdirAll(TestTempDir, os.FileMode(0o755))
-	if err != nil {
-		panic(err)
-	}
-}
+	// unmanagedNS is created by the fluent chart, not by the harness.
+	unmanagedNS = "fluentd"
+)
 
 func TestWatchSelectors(t *testing.T) {
-	common.Initialize(t)
-	ns := "test"
-	releaseNameOverride := "e2e"
-	common.WithCluster("watch-selector", t, func(t *testing.T, c common.Cluster) {
-		setup.LoggingOperator(t, c, setup.LoggingOperatorOptionFunc(func(options *setup.LoggingOperatorOptions) {
-			options.Namespace = ns
-			options.NameOverride = releaseNameOverride
-			options.Args = []string{"-enable-leader-election=true", "-watch-labeled-children=true", "-watch-labeled-secrets=true"}
-		}))
-
-		ctx := context.Background()
-
-		// Managed logging resource which creates a fluentd pod with a secret named: watch-selector-test-fluentd
-		logging := v1beta1.Logging{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "watch-selector-test",
-				Namespace: ns,
-			},
-			Spec: v1beta1.LoggingSpec{
-				ControlNamespace: ns,
-				FluentbitSpec: &v1beta1.FluentbitSpec{
-					ConfigHotReload: &v1beta1.HotReload{
-						Image: v1beta1.ImageSpec{
-							Repository: common.ConfigReloaderRepo,
-							Tag:        common.ConfigReloaderTag,
-						},
-					},
-					BufferVolumeImage: v1beta1.ImageSpec{
-						Repository: common.NodeExporterRepo,
-						Tag:        common.NodeExporterTag,
-					},
-				},
-				FluentdSpec: &v1beta1.FluentdSpec{
+	env := harness.New(t).
+		WithCluster("watch-selector").
+		WithRelease(release).
+		WithControlNamespace(ns).
+		WithOperatorArgs("-enable-leader-election=true", "-watch-labeled-children=true", "-watch-labeled-secrets=true").
+		Start()
+
+	logging := &v1beta1.Logging{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "watch-selector-test",
+			Namespace: ns,
+		},
+		Spec: v1beta1.LoggingSpec{
+			ControlNamespace: ns,
+			FluentbitSpec: &v1beta1.FluentbitSpec{
+				ConfigHotReload: &v1beta1.HotReload{
 					Image: v1beta1.ImageSpec{
-						Repository: common.FluentdImageRepo,
-						Tag:        common.FluentdImageTag,
-					},
-					ConfigReloaderImage: v1beta1.ImageSpec{
 						Repository: common.ConfigReloaderRepo,
 						Tag:        common.ConfigReloaderTag,
 					},
-					BufferVolumeImage: v1beta1.ImageSpec{
-						Repository: common.NodeExporterRepo,
-						Tag:        common.NodeExporterTag,
-					},
+				},
+				BufferVolumeImage: v1beta1.ImageSpec{
+					Repository: common.NodeExporterRepo,
+					Tag:        common.NodeExporterTag,
 				},
 			},
-		}
-		common.RequireNoError(t, c.GetClient().Create(ctx, &logging))
-
-		// Unmanaged resources
-		common.RequireNoError(t, installFluentdSts(c))
-
-		unmanagedSecret := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "unmanaged-fluentd-secret",
-				Namespace: ns,
-				Labels: map[string]string{
-					"app": "fluentd",
+			FluentdSpec: &v1beta1.FluentdSpec{
+				Image: v1beta1.ImageSpec{
+					Repository: common.FluentdImageRepo,
+					Tag:        common.FluentdImageTag,
+				},
+				ConfigReloaderImage: v1beta1.ImageSpec{
+					Repository: common.ConfigReloaderRepo,
+					Tag:        common.ConfigReloaderTag,
+				},
+				BufferVolumeImage: v1beta1.ImageSpec{
+					Repository: common.NodeExporterRepo,
+					Tag:        common.NodeExporterTag,
 				},
 			},
-			Data: map[string][]byte{
-				"key": []byte("value"),
+		},
+	}
+	env.Create(logging)
+
+	require.NoError(t, installFluentdSts(env))
+
+	unmanagedSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "unmanaged-fluentd-secret",
+			Namespace: ns,
+			Labels: map[string]string{
+				"app": "fluentd",
 			},
-		}
-		common.RequireNoError(t, c.GetClient().Create(ctx, unmanagedSecret))
-
-		require.Eventually(t, func() bool {
-			if isManagedFluentdPodRunning := wait.PodShouldBeRunning(t, c.GetClient(), client.ObjectKey{Namespace: ns, Name: logging.Name + "-fluentd-0"}); !isManagedFluentdPodRunning() {
-				t.Logf("managed fluentd pod is not running")
-				return false
-			}
-
-			if isUnmanagedFluentdPodRunning := wait.PodShouldBeRunning(t, c.GetClient(), client.ObjectKey{Namespace: "fluentd", Name: "fluentd-0"}); !isUnmanagedFluentdPodRunning() {
-				t.Logf("unmanaged fluentd pod is not running")
-				return false
-			}
-
-			return true
-		}, 5*time.Minute, 3*time.Second)
-
-		deployedLogging := &v1beta1.Logging{}
-		common.RequireNoError(t, c.GetClient().Get(ctx, client.ObjectKeyFromObject(&logging), deployedLogging))
-
-		// Check if the managed resources are actually controlled by the logging resource
-		managedSts := &appsv1.StatefulSet{}
-		common.RequireNoError(t, c.GetClient().Get(ctx, client.ObjectKey{Namespace: ns, Name: deployedLogging.Name + "-fluentd"}, managedSts))
-		stsOwnerRefMeta := metav1.GetControllerOf(managedSts)
-		require.NotNil(t, stsOwnerRefMeta)
-
-		require.Equal(t, deployedLogging.APIVersion, stsOwnerRefMeta.APIVersion)
-		require.Equal(t, deployedLogging.Kind, stsOwnerRefMeta.Kind)
-		require.Equal(t, deployedLogging.Name, stsOwnerRefMeta.Name)
-		require.True(t, *stsOwnerRefMeta.Controller)
-
-		managedSecret := &corev1.Secret{}
-		common.RequireNoError(t, c.GetClient().Get(ctx, client.ObjectKey{Namespace: ns, Name: deployedLogging.Name + "-fluentd"}, managedSecret))
-		secretOwnerRefMeta := metav1.GetControllerOf(managedSecret)
-		require.NotNil(t, secretOwnerRefMeta)
-
-		require.Equal(t, deployedLogging.APIVersion, secretOwnerRefMeta.APIVersion)
-		require.Equal(t, deployedLogging.Kind, secretOwnerRefMeta.Kind)
-		require.Equal(t, deployedLogging.Name, secretOwnerRefMeta.Name)
-		require.True(t, *secretOwnerRefMeta.Controller)
-
-		// Check if the unmanaged resources are actually not controlled by the operator
-		unmanagedSts := &appsv1.StatefulSet{}
-		common.RequireNoError(t, c.GetClient().Get(ctx, client.ObjectKey{Namespace: "fluentd", Name: "fluentd"}, unmanagedSts))
-		secretOwnerRefMeta = metav1.GetControllerOf(unmanagedSts)
-		require.Nil(t, secretOwnerRefMeta)
-
-		secret := &corev1.Secret{}
-		common.RequireNoError(t, c.GetClient().Get(ctx, client.ObjectKeyFromObject(unmanagedSecret), secret))
-		secretOwnerRefMeta = metav1.GetControllerOf(secret)
-		require.Nil(t, secretOwnerRefMeta)
-	}, func(t *testing.T, c common.Cluster) error {
-		path := filepath.Join(TestTempDir, fmt.Sprintf("cluster-%s.log", t.Name()))
-		t.Logf("Printing cluster logs to %s", path)
-		err := c.PrintLogs(common.PrintLogConfig{
-			Namespaces: []string{ns, "default"},
-			FilePath:   path,
-			Limit:      100 * 1000,
-		})
-		if err != nil {
-			return err
-		}
-
-		loggingOperatorName := "logging-operator-" + releaseNameOverride
-		t.Logf("Collecting coverage files from logging-operator: %s/%s", ns, loggingOperatorName)
-		err = c.CollectTestCoverageFiles(ns, loggingOperatorName)
-		if err != nil {
-			t.Logf("Failed collecting coverage files: %s", err)
-		}
-
-		return nil
-	}, func(o *cluster.Options) {
-		if o.Scheme == nil {
-			o.Scheme = runtime.NewScheme()
-		}
-		common.RequireNoError(t, v1beta1.AddToScheme(o.Scheme))
-		common.RequireNoError(t, apiextensionsv1.AddToScheme(o.Scheme))
-		common.RequireNoError(t, appsv1.AddToScheme(o.Scheme))
-		common.RequireNoError(t, batchv1.AddToScheme(o.Scheme))
-		common.RequireNoError(t, corev1.AddToScheme(o.Scheme))
-		common.RequireNoError(t, rbacv1.AddToScheme(o.Scheme))
-	})
+		},
+		Data: map[string][]byte{
+			"key": []byte("value"),
+		},
+	}
+	env.Create(unmanagedSecret)
+
+	env.WaitForRunning(
+		wait.Pod(ns, logging.Name+"-fluentd-0"),
+		wait.Pod(unmanagedNS, "fluentd-0"),
+	)
+
+	deployedLogging := &v1beta1.Logging{}
+	require.NoError(t, env.Client.Get(env.Ctx, client.ObjectKeyFromObject(logging), deployedLogging))
+
+	// The managed resources have to be owned by the logging resource.
+	managedSts := &appsv1.StatefulSet{}
+	require.NoError(t, env.Client.Get(env.Ctx, client.ObjectKey{Namespace: ns, Name: deployedLogging.Name + "-fluentd"}, managedSts))
+	requireOwnedBy(t, deployedLogging, metav1.GetControllerOf(managedSts))
+
+	managedSecret := &corev1.Secret{}
+	require.NoError(t, env.Client.Get(env.Ctx, client.ObjectKey{Namespace: ns, Name: deployedLogging.Name + "-fluentd"}, managedSecret))
+	requireOwnedBy(t, deployedLogging, metav1.GetControllerOf(managedSecret))
+
+	// The unmanaged ones have to be left alone.
+	unmanagedSts := &appsv1.StatefulSet{}
+	require.NoError(t, env.Client.Get(env.Ctx, client.ObjectKey{Namespace: unmanagedNS, Name: "fluentd"}, unmanagedSts))
+	require.Nil(t, metav1.GetControllerOf(unmanagedSts))
+
+	secret := &corev1.Secret{}
+	require.NoError(t, env.Client.Get(env.Ctx, client.ObjectKeyFromObject(unmanagedSecret), secret))
+	require.Nil(t, metav1.GetControllerOf(secret))
+}
+
+func requireOwnedBy(t *testing.T, owner *v1beta1.Logging, ref *metav1.OwnerReference) {
+	t.Helper()
+
+	require.NotNil(t, ref)
+	require.Equal(t, owner.APIVersion, ref.APIVersion)
+	require.Equal(t, owner.Kind, ref.Kind)
+	require.Equal(t, owner.Name, ref.Name)
+	require.True(t, *ref.Controller)
 }
 
-func installFluentdSts(c common.Cluster) error {
-	manager := helm.New(c.KubeConfigFilePath())
+func installFluentdSts(env *harness.Env) error {
+	manager := helm.New(env.Cluster.KubeConfigFilePath())
 
 	if err := manager.RunRepo(helm.WithArgs("add", "fluent", "https://fluent.github.io/helm-charts")); err != nil {
 		return fmt.Errorf("failed to add fluent repo: %v", err)
@@ -215,7 +148,7 @@ func installFluentdSts(c common.Cluster) error {
 		helm.WithName("fluentd"),
 		helm.WithChart("fluent/fluentd"),
 		helm.WithArgs("--create-namespace"),
-		helm.WithNamespace("fluentd"),
+		helm.WithNamespace(unmanagedNS),
 		helm.WithArgs("--set", "kind=StatefulSet"),
 		helm.WithWait(),
 	); err != nil {