Refactor parallel download testsuite for dynamic test generation

Author: yaozile123

Makefile

@@ -15,6 +15,10 @@
 
 BINDIR ?= $(shell pwd)/bin
 
+ifeq ($(ENABLE_ZB), true)
+    export GCSFUSE_CLIENT_PROTOCOL = grpc
+endif
+
 # The various gcloud and kubectl checks can be slow. Setting SKIP_VAR=true when
 # making only targets like driver, sidecar-mounter and webook will speed up your
 # life.

pkg/sidecar_mounter/sidecar_mounter_config.go

@@ -80,6 +80,9 @@ type sidecarRetryConfig struct {
 
 var prometheusPort = 62990
 
+// disallowedFlags is a map of flags that are not allowed to be passed to gcsfuse directly.
+// Note: If you add more disallowed flags here, you should update the disallowedFlagsMapping
+// in test/e2e/utils/utils.go to ensure they are correctly translated to the config file format.
 var disallowedFlags = map[string]bool{
 	"temp-dir":                 true,
 	"config-file":              true,

test/e2e/e2e_test.go

@@ -18,7 +18,6 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -33,6 +32,7 @@ import (
 	"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/cloud_provider/metadata"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -54,6 +54,7 @@ var (
 
 var _ = func() bool {
 	testing.Init()
+
 	if os.Getenv(clientcmd.RecommendedConfigPathEnvVar) == "" {
 		kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
 		os.Setenv(clientcmd.RecommendedConfigPathEnvVar, kubeconfig)
@@ -84,6 +85,8 @@ var _ = func() bool {
 	if err != nil {
 		klog.Fatalf("Failed to create fake meta data service: %v", err)
 	}
+
+	testsuites.GCSFuseVersionStr = specs.GetGCSFuseVersion()
 	return true
 }()
 
@@ -102,13 +105,6 @@ func TestE2E(t *testing.T) {
 	ginkgo.RunSpecs(t, "Cloud Storage FUSE CSI Driver", suiteConfig, reporterConfig)
 }
 
-var _ = ginkgo.SynchronizedBeforeSuite(func(ctx context.Context) []byte {
-	gcsfuseVersion := specs.GetGCSFuseVersion(ctx)
-	return []byte(gcsfuseVersion)
-}, func(ctx context.Context, data []byte) {
-	os.Setenv(specs.GcsfuseVersionVarName, string(data))
-})
-
 var _ = ginkgo.Describe("E2E Test Suite", func() {
 	GCSFuseCSITestSuites := func() []func() storageframework.TestSuite {
 		suites := []func() storageframework.TestSuite{
@@ -156,6 +152,15 @@ var _ = ginkgo.Describe("E2E Test Suite", func() {
 	testDriverHNS := specs.InitGCSFuseCSITestDriver(c, m, *bucketLocation, *skipGcpSaTest, true, *clientProtocol, *zbFlag)
 
 	ginkgo.Context(fmt.Sprintf("[Driver: %s HNS]", testDriverHNS.GetDriverInfo().Name), func() {
+		// Skip HNS tests suites for ZB since enable ZB will automatically enable HNS
+		// And the test cases in GCSFuseCSITestSuites already includes HNS tests
+		if *zbFlag {
+			ginkgo.It("should be skipped since ZB already enables HNS", func() {
+				ginkgo.Skip("Skipping HNS tests suites for ZB since enable ZB will automatically enable HNS")
+			})
+			return
+		}
+
 		storageframework.DefineTestSuites(testDriverHNS, GCSFuseCSITestSuitesHNS)
 	})
 })

test/e2e/specs/specs.go

@@ -39,10 +39,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/klog/v2"
 
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
@@ -121,8 +119,6 @@ const (
 	driverDaemonsetLabel   = "k8s-app=gcs-fuse-csi-driver"
 
 	IsOSSEnvVar = "IS_OSS"
-
-	GcsfuseVersionVarName = "gcsfuse-version"
 )
 
 var InvalidVolume = fmt.Sprintf("non-existent-test-bucket-%s", rand.String(8))
@@ -1083,107 +1079,14 @@ func (t *TestJob) Cleanup(ctx context.Context) {
 	framework.ExpectNoError(err)
 }
 
-func GetGCSFuseVersion(ctx context.Context) string {
-	config, err := clientcmd.BuildConfigFromFlags("", framework.TestContext.KubeConfig)
-	framework.ExpectNoError(err)
-
-	cs, err := clientset.NewForConfig(config)
-	framework.ExpectNoError(err)
-
-	versionData := os.Getenv(GcsfuseVersionVarName)
-	if versionData != "" {
-		return versionData
-	}
-	klog.Infof("GCSFuseVersion env var has not been set will retrieve it manually")
-
-	gcsfuseVersion, err := deployGCSFuseVersionFetcherPod(ctx, cs)
-	if err != nil {
-		klog.Errorf("Failed to deploy GCS Fuse version fetcher pod: %v", err)
-		framework.ExpectNoError(err)
-	}
-
-	return gcsfuseVersion
-}
-
-func deployGCSFuseVersionFetcherPod(ctx context.Context, clientset clientset.Interface) (string, error) {
-	configMaps, err := clientset.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{
-		FieldSelector: "metadata.name=gcsfusecsi-image-config",
-	})
-	framework.ExpectNoError(err)
-	if len(configMaps.Items) != 1 {
-		framework.Failf("expected one config map `gcsfusecsi-image-config` but found %d", len(configMaps.Items))
-	}
-
-	sidecarImageConfig := configMaps.Items[0]
-
-	image := sidecarImageConfig.Data["sidecar-image"]
-	if image == "" {
-		framework.Failf("expected data for key `sidecar-image` in the config map `gcsfusecsi-image-config`")
-	}
-
-	pod := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: "gcsfuse-version-fetcher-",
-			Namespace:    utils.DefaultNamespace,
-		},
-		Spec: corev1.PodSpec{
-			TerminationGracePeriodSeconds: ptr.To(int64(0)),
-			Containers: []corev1.Container{
-				{
-					Name:    webhook.GcsFuseSidecarName,
-					Image:   image,
-					Command: []string{"/gcsfuse", "--version"},
-				},
-				{
-					Name:    "sleeper",
-					Image:   "busybox",
-					Command: []string{"sleep", "infinity"},
-				},
-			},
-			RestartPolicy: corev1.RestartPolicyNever,
-			Tolerations: []corev1.Toleration{
-				{Operator: corev1.TolerationOpExists},
-			},
-		},
-	}
-
-	createdPod, err := clientset.CoreV1().Pods(utils.DefaultNamespace).Create(ctx, pod, metav1.CreateOptions{})
-	if createdPod != nil {
-		defer clientset.CoreV1().Pods(utils.DefaultNamespace).Delete(context.Background(), createdPod.Name, metav1.DeleteOptions{})
-	}
-	framework.ExpectNoError(err,
-		"Pods.Create should succeed, but failed with error message: %v", err)
-
-	e2epod.WaitForPodRunningInNamespace(ctx, clientset, createdPod)
-	klog.Infof("Pod %s is running, waiting 60s before fetching GCSFuse version from logs", createdPod.Name)
-	time.Sleep(60 * time.Second)
-
-	var logs []byte
-	req := clientset.CoreV1().Pods(utils.DefaultNamespace).GetLogs(createdPod.Name, &corev1.PodLogOptions{Container: webhook.GcsFuseSidecarName})
-	err = wait.PollUntilContextTimeout(ctx, 30*time.Second, time.Minute*10, true, func(context.Context) (bool, error) {
-		logs, err = req.DoRaw(ctx)
-		if err != nil {
-			framework.Logf("failed to read pod logs, retrying: %v", err)
-			return false, nil
-		}
-		return true, nil
-	})
-	if err != nil {
-		return "", fmt.Errorf("failed to read pod logs: %w", err)
-	}
-
-	output := string(logs)
-
-	// Parse: "gcsfuse version 3.7.1-gke.0 ..."
-	l := strings.Split(strings.TrimSpace(output), " ")
-	if len(l) <= 2 {
-		return "", fmt.Errorf("unexpected version output format: %s", output)
-	}
-	return l[2], nil
+// GetGCSFuseVersion returns the GCSFuse version to be tested.
+// The env var is set by the handler.go before running the tests.
+func GetGCSFuseVersion() string {
+	return os.Getenv(utils.GcsfuseVersionVarName)
 }
 
-func GCSFuseVersionAndBranch(ctx context.Context) (*version.Version, string) {
-	vStr := GetGCSFuseVersion(ctx)
+func GCSFuseVersionAndBranch() (*version.Version, string) {
+	vStr := GetGCSFuseVersion()
 	v, branch := utils.GCSFuseBranch(vStr)
 	if v == nil {
 		// This happens for master branch builds. We still need to parse the version.

test/e2e/specs/testdriver.go

@@ -248,7 +248,7 @@ func (n *GCSFuseCSITestDriver) CreateVolume(ctx context.Context, config *storage
 			mountOptions += ",client-protocol=grpc"
 		}
 		if n.gcsfuseVersion == nil || n.gcsfuseBranch == "" {
-			n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch(ctx)
+			n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch()
 		}
 		kernelParamsSupported := n.gcsfuseBranch == utils.MasterBranchName || n.gcsfuseVersion.AtLeast(version.MustParseSemantic(utils.MinGCSFuseKernelParamsVersion))
 

test/e2e/testsuites/failed_mount.go

@@ -147,10 +147,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 		defer tPod.Cleanup(ctx)
 
 		ginkgo.By("Checking that the pod has failed mount error")
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
-		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 
 		if enableSidecarBucketAccessCheck && configPrefix == specs.SkipCSIBucketAccessCheckAndFakeVolumePrefix {
 			tPod.WaitForFailedContainerError(ctx, "Error: failed to reserve container name")
@@ -201,10 +198,10 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 
 		ginkgo.By("Checking that the pod has failed mount error")
 
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion()
 		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if configPrefix == specs.SkipCSIBucketAccessCheckAndInvalidVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.9.0-gke.0"))) {
 			if enableSidecarBucketAccessCheck {
 				tPod.WaitForFailedContainerError(ctx, "Error: failed to reserve container name")
@@ -480,10 +477,10 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 	})
 
 	ginkgo.It("should fail when invalid bool mount options are passed", func() {
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion()
 		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if err == nil && v != nil && v.AtLeast(version.MustParseSemantic("v3.8.0-gke.0")) {
 			testcaseInvalidMountOptions(specs.InvalidBoolMountOptionsVolumePrefix, "invalid syntax")
 		}