Refactor parallel download testsuite for dynamic test generation

Author: yaozile123

test/e2e/e2e_test.go

@@ -18,7 +18,6 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -33,6 +32,7 @@ import (
 	"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/cloud_provider/metadata"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -54,6 +54,7 @@ var (
 
 var _ = func() bool {
 	testing.Init()
+
 	if os.Getenv(clientcmd.RecommendedConfigPathEnvVar) == "" {
 		kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
 		os.Setenv(clientcmd.RecommendedConfigPathEnvVar, kubeconfig)
@@ -84,6 +85,8 @@ var _ = func() bool {
 	if err != nil {
 		klog.Fatalf("Failed to create fake meta data service: %v", err)
 	}
+
+	testsuites.GCSFuseVersionStr = specs.GetGCSFuseVersion()
 	return true
 }()
 
@@ -102,13 +105,6 @@ func TestE2E(t *testing.T) {
 	ginkgo.RunSpecs(t, "Cloud Storage FUSE CSI Driver", suiteConfig, reporterConfig)
 }
 
-var _ = ginkgo.SynchronizedBeforeSuite(func(ctx context.Context) []byte {
-	gcsfuseVersion := specs.GetGCSFuseVersion(ctx)
-	return []byte(gcsfuseVersion)
-}, func(ctx context.Context, data []byte) {
-	os.Setenv(specs.GcsfuseVersionVarName, string(data))
-})
-
 var _ = ginkgo.Describe("E2E Test Suite", func() {
 	GCSFuseCSITestSuites := func() []func() storageframework.TestSuite {
 		suites := []func() storageframework.TestSuite{
@@ -156,6 +152,15 @@ var _ = ginkgo.Describe("E2E Test Suite", func() {
 	testDriverHNS := specs.InitGCSFuseCSITestDriver(c, m, *bucketLocation, *skipGcpSaTest, true, *clientProtocol, *zbFlag)
 
 	ginkgo.Context(fmt.Sprintf("[Driver: %s HNS]", testDriverHNS.GetDriverInfo().Name), func() {
+		// Skip HNS tests suites for ZB since enable ZB will automatically enable HNS
+		// And the test cases in GCSFuseCSITestSuites already includes HNS tests
+		if *zbFlag {
+			ginkgo.It("should be skipped since ZB already enables HNS", func() {
+				ginkgo.Skip("Skipping HNS tests suites for ZB since enable ZB will automatically enable HNS")
+			})
+			return
+		}
+
 		storageframework.DefineTestSuites(testDriverHNS, GCSFuseCSITestSuitesHNS)
 	})
 })

test/e2e/specs/specs.go

@@ -38,10 +38,8 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/klog/v2"
 
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
@@ -120,8 +118,6 @@ const (
 	driverDaemonsetLabel   = "k8s-app=gcs-fuse-csi-driver"
 
 	IsOSSEnvVar = "IS_OSS"
-
-	GcsfuseVersionVarName = "gcsfuse-version"
 )
 
 // Note to developers adding new testing methods - Please check the code path of newly added methods and ensure that those requiring
@@ -1080,107 +1076,14 @@ func (t *TestJob) Cleanup(ctx context.Context) {
 	framework.ExpectNoError(err)
 }
 
-func GetGCSFuseVersion(ctx context.Context) string {
-	config, err := clientcmd.BuildConfigFromFlags("", framework.TestContext.KubeConfig)
-	framework.ExpectNoError(err)
-
-	cs, err := clientset.NewForConfig(config)
-	framework.ExpectNoError(err)
-
-	versionData := os.Getenv(GcsfuseVersionVarName)
-	if versionData != "" {
-		return versionData
-	}
-	klog.Infof("GCSFuseVersion env var has not been set will retrieve it manually")
-
-	gcsfuseVersion, err := deployGCSFuseVersionFetcherPod(ctx, cs)
-	if err != nil {
-		klog.Errorf("Failed to deploy GCS Fuse version fetcher pod: %v", err)
-		framework.ExpectNoError(err)
-	}
-
-	return gcsfuseVersion
-}
-
-func deployGCSFuseVersionFetcherPod(ctx context.Context, clientset clientset.Interface) (string, error) {
-	configMaps, err := clientset.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{
-		FieldSelector: "metadata.name=gcsfusecsi-image-config",
-	})
-	framework.ExpectNoError(err)
-	if len(configMaps.Items) != 1 {
-		framework.Failf("expected one config map `gcsfusecsi-image-config` but found %d", len(configMaps.Items))
-	}
-
-	sidecarImageConfig := configMaps.Items[0]
-
-	image := sidecarImageConfig.Data["sidecar-image"]
-	if image == "" {
-		framework.Failf("expected data for key `sidecar-image` in the config map `gcsfusecsi-image-config`")
-	}
-
-	pod := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: "gcsfuse-version-fetcher-",
-			Namespace:    utils.DefaultNamespace,
-		},
-		Spec: corev1.PodSpec{
-			TerminationGracePeriodSeconds: ptr.To(int64(0)),
-			Containers: []corev1.Container{
-				{
-					Name:    webhook.GcsFuseSidecarName,
-					Image:   image,
-					Command: []string{"/gcsfuse", "--version"},
-				},
-				{
-					Name:    "sleeper",
-					Image:   "busybox",
-					Command: []string{"sleep", "infinity"},
-				},
-			},
-			RestartPolicy: corev1.RestartPolicyNever,
-			Tolerations: []corev1.Toleration{
-				{Operator: corev1.TolerationOpExists},
-			},
-		},
-	}
-
-	createdPod, err := clientset.CoreV1().Pods(utils.DefaultNamespace).Create(ctx, pod, metav1.CreateOptions{})
-	if createdPod != nil {
-		defer clientset.CoreV1().Pods(utils.DefaultNamespace).Delete(context.Background(), createdPod.Name, metav1.DeleteOptions{})
-	}
-	framework.ExpectNoError(err,
-		"Pods.Create should succeed, but failed with error message: %v", err)
-
-	e2epod.WaitForPodRunningInNamespace(ctx, clientset, createdPod)
-	klog.Infof("Pod %s is running, waiting 60s before fetching GCSFuse version from logs", createdPod.Name)
-	time.Sleep(60 * time.Second)
-
-	var logs []byte
-	req := clientset.CoreV1().Pods(utils.DefaultNamespace).GetLogs(createdPod.Name, &corev1.PodLogOptions{Container: webhook.GcsFuseSidecarName})
-	err = wait.PollUntilContextTimeout(ctx, 30*time.Second, time.Minute*10, true, func(context.Context) (bool, error) {
-		logs, err = req.DoRaw(ctx)
-		if err != nil {
-			framework.Logf("failed to read pod logs, retrying: %v", err)
-			return false, nil
-		}
-		return true, nil
-	})
-	if err != nil {
-		return "", fmt.Errorf("failed to read pod logs: %w", err)
-	}
-
-	output := string(logs)
-
-	// Parse: "gcsfuse version 3.7.1-gke.0 ..."
-	l := strings.Split(strings.TrimSpace(output), " ")
-	if len(l) <= 2 {
-		return "", fmt.Errorf("unexpected version output format: %s", output)
-	}
-	return l[2], nil
+// GetGCSFuseVersion returns the GCSFuse version to be tested.
+// The env var is set by the handler.go before running the tests.
+func GetGCSFuseVersion() string {
+	return os.Getenv(utils.GcsfuseVersionVarName)
 }
 
-func GCSFuseVersionAndBranch(ctx context.Context) (*version.Version, string) {
-	vStr := GetGCSFuseVersion(ctx)
+func GCSFuseVersionAndBranch() (*version.Version, string) {
+	vStr := GetGCSFuseVersion()
 	v, branch := utils.GCSFuseBranch(vStr)
 	if v == nil {
 		// This happens for master branch builds. We still need to parse the version.

test/e2e/specs/testdriver.go

@@ -248,7 +248,7 @@ func (n *GCSFuseCSITestDriver) CreateVolume(ctx context.Context, config *storage
 			mountOptions += ",client-protocol=grpc"
 		}
 		if n.gcsfuseVersion == nil || n.gcsfuseBranch == "" {
-			n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch(ctx)
+			n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch()
 		}
 		kernelParamsSupported := n.gcsfuseBranch == utils.MasterBranchName || n.gcsfuseVersion.AtLeast(version.MustParseSemantic(utils.MinGCSFuseKernelParamsVersion))
 

test/e2e/testsuites/failed_mount.go

@@ -153,10 +153,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 		} else {
 			tPod.WaitForFailedMountError(ctx, codes.NotFound.String())
 		}
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
-		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if configPrefix == specs.SkipCSIBucketAccessCheckAndFakeVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.5.0-gke.0"))) {
 			tPod.WaitForLog(ctx, webhook.GcsFuseSidecarName, "bucket does not exist")
 		} else {
@@ -198,10 +195,10 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 
 		ginkgo.By("Checking that the pod has failed mount error")
 
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion()
 		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if configPrefix == specs.SkipCSIBucketAccessCheckAndInvalidVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.9.0-gke.0"))) {
 			if enableSidecarBucketAccessCheck {
 				tPod.WaitForFailedContainerError(ctx, "Error: failed to reserve container name")