Refactor GetGCSFuseVersion to bypass ginkgo framework assertions

Author: yaozile123

test/e2e/e2e_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/cloud_provider/metadata"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -70,17 +71,6 @@ var _ = func() bool {
 		klog.Fatalf("Failed to configure k8s client: %v", err)
 	}
 
-	// klog.Info("Fetching GCSFuse version from cluster")
-	// v := specs.GetGCSFuseVersion(context.Background(), c)
-	// klog.Infof("Detected GCSFuse version: %v", v)
-
-	// klog.Info("Loading test config")
-	// Load the integration test configurations before Ginkgo parses the test tree
-	// if err := utils.LoadTestConfig(v); err != nil {
-	// 	klog.Fatalf("Failed to load test_config.yaml for version %v: %v", v, err)
-	// }
-	// klog.Infof("Successfully loaded %d test packages from test_config.yaml", len(utils.LoadedTestPackages))
-
 	kubeConfig, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig)
 	if err != nil {
 		klog.Fatalf("Failed to load kube config: %v", err)
@@ -96,9 +86,38 @@ var _ = func() bool {
 	if err != nil {
 		klog.Fatalf("Failed to create fake meta data service: %v", err)
 	}
+
+	testsuites.GCSFuseVersionStr = fetchGCSFuseVersion()
 	return true
 }()
 
+// fetchGCSFuseVersion initiates a minimal, standalone framework to query the cluster sidecar
+// during the init() phase. This enables dynamic test tree construction depending on the
+// version found in the remote cluster.
+func fetchGCSFuseVersion() string {
+	config, err := framework.LoadConfig()
+	if err != nil {
+		klog.Fatalf("failed to load kube config for standalone version fetch: %v", err)
+	}
+	k8sClient, err := k8sclientset.NewForConfig(config)
+	if err != nil {
+		klog.Fatalf("failed to create client for standalone version fetch: %v", err)
+	}
+
+	minimalFramework := &framework.Framework{
+		ClientSet: k8sClient,
+		Namespace: &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "default",
+			},
+		},
+	}
+	ctx := context.Background()
+	versionStr := specs.GetGCSFuseVersion(ctx, minimalFramework)
+	klog.Infof("Fetched GCSFuse version globally: %s", versionStr)
+	return versionStr
+}
+
 func TestE2E(t *testing.T) {
 	t.Parallel()
 
@@ -146,25 +165,6 @@ var _ = ginkgo.Describe("E2E Test Suite", func() {
 			suites = append(suites, testsuites.InitGcsFuseCSIProfilesTestSuite)
 		}
 
-		suites = append(suites, []func() storageframework.TestSuite{
-			testsuites.InitGcsFuseCSIVolumesTestSuite,
-			testsuites.InitGcsFuseCSIFailedMountTestSuite,
-			testsuites.InitGcsFuseCSIWorkloadsTestSuite,
-			testsuites.InitGcsFuseCSIMultiVolumeTestSuite,
-			testsuites.InitGcsFuseCSIGCSFuseIntegrationTestSuite,
-			testsuites.InitGcsFuseCSIPerformanceTestSuite,
-			testsuites.InitGcsFuseCSISubPathTestSuite,
-			testsuites.InitGcsFuseCSIAutoTerminationTestSuite,
-			testsuites.InitGcsFuseCSIFileCacheTestSuite,
-			testsuites.InitGcsFuseCSIGCSFuseIntegrationFileCacheTestSuite,
-			testsuites.InitGcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite,
-			testsuites.InitGcsFuseCSIIstioTestSuite,
-			testsuites.InitGcsFuseCSIMetricsTestSuite,
-			testsuites.InitGcsFuseCSIMetadataPrefetchTestSuite,
-			testsuites.InitGcsFuseMountTestSuite,
-			testsuites.InitGcsFuseCSIOIDCTestSuite,
-		}...)
-
 		if *kernelParamsFlag {
 			suites = append(suites, testsuites.InitGcsFuseKernelParamsTestSuite)
 		}

test/e2e/specs/specs.go

@@ -1106,16 +1106,21 @@ func deployGCSFuseVersionFetcherPod(ctx context.Context, clientset clientset.Int
 	configMaps, err := clientset.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{
 		FieldSelector: "metadata.name=gcsfusecsi-image-config",
 	})
-	framework.ExpectNoError(err)
+	if err != nil {
+		klog.Errorf("Expected no error when list configmaps, but found: %v", err)
+		return ""
+	}
 	if len(configMaps.Items) != 1 {
-		framework.Failf("expected one config map `gcsfusecsi-image-config` but found %d", len(configMaps.Items))
+		klog.Errorf("Expected 1 gcsfusecsi-image-config ConfigMap, found %d", len(configMaps.Items))
+		return ""
 	}
 
 	sidecarImageConfig := configMaps.Items[0]
 
 	image := sidecarImageConfig.Data["sidecar-image"]
 	if image == "" {
-		framework.Failf("expected data for key `sidecar-image` in the config map `gcsfusecsi-image-config`")
+		klog.Errorf("sidecar-image data is empty in configmap")
+		return ""
 	}
 
 	pod := &corev1.Pod{

test/e2e/testsuites/failed_mount.go

@@ -153,10 +153,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 		} else {
 			tPod.WaitForFailedMountError(ctx, codes.NotFound.String())
 		}
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
-		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if configPrefix == specs.SkipCSIBucketAccessCheckAndFakeVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.5.0-gke.0"))) {
 			tPod.WaitForLog(ctx, webhook.GcsFuseSidecarName, "bucket does not exist")
 		} else {
@@ -198,10 +195,10 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
 
 		ginkgo.By("Checking that the pod has failed mount error")
 
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx, f)
 		}
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		if configPrefix == specs.SkipCSIBucketAccessCheckAndInvalidVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.9.0-gke.0"))) {
 			if enableSidecarBucketAccessCheck {
 				tPod.WaitForFailedContainerError(ctx, "Error: failed to reserve container name")

test/e2e/testsuites/gcsfuse_integration.go

@@ -70,7 +70,7 @@ const (
 	gkeTempDir = "/gcsfuse-tmp"
 )
 
-var gcsfuseVersionStr = ""
+var GCSFuseVersionStr = ""
 
 const gcsfuseGoVersionLegacyCommand = `grep -o 'go[0-9]\+\.[0-9]\+\.[0-9]\+' ./gcsfuse/tools/cd_scripts/e2e_test.sh | cut -c3-`
 const gcsfuseCentralizedLocationGoVersionCommand = `cat ./gcsfuse/.go-version | tr -d '[:space:]'`
@@ -242,13 +242,15 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 		}
 
 		ginkgo.By("Checking GCSFuse version and skip test if needed")
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		// Check if the GCSFuseVersion is already set, if not, we should set it.
+		// This is for the case that we run this test suite individually.
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx, f)
 		}
-		gcsfuseVersion := version.MustParseSemantic(gcsfuseVersionStr)
+		gcsfuseVersion := version.MustParseSemantic(GCSFuseVersionStr)
 
-		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, gcsfuseVersionStr))
-		gcsfuseTestBranch := skipTestOrProceedWithBranch(gcsfuseVersionStr, testName)
+		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, GCSFuseVersionStr))
+		gcsfuseTestBranch := skipTestOrProceedWithBranch(GCSFuseVersionStr, testName)
 		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse branch %v", testName, gcsfuseTestBranch))
 
 		ginkgo.By("Configuring the test pod")
@@ -590,7 +592,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 		init()
 		defer cleanup()
 
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		// If error != nil, this means we've autogenerated a tag (meaning we run from HEAD)
 		// Otherise, we have a valid tag, and we compare against the supported release.
 		if (err != nil || v.AtLeast(version.MustParseSemantic("2.9.0-gke.0"))) && getClientProtocol(driver) != "grpc" {
@@ -610,7 +612,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 	ginkgo.It(testNamePrefixSucceed+testNameLocalFile+testNameSuffix(1), func() {
 		init()
 		defer cleanup()
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 
 		// If error != nil, this means we've autogenerated a tag (meaning we run from HEAD)
 		// Otherise, we have a valid tag, and we compare against the supported release.
@@ -706,7 +708,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 		init()
 		defer cleanup()
 
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		// If error != nil, this means we've autogenerated a tag (meaning we run from HEAD)
 		// Otherise, we have a valid tag, and we compare against the supported release.
 		if err != nil || v.AtLeast(version.MustParseSemantic("2.7.0-gke.0")) {
@@ -722,7 +724,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 		init()
 		defer cleanup()
 
-		v, err := version.ParseSemantic(gcsfuseVersionStr)
+		v, err := version.ParseSemantic(GCSFuseVersionStr)
 		// If error != nil, this means we've autogenerated a tag (meaning we run from HEAD)
 		// Otherise, we have a valid tag, and we compare against the supported release.
 		if err != nil || v.AtLeast(version.MustParseSemantic("2.7.0-gke.0")) {

test/e2e/testsuites/gcsfuse_integration_file_cache.go

@@ -112,11 +112,11 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor
 
 	gcsfuseIntegrationFileCacheTest := func(testName string, readOnly bool, fileCacheCapacity, fileCacheForRangeRead, metadataCacheTTLSeconds string, mountOptions ...string) {
 		ginkgo.By("Checking GCSFuse version and skip test if needed")
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
+		if GCSFuseVersionStr == "" {
+			GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx, f)
 		}
-		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, gcsfuseVersionStr))
-		gcsfuseTestBranch := skipTestOrProceedWithBranch(gcsfuseVersionStr, testName)
+		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, GCSFuseVersionStr))
+		gcsfuseTestBranch := skipTestOrProceedWithBranch(GCSFuseVersionStr, testName)
 		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse branch %v", testName, gcsfuseTestBranch))
 
 		ginkgo.By("Configuring the test pod")
@@ -134,7 +134,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor
 
 		tPod.SetupTmpVolumeMount("/tmp/gcsfuse_read_cache_test_logs")
 		cacheDir := "cache-dir"
-		gcsfuseVersion := version.MustParseSemantic(gcsfuseVersionStr)
+		gcsfuseVersion := version.MustParseSemantic(GCSFuseVersionStr)
 		if gcsfuseTestBranch == utils.MasterBranchName || gcsfuseVersion.AtLeast(version.MustParseSemantic("v2.4.1-gke.0")) {
 			if hnsEnabled(driver) {
 				cacheDir = "cache-dir-read-cache-hns-true"

test/e2e/testsuites/gcsfuse_integration_file_cache_parallel_downloads.go

@@ -26,8 +26,10 @@ import (
 	"local/test/e2e/utils"
 
 	"github.com/onsi/ginkgo/v2"
+
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
@@ -112,11 +114,8 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 
 	gcsfuseIntegrationFileCacheTest := func(testName string, readOnly bool, fileCacheCapacity, fileCacheForRangeRead, metadataCacheTTLSeconds string, mountOptions ...string) {
 		ginkgo.By("Checking GCSFuse version and skip test if needed")
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx)
-		}
-		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, gcsfuseVersionStr))
-		gcsfuseTestBranch := skipTestOrProceedWithBranch(gcsfuseVersionStr, testName)
+		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, GCSFuseVersionStr))
+		gcsfuseTestBranch := skipTestOrProceedWithBranch(GCSFuseVersionStr, testName)
 		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse branch %v", testName, gcsfuseTestBranch))
 
 		ginkgo.By("Configuring the test pod")
@@ -134,7 +133,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 
 		tPod.SetupTmpVolumeMount("/tmp/gcsfuse_read_cache_test_logs")
 		cacheDir := "cache-dir"
-		gcsfuseVersion := version.MustParseSemantic(gcsfuseVersionStr)
+		gcsfuseVersion := version.MustParseSemantic(GCSFuseVersionStr)
 		if gcsfuseTestBranch == utils.MasterBranchName || gcsfuseVersion.AtLeast(version.MustParseSemantic("v2.4.1-gke.0")) {
 			if hnsEnabled(driver) {
 				cacheDir = "cache-dir-read-cache-hns-true"
@@ -197,11 +196,8 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 	gcsfuseIntegrationFileCacheTestNew := func(testName string, config utils.ParsedConfig) {
 
 		ginkgo.By("Checking GCSFuse version and skip test if needed")
-		if gcsfuseVersionStr == "" {
-			gcsfuseVersionStr = specs.GetGCSFuseVersion(ctx, f)
-		}
-		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, gcsfuseVersionStr))
-		gcsfuseTestBranch := skipTestOrProceedWithBranch(gcsfuseVersionStr, testName)
+		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, GCSFuseVersionStr))
+		gcsfuseTestBranch := skipTestOrProceedWithBranch(GCSFuseVersionStr, testName)
 		ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse branch %v", testName, gcsfuseTestBranch))
 
 		ginkgo.By("Configuring the test pod")
@@ -218,7 +214,6 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 
 		l.volumeResource.VolSource.CSI.VolumeAttributes["fileCacheCapacity"] = config.FileCacheCapacity
 		tPod.SetupTmpVolumeMount(gkeTempDir)
-		gcsfuseVersion := version.MustParseSemantic(gcsfuseVersionStr)
 		framework.Logf("Cache file path: %v", config.CacheDir)
 		tPod.SetupCacheVolumeMount(config.CacheDir, ".volumes/"+volumeName)
 
@@ -257,6 +252,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 		tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("git clone --branch %v https://github.com/GoogleCloudPlatform/gcsfuse.git", gcsfuseTestBranch))
 		tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, "ln -s /usr/bin/python3 /usr/bin/python")
 
+		gcsfuseVersion := version.MustParseSemantic(GCSFuseVersionStr)
 		gcsfuseGoVersionCommand := getGoParsingCommand(*gcsfuseVersion, gcsfuseTestBranch)
 
 		commandArgs := []string{
@@ -475,9 +471,12 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheParallelDownloadsTestSuite) Define
 	// The gcsfuse test_config.yaml is introduced from the gcsfuse v3.5+.
 	// If the gcsfuse version is less than v3.5+, we will use the static tests.
 	// We will remove the static tests in the future.
-	if utils.IsReadFromTestConfig(gcsfuseVersionStr) {
-		generateDynamicTests(gcsfuseVersionStr)
+
+	if utils.IsReadFromTestConfig(GCSFuseVersionStr) {
+		klog.Info("Generating tests based on test config")
+		generateDynamicTests(GCSFuseVersionStr)
 	} else {
+		klog.Info("Generating static tests")
 		generateStaticTests()
 	}
 }

test/e2e/utils/utils.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/onsi/ginkgo/v2"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
@@ -75,9 +76,9 @@ func isVariableSet(v string) bool {
 }
 
 func runCommand(action string, cmd *exec.Cmd) error {
-	cmd.Stdout = os.Stdout
+	cmd.Stdout = ginkgo.GinkgoWriter
 	cmd.Stdin = os.Stdin
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = ginkgo.GinkgoWriter
 
 	klog.Infof("%s", action)
 	klog.Infof("cmd env=%v", cmd.Env)
@@ -278,11 +279,15 @@ func ParseConfigFlags(flagStr string) ParsedConfig {
 // Checks if the gcsfuse version supports test_config.yaml for integration tests.
 func IsReadFromTestConfig(gcsfuseVersionStr string) bool {
 	if gcsfuseVersionStr == "" {
-		// TODO: Remove this after implement the get gcsfuse version from the cluster during test tree init.
+		return false
+	}
+
+	v, branch := GCSFuseBranch(gcsfuseVersionStr)
+	if branch == MasterBranchName {
 		return true
 	}
-	gcsfuseVersion := version.MustParseSemantic(gcsfuseVersionStr)
-	return gcsfuseVersion.AtLeast(version.MustParseSemantic(MinGCSFuseTestConfigVersion)) || gcsfuseVersionStr == MasterBranchName
+
+	return v.AtLeast(version.MustParseSemantic(MinGCSFuseTestConfigVersion))
 }
 
 // Extracts the only-dir UUID from the mountOptions string.