Merge pull request #1274 from yaozile123/file-cache-testsuit-refactor

File cache testsuit refactor

Author: yaozile123

pkg/sidecar_mounter/sidecar_mounter_config.go

@@ -80,23 +80,28 @@ type sidecarRetryConfig struct {
 
 var prometheusPort = 62990
 
-// disallowedFlags is a map of flags that are not allowed to be passed to gcsfuse directly.
-// Note: If you add more disallowed flags here, you should update the disallowedFlagsMapping
-// in test/e2e/utils/utils.go to ensure they are correctly translated to the config file format.
-var disallowedFlags = map[string]bool{
-	"temp-dir":                 true,
-	"config-file":              true,
-	"foreground":               true,
-	"log-file":                 true,
-	"log-format":               true,
-	"key-file":                 true,
-	"token-url":                true,
-	"reuse-token-from-url":     true,
-	"o":                        true,
-	"cache-dir":                true,
-	"prometheus-port":          true,
-	"kernel-params-file":       true,
-	KernelParamsFileConfigFlag: true,
+// DisallowedFlags is a map of flags that are disallowed to be passed to sidecar mounter directly.
+// They are mapped to their config file style representation so that they can be passed in config file format
+// as a workaround to bypass the disallowed flags validation.
+// Note: If you add a new disallowed flag here that is needed for integration testing,
+// you should also update the ParseConfigFlags() logic in test/e2e/utils/utils.go to handle it.
+var DisallowedFlags = map[string]string{
+	// The following flags are explicitly handled in test/e2e/utils/utils.go
+	"log-file":   "logging:file-path",
+	"log-format": "logging:format",
+	"o":          "o",
+	"cache-dir":  "cache-dir",
+
+	// --- Unused in test/e2e/utils/utils.go ---
+	"temp-dir":                 "temp-dir",
+	"config-file":              "config-file",
+	"foreground":               "foreground",
+	"key-file":                 "gcs-auth:key-file",
+	"token-url":                "gcs-auth:token-url",
+	"reuse-token-from-url":     "gcs-auth:reuse-token-from-url",
+	"prometheus-port":          "prometheus-port",
+	"kernel-params-file":       "file-system:kernel-params-file",
+	KernelParamsFileConfigFlag: KernelParamsFileConfigFlag,
 }
 
 // Fetch the following information from a given socket path:
@@ -218,7 +223,7 @@ func (mc *MountConfig) prepareMountArgs() {
 				continue
 			}
 
-			if disallowedFlags[f] {
+			if _, ok := DisallowedFlags[f]; ok {
 				invalidArgs = append(invalidArgs, arg)
 			} else {
 				configFileFlagMap[f] = v
@@ -239,7 +244,7 @@ func (mc *MountConfig) prepareMountArgs() {
 		}
 
 		flag := argPair[0]
-		if disallowedFlags[flag] {
+		if _, ok := DisallowedFlags[flag]; ok {
 			invalidArgs = append(invalidArgs, arg)
 
 			continue

test/e2e/testsuites/gcsfuse_integration.go

@@ -56,6 +56,7 @@ const (
 	testNameInactiveStreamTimeout = "inactive_stream_timeout"
 	testNameBufferedReads         = "buffered_read"
 	testNameRenameSymlink         = "rename_symlink"
+	testNameRapidAppends          = "rapid_appends"
 
 	testNamePrefixSucceed = "should succeed in "
 
@@ -72,6 +73,15 @@ const (
 	gcsfuseGoEnvSetupFormat = "export GO_VERSION=$(%v) && export GOTOOLCHAIN=go$GO_VERSION && export PATH=$PATH:/usr/local/go/bin"
 )
 
+// testPackageTimeoutMap controls the duration of the `-timeout` flag passed to the underlying `go test` executions.
+// If you need to extend the timeout for a specific test package, add or modify its limit in minutes here.
+var testPackageTimeoutMap = map[string]int{
+	testNameListLargeDir:    60,
+	testNameWriteLargeFiles: 60,
+	testNameReadLargeFiles:  60,
+	testNameRapidAppends:    60,
+}
+
 var GCSFuseVersionStr = ""
 
 const gcsfuseGoVersionLegacyCommand = `grep -o 'go[0-9]\+\.[0-9]\+\.[0-9]\+' ./gcsfuse/tools/cd_scripts/e2e_test.sh | cut -c3-`
@@ -94,6 +104,31 @@ func zbEnabled(driver storageframework.TestDriver) bool {
 	return gcsfuseCSITestDriver.EnableZB
 }
 
+// flatEnabled checks if the Flat Namespace feature is enabled for the given driver.
+func flatEnabled(driver storageframework.TestDriver) bool {
+	gcsfuseCSITestDriver, ok := driver.(*specs.GCSFuseCSITestDriver)
+	gomega.Expect(ok).To(gomega.BeTrue(), "failed to cast storageframework.TestDriver to *specs.GCSFuseCSITestDriver")
+
+	return !gcsfuseCSITestDriver.EnableHierarchicalNamespace && !gcsfuseCSITestDriver.EnableZB
+}
+
+// isConfigCompatible returns true if the parsed config is compatible with the current driver setup.
+func isConfigCompatible(config utils.TestConfig, driver storageframework.TestDriver) bool {
+	if !config.RunOnGke {
+		return false
+	}
+	if !config.Compatible.Flat && flatEnabled(driver) {
+		return false
+	}
+	if !config.Compatible.HNS && hnsEnabled(driver) && !zbEnabled(driver) {
+		return false
+	}
+	if !config.Compatible.Zonal && zbEnabled(driver) {
+		return false
+	}
+	return true
+}
+
 // getGoParsingCommand returns the command to get the go version for the gcsfuse integration tests based on the gcsfuse version and branch.
 // In gcsfuse v3.7.0 the go location was moved to a centralized dir ./gcsfuse/.go-version.
 // If using v3.6.0 or older, we use gcsfuseGoVersionLegacyCommand; otherwise we use gcsfuseCentralizedLocationGoVersionCommand.
@@ -113,6 +148,72 @@ func getClientProtocol(driver storageframework.TestDriver) string {
 	return gcsfuseCSITestDriver.ClientProtocol
 }
 
+type TestCommandConfig struct {
+	TestPkg            string
+	TestName           string
+	GoEnvSetupCmd      string
+	MountPath          string
+	SecondaryMountPath string // Optional
+	BucketName         string
+	OnlyDir            string
+}
+
+// generateTestCommand constructs the test command for validating file cache parameters.
+// This is used dynamically across the file cache logic tests including parallel downloads.
+func generateTestCommand(opts TestCommandConfig) string {
+	timeoutStr := ""
+	if t, ok := testPackageTimeoutMap[opts.TestPkg]; ok {
+		timeoutStr = fmt.Sprintf(" -timeout %dm", t)
+	}
+
+	goTestCmd := fmt.Sprintf("GODEBUG=asyncpreemptoff=1 go test ./%v/... -p 1 --integrationTest -v --config-file=../test_config.yaml%s", opts.TestPkg, timeoutStr)
+	if opts.TestName != "" {
+		goTestCmd = fmt.Sprintf("GODEBUG=asyncpreemptoff=1 go test ./%v/... -p 1 --integrationTest -v -run ^%v$ --config-file=../test_config.yaml%s", opts.TestPkg, opts.TestName, timeoutStr)
+	}
+
+	commandArgs := []string{
+		fmt.Sprintf(gcsfuseGoEnvSetupFormat, opts.GoEnvSetupCmd),
+		fmt.Sprintf("export MOUNTED_DIR=%q", opts.MountPath),
+	}
+
+	if opts.SecondaryMountPath != "" {
+		commandArgs = append(commandArgs, fmt.Sprintf("export MOUNTED_DIR_SECONDARY=%q", opts.SecondaryMountPath))
+	}
+
+	commandArgs = append(commandArgs,
+		fmt.Sprintf("export BUCKET_NAME=%q", opts.BucketName),
+		fmt.Sprintf("export ONLY_DIR=%q", opts.OnlyDir),
+		fmt.Sprintf("cd %v", gcsfuseIntegrationTestsBasePath),
+		goTestCmd,
+	)
+
+	return strings.Join(commandArgs, " && ")
+}
+
+// configureLargeFileResources configures the pod and sidecar resources for memory-intensive large file tests.
+// Note: We only increase memory limits for specific tests like testNameWriteLargeFiles, testNameReadLargeFiles
+// and testNameRapidAppends. Other test cases run stable within the default memory for now
+// limits and do not require additional resources.
+func configureLargeFileResources(tPod *specs.TestPod, testNameOrPkg string, driver storageframework.TestDriver) (string, string) {
+	sidecarMemoryLimit := defaultSidecarMemoryLimit
+	sidecarMemoryRequest := defaultSidecarMemoryRequest
+
+	if testNameOrPkg == testNameWriteLargeFiles || testNameOrPkg == testNameReadLargeFiles {
+		tPod.SetResource("1", "8Gi", "5Gi")
+		sidecarMemoryLimit = "1Gi"
+		if zbEnabled(driver) {
+			sidecarMemoryRequest = "1Gi"
+			sidecarMemoryLimit = "2Gi"
+		}
+	}
+	if testNameOrPkg == testNameRapidAppends {
+		sidecarMemoryRequest = "2Gi"
+		sidecarMemoryLimit = "3Gi"
+	}
+
+	return sidecarMemoryRequest, sidecarMemoryLimit
+}
+
 type gcsFuseCSIGCSFuseIntegrationTestSuite struct {
 	tsInfo storageframework.TestSuiteInfo
 }
@@ -260,17 +361,8 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
 		tPod := specs.NewTestPod(f.ClientSet, f.Namespace)
 		tPod.SetImage(specs.GolangImage)
 		tPod.SetResource("1", "5Gi", "5Gi")
-		sidecarMemoryLimit := defaultSidecarMemoryLimit
-		sidecarMemoryRequest := defaultSidecarMemoryRequest
-
-		if testName == testNameWriteLargeFiles || testName == testNameReadLargeFiles {
-			tPod.SetResource("1", "6Gi", "5Gi")
-			sidecarMemoryLimit = "1Gi"
-			if zbEnabled(driver) {
-				sidecarMemoryRequest = "1Gi"
-				sidecarMemoryLimit = "2Gi"
-			}
-		}
+
+		sidecarMemoryRequest, sidecarMemoryLimit := configureLargeFileResources(tPod, testName, driver)
 
 		mo := l.volumeResource.VolSource.CSI.VolumeAttributes["mountOptions"]
 		if testName == testNameExplicitDir && strings.Contains(mo, "only-dir") {