Skip to content

Commit 56b6b8e

Browse files
yaozile123yaozile123
committed
Move GetGCSFuseVersion before ginkgo start
1 parent 4b09dac commit 56b6b8e

10 files changed

Lines changed: 202 additions & 157 deletions

File tree

‎test/e2e/e2e_test.go‎

Lines changed: 24 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ limitations under the License.
1818
package main
1919

2020
import (
21-
"context"
2221
"flag"
2322
"fmt"
2423
"os"
@@ -87,36 +86,36 @@ var _ = func() bool {
8786
klog.Fatalf("Failed to create fake meta data service: %v", err)
8887
}
8988

90-
testsuites.GCSFuseVersionStr = fetchGCSFuseVersion()
89+
testsuites.GCSFuseVersionStr = specs.GetGCSFuseVersion()
9190
return true
9291
}()
9392

9493
// fetchGCSFuseVersion initiates a minimal, standalone framework to query the cluster sidecar
9594
// during the init() phase. This enables dynamic test tree construction depending on the
9695
// version found in the remote cluster.
97-
func fetchGCSFuseVersion() string {
98-
config, err := framework.LoadConfig()
99-
if err != nil {
100-
klog.Fatalf("failed to load kube config for standalone version fetch: %v", err)
101-
}
102-
k8sClient, err := k8sclientset.NewForConfig(config)
103-
if err != nil {
104-
klog.Fatalf("failed to create client for standalone version fetch: %v", err)
105-
}
106-
107-
minimalFramework := &framework.Framework{
108-
ClientSet: k8sClient,
109-
Namespace: &corev1.Namespace{
110-
ObjectMeta: metav1.ObjectMeta{
111-
Name: "default",
112-
},
113-
},
114-
}
115-
ctx := context.Background()
116-
versionStr := specs.GetGCSFuseVersion(ctx, minimalFramework)
117-
klog.Infof("Fetched GCSFuse version globally: %s", versionStr)
118-
return versionStr
119-
}
96+
// func fetchGCSFuseVersion() string {
97+
// config, err := framework.LoadConfig()
98+
// if err != nil {
99+
// klog.Fatalf("failed to load kube config for standalone version fetch: %v", err)
100+
// }
101+
// k8sClient, err := k8sclientset.NewForConfig(config)
102+
// if err != nil {
103+
// klog.Fatalf("failed to create client for standalone version fetch: %v", err)
104+
// }
105+
106+
// minimalFramework := &framework.Framework{
107+
// ClientSet: k8sClient,
108+
// Namespace: &corev1.Namespace{
109+
// ObjectMeta: metav1.ObjectMeta{
110+
// Name: "default",
111+
// },
112+
// },
113+
// }
114+
// ctx := context.Background()
115+
// versionStr := specs.GetGCSFuseVersion(ctx, minimalFramework)
116+
// klog.Infof("Fetched GCSFuse version globally: %s", versionStr)
117+
// return versionStr
118+
// }
120119

121120
func TestE2E(t *testing.T) {
122121
t.Parallel()
@@ -133,13 +132,6 @@ func TestE2E(t *testing.T) {
133132
ginkgo.RunSpecs(t, "Cloud Storage FUSE CSI Driver", suiteConfig, reporterConfig)
134133
}
135134

136-
var _ = ginkgo.SynchronizedBeforeSuite(func(ctx context.Context) []byte {
137-
gcsfuseVersion := specs.GetGCSFuseVersion(ctx)
138-
return []byte(gcsfuseVersion)
139-
}, func(ctx context.Context, data []byte) {
140-
os.Setenv(specs.GcsfuseVersionVarName, string(data))
141-
})
142-
143135
var _ = ginkgo.Describe("E2E Test Suite", func() {
144136
GCSFuseCSITestSuites := func() []func() storageframework.TestSuite {
145137
suites := []func() storageframework.TestSuite{

‎test/e2e/specs/specs.go‎

Lines changed: 24 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,8 @@ import (
3838
"k8s.io/apimachinery/pkg/fields"
3939
"k8s.io/apimachinery/pkg/util/version"
4040
"k8s.io/apimachinery/pkg/util/wait"
41-
"k8s.io/klog/v2"
4241

4342
clientset "k8s.io/client-go/kubernetes"
44-
"k8s.io/client-go/tools/clientcmd"
4543
"k8s.io/kubernetes/pkg/kubelet/events"
4644
"k8s.io/kubernetes/test/e2e/framework"
4745
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
@@ -120,8 +118,6 @@ const (
120118
driverDaemonsetLabel = "k8s-app=gcs-fuse-csi-driver"
121119

122120
IsOSSEnvVar = "IS_OSS"
123-
124-
GcsfuseVersionVarName = "gcsfuse-version"
125121
)
126122

127123
// Note to developers adding new testing methods - Please check the code path of newly added methods and ensure that those requiring
@@ -1080,107 +1076,30 @@ func (t *TestJob) Cleanup(ctx context.Context) {
10801076
framework.ExpectNoError(err)
10811077
}
10821078

1083-
func GetGCSFuseVersion(ctx context.Context) string {
1084-
config, err := clientcmd.BuildConfigFromFlags("", framework.TestContext.KubeConfig)
1085-
framework.ExpectNoError(err)
1086-
1087-
cs, err := clientset.NewForConfig(config)
1088-
framework.ExpectNoError(err)
1089-
1090-
versionData := os.Getenv(GcsfuseVersionVarName)
1091-
if versionData != "" {
1092-
return versionData
1093-
}
1094-
klog.Infof("GCSFuseVersion env var has not been set will retrieve it manually")
1095-
1096-
gcsfuseVersion, err := deployGCSFuseVersionFetcherPod(ctx, cs)
1097-
if err != nil {
1098-
klog.Errorf("Failed to deploy GCS Fuse version fetcher pod: %v", err)
1099-
framework.ExpectNoError(err)
1100-
}
1101-
1102-
return gcsfuseVersion
1103-
}
1104-
1105-
func deployGCSFuseVersionFetcherPod(ctx context.Context, clientset clientset.Interface) (string, error) {
1106-
configMaps, err := clientset.CoreV1().ConfigMaps("").List(ctx, metav1.ListOptions{
1107-
FieldSelector: "metadata.name=gcsfusecsi-image-config",
1108-
})
1109-
framework.ExpectNoError(err)
1110-
if len(configMaps.Items) != 1 {
1111-
framework.Failf("expected one config map `gcsfusecsi-image-config` but found %d", len(configMaps.Items))
1112-
}
1113-
1114-
sidecarImageConfig := configMaps.Items[0]
1115-
1116-
image := sidecarImageConfig.Data["sidecar-image"]
1117-
if image == "" {
1118-
framework.Failf("expected data for key `sidecar-image` in the config map `gcsfusecsi-image-config`")
1119-
}
1120-
1121-
pod := &corev1.Pod{
1122-
ObjectMeta: metav1.ObjectMeta{
1123-
GenerateName: "gcsfuse-version-fetcher-",
1124-
Namespace: utils.DefaultNamespace,
1125-
},
1126-
Spec: corev1.PodSpec{
1127-
TerminationGracePeriodSeconds: ptr.To(int64(0)),
1128-
Containers: []corev1.Container{
1129-
{
1130-
Name: webhook.GcsFuseSidecarName,
1131-
Image: image,
1132-
Command: []string{"/gcsfuse", "--version"},
1133-
},
1134-
{
1135-
Name: "sleeper",
1136-
Image: "busybox",
1137-
Command: []string{"sleep", "infinity"},
1138-
},
1139-
},
1140-
RestartPolicy: corev1.RestartPolicyNever,
1141-
Tolerations: []corev1.Toleration{
1142-
{Operator: corev1.TolerationOpExists},
1143-
},
1144-
},
1145-
}
1146-
1147-
createdPod, err := clientset.CoreV1().Pods(utils.DefaultNamespace).Create(ctx, pod, metav1.CreateOptions{})
1148-
if createdPod != nil {
1149-
defer clientset.CoreV1().Pods(utils.DefaultNamespace).Delete(context.Background(), createdPod.Name, metav1.DeleteOptions{})
1150-
}
1151-
framework.ExpectNoError(err,
1152-
"Pods.Create should succeed, but failed with error message: %v", err)
1153-
1154-
e2epod.WaitForPodRunningInNamespace(ctx, clientset, createdPod)
1155-
klog.Infof("Pod %s is running, waiting 60s before fetching GCSFuse version from logs", createdPod.Name)
1156-
time.Sleep(60 * time.Second)
1157-
1158-
var logs []byte
1159-
req := clientset.CoreV1().Pods(utils.DefaultNamespace).GetLogs(createdPod.Name, &corev1.PodLogOptions{Container: webhook.GcsFuseSidecarName})
1160-
err = wait.PollUntilContextTimeout(ctx, 30*time.Second, time.Minute*10, true, func(context.Context) (bool, error) {
1161-
logs, err = req.DoRaw(ctx)
1162-
if err != nil {
1163-
framework.Logf("failed to read pod logs, retrying: %v", err)
1164-
return false, nil
1165-
}
1166-
return true, nil
1167-
})
1168-
if err != nil {
1169-
return "", fmt.Errorf("failed to read pod logs: %w", err)
1170-
}
1171-
1172-
output := string(logs)
1173-
1174-
// Parse: "gcsfuse version 3.7.1-gke.0 ..."
1175-
l := strings.Split(strings.TrimSpace(output), " ")
1176-
if len(l) <= 2 {
1177-
return "", fmt.Errorf("unexpected version output format: %s", output)
1178-
}
1179-
return l[2], nil
1180-
}
1181-
1182-
func GCSFuseVersionAndBranch(ctx context.Context) (*version.Version, string) {
1183-
vStr := GetGCSFuseVersion(ctx)
1079+
// GetGCSFuseVersion returns the GCSFuse version to be tested.
1080+
// It prioritizes the E2E_GCSFUSE_VERSION environment variable set by the handler,
1081+
// and falls back to reading the static file for local execution without the wrapper.
1082+
func GetGCSFuseVersion() string {
1083+
return os.Getenv(utils.GcsfuseVersionVarName)
1084+
// versionData := os.Getenv(GcsfuseVersionVarName)
1085+
// if versionData != "" {
1086+
// return versionData
1087+
// }
1088+
1089+
// // Fallback to reading file if running purely locally via 'go test' without handler.go wrapper
1090+
// klog.Warningf("GCSFuseVersion env var has not been set. Falling back to reading cmd/sidecar_mounter/gcsfuse_binary")
1091+
// content, err := os.ReadFile("cmd/sidecar_mounter/gcsfuse_binary")
1092+
// if err != nil {
1093+
// content, err = os.ReadFile("../../cmd/sidecar_mounter/gcsfuse_binary")
1094+
// if err != nil {
1095+
// klog.Fatalf("failed to read fallback cmd/sidecar_mounter/gcsfuse_binary: %v", err)
1096+
// }
1097+
// }
1098+
// return strings.TrimSpace(string(content))
1099+
}
1100+
1101+
func GCSFuseVersionAndBranch() (*version.Version, string) {
1102+
vStr := GetGCSFuseVersion()
11841103
v, branch := utils.GCSFuseBranch(vStr)
11851104
if v == nil {
11861105
// This happens for master branch builds. We still need to parse the version.

‎test/e2e/specs/testdriver.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ func (n *GCSFuseCSITestDriver) CreateVolume(ctx context.Context, config *storage
248248
mountOptions += ",client-protocol=grpc"
249249
}
250250
if n.gcsfuseVersion == nil || n.gcsfuseBranch == "" {
251-
n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch(ctx)
251+
n.gcsfuseVersion, n.gcsfuseBranch = GCSFuseVersionAndBranch()
252252
}
253253
kernelParamsSupported := n.gcsfuseBranch == utils.MasterBranchName || n.gcsfuseVersion.AtLeast(version.MustParseSemantic(utils.MinGCSFuseKernelParamsVersion))
254254

‎test/e2e/testsuites/failed_mount.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
196196
ginkgo.By("Checking that the pod has failed mount error")
197197

198198
if GCSFuseVersionStr == "" {
199-
GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx)
199+
GCSFuseVersionStr = specs.GetGCSFuseVersion()
200200
}
201201
v, err := version.ParseSemantic(GCSFuseVersionStr)
202202
if configPrefix == specs.SkipCSIBucketAccessCheckAndInvalidVolumePrefix && (err != nil || v.AtLeast(version.MustParseSemantic("v2.9.0-gke.0"))) {

‎test/e2e/testsuites/gcsfuse_integration.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,9 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
165165
framework.ExpectNoError(err, "while cleaning up")
166166
}
167167

168-
isKernelParamSupported := func(ctx context.Context) bool {
168+
isKernelParamSupported := func() bool {
169169
if l.gcsfuseBranch != "" || l.gcsfuseVersion == nil {
170-
l.gcsfuseVersion, l.gcsfuseBranch = specs.GCSFuseVersionAndBranch(ctx)
170+
l.gcsfuseVersion, l.gcsfuseBranch = specs.GCSFuseVersionAndBranch()
171171
}
172172
return l.gcsfuseBranch == utils.MasterBranchName || l.gcsfuseVersion.AtLeast(version.MustParseSemantic(utils.MinGCSFuseKernelParamsVersion))
173173
}
@@ -246,7 +246,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
246246
// Check if the GCSFuseVersion is already set, if not, we should set it.
247247
// This is for the case that we run this test suite individually.
248248
if GCSFuseVersionStr == "" {
249-
GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx)
249+
GCSFuseVersionStr = specs.GetGCSFuseVersion()
250250
}
251251
gcsfuseVersion := version.MustParseSemantic(GCSFuseVersionStr)
252252

@@ -576,7 +576,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
576576
init()
577577
defer cleanup()
578578
mountOptions := []string{"implicit-dirs=true"}
579-
if isKernelParamSupported(ctx) {
579+
if isKernelParamSupported() {
580580
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
581581
}
582582
gcsfuseIntegrationTest(testNameReadLargeFiles, false, mountOptions...)
@@ -643,7 +643,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
643643
defer cleanup()
644644

645645
mountOptions := []string{"implicit-dirs=true", "kernel-list-cache-ttl-secs=-1"}
646-
if isKernelParamSupported(ctx) {
646+
if isKernelParamSupported() {
647647
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
648648
}
649649
gcsfuseIntegrationTest(testNameConcurrentOperations, false, mountOptions...)
@@ -772,7 +772,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
772772
init()
773773
defer cleanup()
774774
mountOptions := []string{"read-inactive-stream-timeout=0s", "logging:format:json", fmt.Sprintf("logging:file-path:%s", inactive_stream_timeout_log_file), "log-severity=trace"}
775-
if isKernelParamSupported(ctx) {
775+
if isKernelParamSupported() {
776776
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
777777
}
778778
gcsfuseIntegrationTest(testNameInactiveStreamTimeout+":TestTimeoutDisabledSuite", false, mountOptions...)
@@ -782,7 +782,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
782782
init()
783783
defer cleanup()
784784
mountOptions := []string{"read-inactive-stream-timeout=1s", "logging:format:json", fmt.Sprintf("logging:file-path:%s", inactive_stream_timeout_log_file), "log-severity=trace"}
785-
if isKernelParamSupported(ctx) {
785+
if isKernelParamSupported() {
786786
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
787787
}
788788
gcsfuseIntegrationTest(testNameInactiveStreamTimeout+":TestTimeoutEnabledSuite/TestReaderCloses", false, mountOptions...)
@@ -792,7 +792,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
792792
init()
793793
defer cleanup()
794794
mountOptions := []string{"read-inactive-stream-timeout=1s", "logging:format:json", fmt.Sprintf("logging:file-path:%s", inactive_stream_timeout_log_file), "log-severity=trace"}
795-
if isKernelParamSupported(ctx) {
795+
if isKernelParamSupported() {
796796
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
797797
}
798798
gcsfuseIntegrationTest(testNameInactiveStreamTimeout+":TestTimeoutEnabledSuite/TestReaderStaysOpenWithinTimeout", false, mountOptions...)
@@ -802,7 +802,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
802802
init()
803803
defer cleanup()
804804
mountOptions := []string{"enable-buffered-read", "read-block-size-mb=8", "read-max-blocks-per-handle=20", "read-start-blocks-per-handle=1", "read-min-blocks-per-handle=2", "logging:format:json", fmt.Sprintf("logging:file-path:%s", buffered_reads_log_file), "log-severity=trace"}
805-
if isKernelParamSupported(ctx) {
805+
if isKernelParamSupported() {
806806
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
807807
}
808808
gcsfuseIntegrationTest(testNameBufferedReads+":TestSequentialReadSuite", false, mountOptions...)
@@ -812,7 +812,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
812812
init()
813813
defer cleanup()
814814
mountOptions := []string{"enable-buffered-read", "read-block-size-mb=8", "read-max-blocks-per-handle=20", "read-start-blocks-per-handle=2", "read-min-blocks-per-handle=2", "logging:format:json", fmt.Sprintf("logging:file-path:%s", buffered_reads_log_file), "log-severity=trace"}
815-
if isKernelParamSupported(ctx) {
815+
if isKernelParamSupported() {
816816
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
817817
}
818818
gcsfuseIntegrationTest(testNameBufferedReads+":TestFallbackSuites/TestRandomRead_LargeFile_Fallback", false, mountOptions...)
@@ -822,7 +822,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
822822
init()
823823
defer cleanup()
824824
mountOptions := []string{"enable-buffered-read", "read-block-size-mb=8", "read-max-blocks-per-handle=20", "read-start-blocks-per-handle=2", "read-min-blocks-per-handle=2", "logging:format:json", fmt.Sprintf("logging:file-path:%s", buffered_reads_log_file), "log-severity=trace"}
825-
if isKernelParamSupported(ctx) {
825+
if isKernelParamSupported() {
826826
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
827827
}
828828
gcsfuseIntegrationTest(testNameBufferedReads+":TestFallbackSuites/TestRandomRead_SmallFile_NoFallback", false, mountOptions...)
@@ -832,7 +832,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew
832832
init()
833833
defer cleanup()
834834
mountOptions := []string{"enable-buffered-read", "read-block-size-mb=8", "read-max-blocks-per-handle=10", "read-start-blocks-per-handle=2", "read-min-blocks-per-handle=2", "read-global-max-blocks=1", "logging:format:json", fmt.Sprintf("logging:file-path:%s", buffered_reads_log_file), "log-severity=trace"}
835-
if isKernelParamSupported(ctx) {
835+
if isKernelParamSupported() {
836836
mountOptions = append(mountOptions, "file-system:enable-kernel-reader:false")
837837
}
838838
gcsfuseIntegrationTest(testNameBufferedReads+":TestFallbackSuites/TestNewBufferedReader_InsufficientGlobalPool_NoReaderAdded", false, mountOptions...)

‎test/e2e/testsuites/gcsfuse_integration_file_cache.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationFileCacheTestSuite) DefineTests(driver stor
113113
gcsfuseIntegrationFileCacheTest := func(testName string, readOnly bool, fileCacheCapacity, fileCacheForRangeRead, metadataCacheTTLSeconds string, mountOptions ...string) {
114114
ginkgo.By("Checking GCSFuse version and skip test if needed")
115115
if GCSFuseVersionStr == "" {
116-
GCSFuseVersionStr = specs.GetGCSFuseVersion(ctx)
116+
GCSFuseVersionStr = specs.GetGCSFuseVersion()
117117
}
118118
ginkgo.By(fmt.Sprintf("Running integration test %v with GCSFuse version %v", testName, GCSFuseVersionStr))
119119
gcsfuseTestBranch := skipTestOrProceedWithBranch(GCSFuseVersionStr, testName)

‎test/e2e/testsuites/kernel_params.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,8 @@ func kernelParamsConfigFilePath(volumeName string) string {
127127
// }
128128
const kernelParamExtractRegex = `"name"\s*:\s*"%s".*?"value"\s*:\s*"(.*?)"`
129129

130-
func skipIfKernelParamsNotSupported(ctx context.Context) {
131-
gcsfuseVersion, branch := specs.GCSFuseVersionAndBranch(ctx)
130+
func skipIfKernelParamsNotSupported() {
131+
gcsfuseVersion, branch := specs.GCSFuseVersionAndBranch()
132132

133133
// Running since we are on master branch.
134134
if branch == utils.MasterBranchName {
@@ -201,7 +201,7 @@ func (t *gcsFuseCSIKernelParamsTestSuite) DefineTests(driver storageframework.Te
201201
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
202202

203203
init := func(configPrefix ...string) {
204-
skipIfKernelParamsNotSupported(ctx)
204+
skipIfKernelParamsNotSupported()
205205

206206
l = local{}
207207
l.config = driver.PrepareTest(ctx, f)

‎test/e2e/testsuites/mount.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,12 +321,12 @@ func (t *gcsFuseCSIMountTestSuite) DefineTests(driver storageframework.TestDrive
321321
})
322322

323323
ginkgo.It("should not pass kernel params file from csi driver to gcsfuse when the feature is disabled", func() {
324-
skipIfKernelParamsNotSupported(ctx)
324+
skipIfKernelParamsNotSupported()
325325
testGcsfuseKernelParamsFileFlagFilters()
326326
})
327327

328328
ginkgo.It("should pass kernel params file from csi driver to gcsfuse when feature is enabled", func() {
329-
skipIfKernelParamsNotSupported(ctx)
329+
skipIfKernelParamsNotSupported()
330330
testGcsfuseKernelParamsFileFlag()
331331
})
332332
}

0 commit comments

Comments
 (0)