Skip to content

Commit cfe083c

Browse files
anurag4DSBanurag4DSB
committed
S3CSI-29: Add prefix mountoption test
1 parent 8381601 commit cfe083c

File tree

3 files changed

+437
-0
lines changed

3 files changed

+437
-0
lines changed

‎tests/e2e/customsuites/mountoptions.go

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
1818
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
1919
admissionapi "k8s.io/pod-security-admission/api"
20+
21+
"github.com/scality/mountpoint-s3-csi-driver/tests/e2e/pkg/s3client"
2022
)
2123

2224
// s3CSIMountOptionsTestSuite implements the Kubernetes storage framework TestSuite interface.
@@ -437,4 +439,314 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri
437439
ginkgo.By("Verifying read from the volume")
438440
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q '%s'", fileInVol, testContent))
439441
})
442+
443+
// This test verifies that when a volume is mounted with the --prefix option,
444+
// but no files are created, the prefix doesn't exist in the bucket.
445+
//
446+
// Test scenario:
447+
//
448+
// [Pod]
449+
// |
450+
// ↓
451+
// [S3 Volume with --prefix=test-prefix/]
452+
// |
453+
// ↓
454+
// [No prefix created]
455+
//
456+
// The test specifically checks:
457+
// 1. The prefix doesn't exist in the bucket before mounting
458+
// 2. The volume with prefix option can be successfully mounted and accessed
459+
// 3. The prefix still doesn't exist in the bucket after mounting
460+
//
461+
// This validates that the S3 CSI driver doesn't implicitly create the prefix
462+
// in the S3 bucket just by mounting with the prefix option.
463+
ginkgo.It("should not create prefix in bucket when no files are created", func(ctx context.Context) {
464+
// We need to access the S3 client directly to verify objects
465+
s3Client := s3client.New("", "", "") // Default credentials/region
466+
var err error
467+
468+
// Create volume with standard non-root options plus prefix option
469+
prefix := "empty-prefix/"
470+
resource := BuildVolumeWithOptions(
471+
ctx,
472+
l.config,
473+
pattern,
474+
DefaultNonRootUser,
475+
DefaultNonRootGroup,
476+
"", // No specific file mode
477+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
478+
)
479+
l.resources = append(l.resources, resource)
480+
481+
// Extract the bucket name from the volume for verification
482+
bucketName := GetBucketNameFromVolumeResource(resource)
483+
if bucketName == "" {
484+
framework.Failf("Failed to extract bucket name from volume resource")
485+
}
486+
487+
// List all objects in the bucket to verify the prefix doesn't exist BEFORE mounting
488+
ginkgo.By("Verifying prefix doesn't exist in bucket before mounting")
489+
rootListOutputBefore, err := s3Client.ListObjects(ctx, bucketName)
490+
framework.ExpectNoError(err, "Failed to list objects in bucket before mounting")
491+
492+
// Check if any objects with the prefix exist before mounting
493+
prefixExistsBefore := false
494+
for _, obj := range rootListOutputBefore.Contents {
495+
if strings.HasPrefix(*obj.Key, prefix) {
496+
prefixExistsBefore = true
497+
break
498+
}
499+
}
500+
501+
if prefixExistsBefore {
502+
framework.Failf("Prefix %s already exists in bucket %s before mounting", prefix, bucketName)
503+
} else {
504+
framework.Logf("Verified prefix %s does not exist in bucket %s before mounting", prefix, bucketName)
505+
}
506+
507+
// Create pod with the prefixed volume
508+
ginkgo.By("Creating pod with a volume using prefix option")
509+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
510+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
511+
framework.ExpectNoError(err)
512+
defer func() {
513+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
514+
}()
515+
516+
// Verify the mount point exists and is accessible
517+
volPath := "/mnt/volume1"
518+
ginkgo.By("Verifying volume is mounted and accessible")
519+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("ls -la %s", volPath))
520+
521+
// List all objects in the bucket to verify the prefix doesn't exist AFTER mounting
522+
ginkgo.By("Verifying prefix doesn't exist in bucket after mounting")
523+
rootListOutputAfter, err := s3Client.ListObjects(ctx, bucketName)
524+
framework.ExpectNoError(err, "Failed to list objects in bucket after mounting")
525+
526+
// Check if any objects with the prefix exist after mounting
527+
prefixExistsAfter := false
528+
for _, obj := range rootListOutputAfter.Contents {
529+
if strings.HasPrefix(*obj.Key, prefix) {
530+
prefixExistsAfter = true
531+
break
532+
}
533+
}
534+
535+
if prefixExistsAfter {
536+
framework.Failf("Prefix %s was created in bucket %s just by mounting", prefix, bucketName)
537+
} else {
538+
framework.Logf("Verified prefix %s was not created in bucket %s just by mounting", prefix, bucketName)
539+
}
540+
})
541+
542+
// This test verifies that the --prefix mount option correctly isolates
543+
// objects within a specific prefix in the S3 bucket.
544+
//
545+
// Test scenario:
546+
//
547+
// [Pod]
548+
// |
549+
// ↓
550+
// [S3 Volume with --prefix=test-prefix/]
551+
// |
552+
// ↓
553+
// [Files stored under test-prefix/ in S3]
554+
//
555+
// Expected results:
556+
// - Files created in the mounted volume are stored under the specified prefix in S3
557+
// - The files can be accessed through the mounted path without the prefix in the path
558+
// - No objects are created at the root of the bucket (outside the prefix)
559+
//
560+
// This validates that the S3 CSI driver correctly handles the --prefix mount option
561+
// to properly isolate multiple users or applications within a single bucket.
562+
ginkgo.It("should store files under specified prefix when using --prefix option", func(ctx context.Context) {
563+
// We need to access the S3 client directly to verify objects
564+
s3Client := s3client.New("", "", "") // Default credentials/region
565+
566+
// Create volume with standard non-root options plus prefix option
567+
prefix := "test-prefix/"
568+
resource := BuildVolumeWithOptions(
569+
ctx,
570+
l.config,
571+
pattern,
572+
DefaultNonRootUser,
573+
DefaultNonRootGroup,
574+
"", // No specific file mode
575+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
576+
)
577+
l.resources = append(l.resources, resource)
578+
579+
// Extract the bucket name from the volume for verification
580+
bucketName := GetBucketNameFromVolumeResource(resource)
581+
if bucketName == "" {
582+
framework.Failf("Failed to extract bucket name from volume resource")
583+
}
584+
585+
// Create pod with the prefixed volume
586+
ginkgo.By("Creating pod with a volume using prefix option")
587+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
588+
var err error
589+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
590+
framework.ExpectNoError(err)
591+
defer func() {
592+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
593+
}()
594+
595+
volPath := "/mnt/volume1"
596+
testFileName := "prefix-test.txt"
597+
fileInVol := fmt.Sprintf("%s/%s", volPath, testFileName)
598+
testContent := "Testing prefix mount option"
599+
600+
// Write a file to the volume
601+
ginkgo.By("Writing a file to the volume")
602+
WriteAndVerifyFile(f, pod, fileInVol, testContent)
603+
604+
// Verify file can be read from the pod
605+
ginkgo.By("Verifying file can be read from pod")
606+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q '%s'", fileInVol, testContent))
607+
608+
// List objects in the bucket to verify the object was created under the prefix
609+
ginkgo.By(fmt.Sprintf("Verifying object exists under prefix %s in bucket", prefix))
610+
611+
prefixListOutput, err := s3Client.ListObjectsWithPrefix(ctx, bucketName, prefix)
612+
framework.ExpectNoError(err, "Failed to list objects with prefix")
613+
614+
// Verify object exists under the prefix
615+
foundObjectUnderPrefix := false
616+
expectedKey := prefix + testFileName
617+
for _, obj := range prefixListOutput.Contents {
618+
if *obj.Key == expectedKey {
619+
foundObjectUnderPrefix = true
620+
break
621+
}
622+
}
623+
if !foundObjectUnderPrefix {
624+
framework.Failf("Object %s not found under prefix %s in bucket %s", testFileName, prefix, bucketName)
625+
} else {
626+
framework.Logf("Successfully found object %s under prefix %s in bucket %s", testFileName, prefix, bucketName)
627+
}
628+
629+
// List objects in the bucket without the prefix to verify no objects exist at the root
630+
ginkgo.By("Verifying no objects exist at the root of the bucket")
631+
rootListOutput, err := s3Client.ListObjects(ctx, bucketName)
632+
framework.ExpectNoError(err, "Failed to list objects in bucket")
633+
634+
// Verify no objects exist at the root (that don't have the prefix)
635+
for _, obj := range rootListOutput.Contents {
636+
if !strings.HasPrefix(*obj.Key, prefix) {
637+
framework.Failf("Found unexpected object %s at root of bucket", *obj.Key)
638+
}
639+
}
640+
framework.Logf("No unexpected objects found at root of bucket - all objects have the prefix %s", prefix)
641+
})
642+
643+
// This test verifies that when objects are created directly in S3 under a prefix,
644+
// they are visible when mounting with that prefix, and new objects created through
645+
// the mount are also visible when listing the prefix directly from S3.
646+
//
647+
// Test scenario:
648+
//
649+
// [Direct S3 API] → [Objects under prefix] ← [Mounted Volume]
650+
//
651+
// The test specifically:
652+
// 1. Creates objects directly in S3 under a specific prefix
653+
// 2. Mounts a volume with that same prefix
654+
// 3. Verifies the pre-created objects are visible through the mount
655+
// 4. Creates new objects through the mount
656+
// 5. Verifies the new objects are visible when listing the prefix via S3 API
657+
//
658+
// This validates that the prefix mount option works bidirectionally with objects
659+
// created both through the S3 API and through the mounted volume.
660+
ginkgo.It("should see objects created directly in S3 under prefix and make new objects visible to S3", func(ctx context.Context) {
661+
// We need to access the S3 client directly to create and list objects
662+
s3Client := s3client.New(s3client.DefaultRegion, "", "") // Using DefaultRegion from s3client
663+
var err error
664+
665+
// Create volume with standard non-root options plus prefix option
666+
prefix := "test-both-directions/"
667+
resource := BuildVolumeWithOptions(
668+
ctx,
669+
l.config,
670+
pattern,
671+
DefaultNonRootUser,
672+
DefaultNonRootGroup,
673+
"", // No specific file mode
674+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
675+
)
676+
l.resources = append(l.resources, resource)
677+
678+
// Extract the bucket name from the volume for direct S3 operations
679+
bucketName := GetBucketNameFromVolumeResource(resource)
680+
if bucketName == "" {
681+
framework.Failf("Failed to extract bucket name from volume resource")
682+
}
683+
684+
directFileKeys := []string{
685+
"direct-file1.txt",
686+
"direct-file2.txt",
687+
"subdir/direct-file3.txt",
688+
}
689+
690+
// Create objects directly in S3 under the prefix
691+
ginkgo.By(fmt.Sprintf("Creating objects directly in S3 under prefix %s", prefix))
692+
err = s3Client.CreateObjectsInS3(ctx, bucketName, prefix, directFileKeys)
693+
framework.ExpectNoError(err, "Failed to create objects directly in S3")
694+
695+
// Verify objects exist in S3
696+
ginkgo.By(fmt.Sprintf("Verifying objects exist in S3 under prefix %s", prefix))
697+
err = s3Client.VerifyObjectsExistInS3(ctx, bucketName, prefix, directFileKeys)
698+
framework.ExpectNoError(err, "Failed to verify objects exist in S3")
699+
700+
// Create pod with the prefixed volume
701+
ginkgo.By("Creating pod with a volume that uses the same prefix")
702+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
703+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
704+
framework.ExpectNoError(err)
705+
defer func() {
706+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
707+
}()
708+
709+
// Verify the directly created files are visible through the mount
710+
volPath := "/mnt/volume1"
711+
ginkgo.By("Verifying directly created files are visible through the mount")
712+
713+
// Verify files exist in pod using our helper method
714+
VerifyFilesExistInPod(f, pod, volPath, directFileKeys)
715+
716+
// Additional verification for subdirectory
717+
subdirPath := fmt.Sprintf("%s/subdir", volPath)
718+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("test -d %s", subdirPath))
719+
720+
// Define mount-created file paths (these will be created through the mount)
721+
mountCreatedFiles := []string{
722+
"mount-file1.txt",
723+
"mount-file2.txt",
724+
"subdir/mount-file3.txt",
725+
}
726+
727+
// Create files in pod using our helper method
728+
CreateFilesInPod(f, pod, volPath, mountCreatedFiles)
729+
730+
// Verify new files are visible via S3 API with the prefix
731+
ginkgo.By("Verifying new files are visible via S3 API with the prefix")
732+
733+
// Verify objects exist in S3
734+
err = s3Client.VerifyObjectsExistInS3(ctx, bucketName, prefix, mountCreatedFiles)
735+
framework.ExpectNoError(err, "Failed to verify mount-created objects exist in S3")
736+
737+
// Additional verification that all objects are present (both direct and mount-created)
738+
allFiles := append([]string{}, directFileKeys...)
739+
allFiles = append(allFiles, mountCreatedFiles...)
740+
741+
prefixListAfter, err := s3Client.ListObjectsWithPrefix(ctx, bucketName, prefix)
742+
framework.ExpectNoError(err, "Failed to list objects with prefix after creating files through mount")
743+
744+
// We should have all files (direct + mount-created)
745+
if len(prefixListAfter.Contents) < len(allFiles) {
746+
framework.Failf("Expected at least %d objects after creating files through mount, but found %d",
747+
len(allFiles), len(prefixListAfter.Contents))
748+
}
749+
750+
framework.Logf("Successfully verified bidirectional visibility between S3 API and mounted volume with prefix")
751+
})
440752
}

‎tests/e2e/customsuites/util.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,3 +481,46 @@ func CreateTestFileAndDir(f *framework.Framework, pod *v1.Pod, basePath, prefix
481481

482482
return file, dir
483483
}
484+
485+
// GetBucketNameFromVolumeResource extracts the bucket name from a VolumeResource
486+
func GetBucketNameFromVolumeResource(resource *storageframework.VolumeResource) string {
487+
var bucketName string
488+
if csiSpec := resource.Pv.Spec.CSI; csiSpec != nil {
489+
if attrs := csiSpec.VolumeAttributes; attrs != nil {
490+
bucketName = attrs["bucketName"]
491+
}
492+
}
493+
return bucketName
494+
}
495+
496+
// VerifyFilesExistInPod verifies all the given file paths exist in the pod
497+
func VerifyFilesExistInPod(f *framework.Framework, pod *v1.Pod, basePath string, filePaths []string) {
498+
ginkgo.By(fmt.Sprintf("Verifying %d files exist in pod at path %s", len(filePaths), basePath))
499+
500+
for i, filePath := range filePaths {
501+
fullPath := fmt.Sprintf("%s/%s", basePath, filePath)
502+
503+
// Verify file exists
504+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("test -f %s", fullPath))
505+
506+
// Verify file content if it has a standard pattern
507+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q 'Content for file %d'", fullPath, i+1))
508+
}
509+
}
510+
511+
// CreateFilesInPod creates multiple files in the pod at the given base path
512+
func CreateFilesInPod(f *framework.Framework, pod *v1.Pod, basePath string, filePaths []string) {
513+
ginkgo.By(fmt.Sprintf("Creating %d files in pod at path %s", len(filePaths), basePath))
514+
515+
for i, filePath := range filePaths {
516+
fullPath := fmt.Sprintf("%s/%s", basePath, filePath)
517+
dirPath := filepath.Dir(fullPath)
518+
519+
// Create directory if it doesn't exist
520+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("mkdir -p %s", dirPath))
521+
522+
// Create the file with content
523+
fileContent := fmt.Sprintf("Content for file %d created through mount", i+1)
524+
WriteAndVerifyFile(f, pod, fullPath, fileContent)
525+
}
526+
}

0 commit comments

Comments
 (0)