Skip to content

Commit fe6bfda

Browse files
anurag4DSBanurag4DSB
committed
S3CSI-29: Add prefix mountoption test
1 parent 8381601 commit fe6bfda

File tree

3 files changed

+439
-0
lines changed

3 files changed

+439
-0
lines changed

‎tests/e2e/customsuites/mountoptions.go

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
1818
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
1919
admissionapi "k8s.io/pod-security-admission/api"
20+
21+
"github.com/scality/mountpoint-s3-csi-driver/tests/e2e/pkg/s3client"
2022
)
2123

2224
// s3CSIMountOptionsTestSuite implements the Kubernetes storage framework TestSuite interface.
@@ -437,4 +439,316 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri
437439
ginkgo.By("Verifying read from the volume")
438440
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q '%s'", fileInVol, testContent))
439441
})
442+
443+
// This test verifies that when a volume is mounted with the --prefix option,
444+
// but no files are created, the prefix doesn't exist in the bucket.
445+
//
446+
// Test scenario:
447+
//
448+
// [Pod]
449+
// |
450+
// ↓
451+
// [S3 Volume with --prefix=test-prefix/]
452+
// |
453+
// ↓
454+
// [No prefix created]
455+
//
456+
// The test specifically checks:
457+
// 1. The prefix doesn't exist in the bucket before mounting
458+
// 2. The volume with prefix option can be successfully mounted and accessed
459+
// 3. The prefix still doesn't exist in the bucket after mounting
460+
//
461+
// This validates that the S3 CSI driver doesn't implicitly create the prefix
462+
// in the S3 bucket just by mounting with the prefix option.
463+
ginkgo.It("should not create prefix in bucket when no files are created", func(ctx context.Context) {
464+
// We need to access the S3 client directly to verify objects
465+
s3Client := s3client.New("", "", "") // Default credentials/region
466+
var err error
467+
468+
// Create volume with standard non-root options plus prefix option
469+
prefix := "empty-prefix/"
470+
resource := BuildVolumeWithOptions(
471+
ctx,
472+
l.config,
473+
pattern,
474+
DefaultNonRootUser,
475+
DefaultNonRootGroup,
476+
"", // No specific file mode
477+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
478+
)
479+
l.resources = append(l.resources, resource)
480+
481+
// Extract the bucket name from the volume for verification
482+
bucketName := GetBucketNameFromVolumeResource(resource)
483+
if bucketName == "" {
484+
framework.Failf("Failed to extract bucket name from volume resource")
485+
}
486+
487+
// List all objects in the bucket to verify the prefix doesn't exist BEFORE mounting
488+
ginkgo.By("Verifying prefix doesn't exist in bucket before mounting")
489+
rootListOutputBefore, err := s3Client.ListObjects(ctx, bucketName)
490+
framework.ExpectNoError(err, "Failed to list objects in bucket before mounting")
491+
492+
// Check if any objects with the prefix exist before mounting
493+
prefixExistsBefore := false
494+
for _, obj := range rootListOutputBefore.Contents {
495+
if strings.HasPrefix(*obj.Key, prefix) {
496+
prefixExistsBefore = true
497+
break
498+
}
499+
}
500+
501+
if prefixExistsBefore {
502+
framework.Failf("Prefix %s already exists in bucket %s before mounting", prefix, bucketName)
503+
} else {
504+
framework.Logf("Verified prefix %s does not exist in bucket %s before mounting", prefix, bucketName)
505+
}
506+
507+
// Create pod with the prefixed volume
508+
ginkgo.By("Creating pod with a volume using prefix option")
509+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
510+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
511+
framework.ExpectNoError(err)
512+
defer func() {
513+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
514+
}()
515+
516+
// Verify the mount point exists and is accessible
517+
volPath := "/mnt/volume1"
518+
ginkgo.By("Verifying volume is mounted and accessible")
519+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("ls -la %s", volPath))
520+
521+
// List all objects in the bucket to verify the prefix doesn't exist AFTER mounting
522+
ginkgo.By("Verifying prefix doesn't exist in bucket after mounting")
523+
rootListOutputAfter, err := s3Client.ListObjects(ctx, bucketName)
524+
framework.ExpectNoError(err, "Failed to list objects in bucket after mounting")
525+
526+
// Check if any objects with the prefix exist after mounting
527+
prefixExistsAfter := false
528+
for _, obj := range rootListOutputAfter.Contents {
529+
if strings.HasPrefix(*obj.Key, prefix) {
530+
prefixExistsAfter = true
531+
break
532+
}
533+
}
534+
535+
if prefixExistsAfter {
536+
framework.Failf("Prefix %s was created in bucket %s just by mounting", prefix, bucketName)
537+
} else {
538+
framework.Logf("Verified prefix %s was not created in bucket %s just by mounting", prefix, bucketName)
539+
}
540+
})
541+
542+
// This test verifies that the --prefix mount option correctly isolates
543+
// objects within a specific prefix in the S3 bucket.
544+
//
545+
// Test scenario:
546+
//
547+
// [Pod]
548+
// |
549+
// ↓
550+
// [S3 Volume with --prefix=test-prefix/]
551+
// |
552+
// ↓
553+
// [Files stored under test-prefix/ in S3]
554+
//
555+
// Expected results:
556+
// - Files created in the mounted volume are stored under the specified prefix in S3
557+
// - The files can be accessed through the mounted path without the prefix in the path
558+
// - No objects are created at the root of the bucket (outside the prefix)
559+
//
560+
// This validates that the S3 CSI driver correctly handles the --prefix mount option
561+
// to properly isolate multiple users or applications within a single bucket.
562+
ginkgo.It("should store files under specified prefix when using --prefix option", func(ctx context.Context) {
563+
// We need to access the S3 client directly to verify objects
564+
s3Client := s3client.New("", "", "") // Default credentials/region
565+
566+
// Create volume with standard non-root options plus prefix option
567+
prefix := "test-prefix/"
568+
resource := BuildVolumeWithOptions(
569+
ctx,
570+
l.config,
571+
pattern,
572+
DefaultNonRootUser,
573+
DefaultNonRootGroup,
574+
"", // No specific file mode
575+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
576+
)
577+
l.resources = append(l.resources, resource)
578+
579+
// Extract the bucket name from the volume for verification
580+
bucketName := GetBucketNameFromVolumeResource(resource)
581+
if bucketName == "" {
582+
framework.Failf("Failed to extract bucket name from volume resource")
583+
}
584+
585+
// Create pod with the prefixed volume
586+
ginkgo.By("Creating pod with a volume using prefix option")
587+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
588+
var err error
589+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
590+
framework.ExpectNoError(err)
591+
defer func() {
592+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
593+
}()
594+
595+
volPath := "/mnt/volume1"
596+
testFileName := "prefix-test.txt"
597+
fileInVol := fmt.Sprintf("%s/%s", volPath, testFileName)
598+
testContent := "Testing prefix mount option"
599+
600+
// Write a file to the volume
601+
ginkgo.By("Writing a file to the volume")
602+
WriteAndVerifyFile(f, pod, fileInVol, testContent)
603+
604+
// Verify file can be read from the pod
605+
ginkgo.By("Verifying file can be read from pod")
606+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q '%s'", fileInVol, testContent))
607+
608+
// List objects in the bucket to verify the object was created under the prefix
609+
ginkgo.By(fmt.Sprintf("Verifying object exists under prefix %s in bucket", prefix))
610+
611+
// Use our new ListObjectsWithPrefix method
612+
prefixListOutput, err := s3Client.ListObjectsWithPrefix(ctx, bucketName, prefix)
613+
framework.ExpectNoError(err, "Failed to list objects with prefix")
614+
615+
// Verify object exists under the prefix
616+
foundObjectUnderPrefix := false
617+
expectedKey := prefix + testFileName
618+
for _, obj := range prefixListOutput.Contents {
619+
if *obj.Key == expectedKey {
620+
foundObjectUnderPrefix = true
621+
break
622+
}
623+
}
624+
if !foundObjectUnderPrefix {
625+
framework.Failf("Object %s not found under prefix %s in bucket %s", testFileName, prefix, bucketName)
626+
} else {
627+
framework.Logf("Successfully found object %s under prefix %s in bucket %s", testFileName, prefix, bucketName)
628+
}
629+
630+
// List objects in the bucket without the prefix to verify no objects exist at the root
631+
ginkgo.By("Verifying no objects exist at the root of the bucket")
632+
rootListOutput, err := s3Client.ListObjects(ctx, bucketName)
633+
framework.ExpectNoError(err, "Failed to list objects in bucket")
634+
635+
// Verify no objects exist at the root (that don't have the prefix)
636+
for _, obj := range rootListOutput.Contents {
637+
if !strings.HasPrefix(*obj.Key, prefix) {
638+
framework.Failf("Found unexpected object %s at root of bucket", *obj.Key)
639+
}
640+
}
641+
framework.Logf("No unexpected objects found at root of bucket - all objects have the prefix %s", prefix)
642+
})
643+
644+
// This test verifies that when objects are created directly in S3 under a prefix,
645+
// they are visible when mounting with that prefix, and new objects created through
646+
// the mount are also visible when listing the prefix directly from S3.
647+
//
648+
// Test scenario:
649+
//
650+
// [Direct S3 API] → [Objects under prefix] ← [Mounted Volume]
651+
//
652+
// The test specifically:
653+
// 1. Creates objects directly in S3 under a specific prefix
654+
// 2. Mounts a volume with that same prefix
655+
// 3. Verifies the pre-created objects are visible through the mount
656+
// 4. Creates new objects through the mount
657+
// 5. Verifies the new objects are visible when listing the prefix via S3 API
658+
//
659+
// This validates that the prefix mount option works bidirectionally with objects
660+
// created both through the S3 API and through the mounted volume.
661+
ginkgo.It("should see objects created directly in S3 under prefix and make new objects visible to S3", func(ctx context.Context) {
662+
// We need to access the S3 client directly to create and list objects
663+
s3Client := s3client.New(s3client.DefaultRegion, "", "") // Using DefaultRegion from s3client
664+
var err error
665+
666+
// Create volume with standard non-root options plus prefix option
667+
prefix := "test-both-directions/"
668+
resource := BuildVolumeWithOptions(
669+
ctx,
670+
l.config,
671+
pattern,
672+
DefaultNonRootUser,
673+
DefaultNonRootGroup,
674+
"", // No specific file mode
675+
fmt.Sprintf("prefix=%s", prefix), // Add the prefix option
676+
)
677+
l.resources = append(l.resources, resource)
678+
679+
// Extract the bucket name from the volume for direct S3 operations
680+
bucketName := GetBucketNameFromVolumeResource(resource)
681+
if bucketName == "" {
682+
framework.Failf("Failed to extract bucket name from volume resource")
683+
}
684+
685+
// Define direct file paths (these will be created directly in S3)
686+
directFileKeys := []string{
687+
"direct-file1.txt",
688+
"direct-file2.txt",
689+
"subdir/direct-file3.txt",
690+
}
691+
692+
// Create objects directly in S3 under the prefix using the s3client method
693+
ginkgo.By(fmt.Sprintf("Creating objects directly in S3 under prefix %s", prefix))
694+
err = s3Client.CreateObjectsInS3(ctx, bucketName, prefix, directFileKeys)
695+
framework.ExpectNoError(err, "Failed to create objects directly in S3")
696+
697+
// Verify objects exist in S3 with the s3client method
698+
ginkgo.By(fmt.Sprintf("Verifying objects exist in S3 under prefix %s", prefix))
699+
err = s3Client.VerifyObjectsExistInS3(ctx, bucketName, prefix, directFileKeys)
700+
framework.ExpectNoError(err, "Failed to verify objects exist in S3")
701+
702+
// Create pod with the prefixed volume
703+
ginkgo.By("Creating pod with a volume that uses the same prefix")
704+
pod := MakeNonRootPodWithVolume(f.Namespace.Name, []*v1.PersistentVolumeClaim{resource.Pvc}, "")
705+
pod, err = createPod(ctx, f.ClientSet, f.Namespace.Name, pod)
706+
framework.ExpectNoError(err)
707+
defer func() {
708+
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, f.ClientSet, pod))
709+
}()
710+
711+
// Verify the directly created files are visible through the mount
712+
volPath := "/mnt/volume1"
713+
ginkgo.By("Verifying directly created files are visible through the mount")
714+
715+
// Verify files exist in pod using our helper method
716+
VerifyFilesExistInPod(f, pod, volPath, directFileKeys)
717+
718+
// Additional verification for subdirectory
719+
subdirPath := fmt.Sprintf("%s/subdir", volPath)
720+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("test -d %s", subdirPath))
721+
722+
// Define mount-created file paths (these will be created through the mount)
723+
mountCreatedFiles := []string{
724+
"mount-file1.txt",
725+
"mount-file2.txt",
726+
"subdir/mount-file3.txt",
727+
}
728+
729+
// Create files in pod using our helper method
730+
CreateFilesInPod(f, pod, volPath, mountCreatedFiles)
731+
732+
// Verify new files are visible via S3 API with the prefix
733+
ginkgo.By("Verifying new files are visible via S3 API with the prefix")
734+
735+
// Verify objects exist in S3 using the s3client method
736+
err = s3Client.VerifyObjectsExistInS3(ctx, bucketName, prefix, mountCreatedFiles)
737+
framework.ExpectNoError(err, "Failed to verify mount-created objects exist in S3")
738+
739+
// Additional verification that all objects are present (both direct and mount-created)
740+
allFiles := append([]string{}, directFileKeys...)
741+
allFiles = append(allFiles, mountCreatedFiles...)
742+
743+
prefixListAfter, err := s3Client.ListObjectsWithPrefix(ctx, bucketName, prefix)
744+
framework.ExpectNoError(err, "Failed to list objects with prefix after creating files through mount")
745+
746+
// We should have all files (direct + mount-created)
747+
if len(prefixListAfter.Contents) < len(allFiles) {
748+
framework.Failf("Expected at least %d objects after creating files through mount, but found %d",
749+
len(allFiles), len(prefixListAfter.Contents))
750+
}
751+
752+
framework.Logf("Successfully verified bidirectional visibility between S3 API and mounted volume with prefix")
753+
})
440754
}

‎tests/e2e/customsuites/util.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,3 +481,46 @@ func CreateTestFileAndDir(f *framework.Framework, pod *v1.Pod, basePath, prefix
481481

482482
return file, dir
483483
}
484+
485+
// GetBucketNameFromVolumeResource extracts the bucket name from a VolumeResource
486+
func GetBucketNameFromVolumeResource(resource *storageframework.VolumeResource) string {
487+
var bucketName string
488+
if csiSpec := resource.Pv.Spec.CSI; csiSpec != nil {
489+
if attrs := csiSpec.VolumeAttributes; attrs != nil {
490+
bucketName = attrs["bucketName"]
491+
}
492+
}
493+
return bucketName
494+
}
495+
496+
// VerifyFilesExistInPod verifies all the given file paths exist in the pod
497+
func VerifyFilesExistInPod(f *framework.Framework, pod *v1.Pod, basePath string, filePaths []string) {
498+
ginkgo.By(fmt.Sprintf("Verifying %d files exist in pod at path %s", len(filePaths), basePath))
499+
500+
for i, filePath := range filePaths {
501+
fullPath := fmt.Sprintf("%s/%s", basePath, filePath)
502+
503+
// Verify file exists
504+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("test -f %s", fullPath))
505+
506+
// Verify file content if it has a standard pattern
507+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("cat %s | grep -q 'Content for file %d'", fullPath, i+1))
508+
}
509+
}
510+
511+
// CreateFilesInPod creates multiple files in the pod at the given base path
512+
func CreateFilesInPod(f *framework.Framework, pod *v1.Pod, basePath string, filePaths []string) {
513+
ginkgo.By(fmt.Sprintf("Creating %d files in pod at path %s", len(filePaths), basePath))
514+
515+
for i, filePath := range filePaths {
516+
fullPath := fmt.Sprintf("%s/%s", basePath, filePath)
517+
dirPath := filepath.Dir(fullPath)
518+
519+
// Create directory if it doesn't exist
520+
e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("mkdir -p %s", dirPath))
521+
522+
// Create the file with content
523+
fileContent := fmt.Sprintf("Content for file %d created through mount", i+1)
524+
WriteAndVerifyFile(f, pod, fullPath, fileContent)
525+
}
526+
}

0 commit comments

Comments
 (0)