From 32915d019760d3e6de9572d6f86729d0a0097b5b Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Thu, 8 May 2025 15:22:41 +0530 Subject: [PATCH 1/8] Added versioning configuration for COS buckets in CSI driver --- deploy/ibmCloud/cos-s3fs-standard-sc.yaml | 13 ++++---- deploy/ibmCloud/kustomization.yaml | 4 +-- examples/kubernetes/cos-csi-app-s3fs.yaml | 2 +- .../kubernetes/cos-s3-csi-s3fs-secret.yaml | 9 ++--- pkg/driver/controllerserver.go | 25 ++++++++++++++ pkg/s3client/fake_s3client.go | 8 +++++ pkg/s3client/s3client.go | 19 +++++++++++ pkg/s3client/s3client_test.go | 33 +++++++++++++++---- tests/sanity/sanity_test.go | 4 +++ 9 files changed, 97 insertions(+), 20 deletions(-) diff --git a/deploy/ibmCloud/cos-s3fs-standard-sc.yaml b/deploy/ibmCloud/cos-s3fs-standard-sc.yaml index d6aa6cfc..20141a02 100644 --- a/deploy/ibmCloud/cos-s3fs-standard-sc.yaml +++ b/deploy/ibmCloud/cos-s3fs-standard-sc.yaml @@ -7,17 +7,18 @@ metadata: ibm.cos.storageclass/tier: standard provisioner: cos.s3.csi.ibm.io mountOptions: - - "multipart_size=62" - - "max_dirty_data=51200" - - "parallel_count=8" - - "max_stat_cache_size=100000" - - "retries=5" - - "kernel_cache" + - "multipart_size=62" + - "max_dirty_data=51200" + - "parallel_count=8" + - "max_stat_cache_size=100000" + - "retries=5" + - "kernel_cache" parameters: mounter: "s3fs" client: "awss3" cosEndpoint: "https://s3.direct.us-west.cloud-object-storage.appdomain.cloud" locationConstraint: "us-west-standard" + versioningEnabled: "true" csi.storage.k8s.io/node-publish-secret-name: ${pvc.annotations['cos.csi.driver/secret']} csi.storage.k8s.io/node-publish-secret-namespace: ${pvc.namespace} reclaimPolicy: Delete diff --git a/deploy/ibmCloud/kustomization.yaml b/deploy/ibmCloud/kustomization.yaml index 0109da16..0681b935 100644 --- a/deploy/ibmCloud/kustomization.yaml +++ b/deploy/ibmCloud/kustomization.yaml @@ -19,8 +19,8 @@ images: newName: k8s.gcr.io/sig-storage/csi-provisioner newTag: v5.1.0 - name: cos-driver-image - newName: icr.io/ibm/ibm-object-csi-driver - newTag: v0.1.16 + newName: docker.io/prachi102/ibm-object-csi-driver + newTag: v0.1.20 - name: driver-registrar-image newName: k8s.gcr.io/sig-storage/csi-node-driver-registrar newTag: v2.12.0 diff --git a/examples/kubernetes/cos-csi-app-s3fs.yaml b/examples/kubernetes/cos-csi-app-s3fs.yaml index 8bd72a82..9d3068b3 100644 --- a/examples/kubernetes/cos-csi-app-s3fs.yaml +++ b/examples/kubernetes/cos-csi-app-s3fs.yaml @@ -13,4 +13,4 @@ spec: volumes: - name: cos-csi-volume persistentVolumeClaim: - claimName: cos-s3fs-pvc + claimName: cos-s3fs-pvc1 diff --git a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml index e86d601e..013e144b 100644 --- a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml +++ b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml @@ -5,17 +5,18 @@ metadata: name: cos-s3fs-secret namespace: default data: - # Bucket name: echo -n "nkcode-devtest01" | base64 - bucketName: YmhhdGVzdHRvcG9sb2d5 + # Bucket name: echo -n "psacode-versiontest03" | base64 + bucketName: cHNhY29kZS12ZXJzaW9udGVzdDAz # apiKey: # base64 encoded IAM API Key # serviceId: # base64 encoded IAM Service Instance ID # HMAC Access Key and Secret Key # echo -n "" | base64 # echo -n "" | base64 - accessKey: bXktYWNjZXNzLWtleQ== - secretKey: bXktc2VjcmV0LWtleQ== + accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= + secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm # echo -n | base64 # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN stringData: # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. mountOptions: | + diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index 4a825288..a0102399 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -48,6 +48,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum kpRootKeyCrn string pvcName string pvcNamespace string + versioningEnabled bool ) secretMapCustom := make(map[string]string) @@ -164,6 +165,16 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum bucketName = secretMapCustom["bucketName"] } + // Check for versioningEnabled parameter + if val, ok := params["versioningEnabled"]; ok && strings.ToLower(val) == "true" { + versioningEnabled = true + klog.Infof("Versioning enabled for volume: %s", volumeID) + } + if val, ok := secretMap["versioningEnabled"]; ok && strings.ToLower(val) == "true" { + versioningEnabled = true + klog.Infof("Versioning enabled via secret for volume: %s", volumeID) + } + creds, err := getCredentials(secretMap) if err != nil { return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Error in getting credentials %v", err)) @@ -184,6 +195,13 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum params["userProvidedBucket"] = "false" klog.Infof("Created bucket: %s", bucketName) } + // Enable versioning for existing or newly created user-provided bucket + if versioningEnabled { + if err := sess.EnableBucketVersioning(bucketName); err != nil { + klog.Errorf("Failed to enable versioning for bucket: %s, error: %v", bucketName, err) + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", bucketName, err)) + } + } params["bucketName"] = bucketName } else { // Generate random temp bucket name based on volume id @@ -197,6 +215,13 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum if err != nil { return nil, status.Error(codes.PermissionDenied, fmt.Sprintf("%v: %v", err, tempBucketName)) } + // Enable versioning for new temp bucket + if versioningEnabled { + if err := sess.EnableBucketVersioning(tempBucketName); err != nil { + klog.Errorf("Failed to enable versioning for temp bucket: %s, error: %v", tempBucketName, err) + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", tempBucketName, err)) + } + } klog.Infof("Created temp bucket: %s", tempBucketName) params["userProvidedBucket"] = "false" params["bucketName"] = tempBucketName diff --git a/pkg/s3client/fake_s3client.go b/pkg/s3client/fake_s3client.go index 54661d90..9383f8d3 100644 --- a/pkg/s3client/fake_s3client.go +++ b/pkg/s3client/fake_s3client.go @@ -11,6 +11,7 @@ type FakeCOSSessionFactory struct { FailCheckBucketAccess bool FailCreateBucket bool FailDeleteBucket bool + FailEnableVersioning bool } type fakeCOSSession struct { @@ -31,6 +32,13 @@ func (s *fakeCOSSession) CheckBucketAccess(bucket string) error { return nil } +func (s *fakeCOSSession) EnableBucketVersioning(bucketName string) error { + if s.factory.FailEnableVersioning { + return errors.New("failed to enable bucket versioning") + } + return nil +} + func (s *fakeCOSSession) CheckObjectPathExistence(bucket, objectpath string) (bool, error) { return true, nil } diff --git a/pkg/s3client/s3client.go b/pkg/s3client/s3client.go index 1236810f..41c8a0a4 100644 --- a/pkg/s3client/s3client.go +++ b/pkg/s3client/s3client.go @@ -61,6 +61,8 @@ type ObjectStorageSession interface { // DeleteBucket methods deletes a bucket (with all of its objects) DeleteBucket(bucket string) error + + EnableBucketVersioning(bucket string) error } // COSSessionFactory represents a COS (S3) session factory @@ -91,6 +93,7 @@ type s3API interface { ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) DeleteObject(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) DeleteBucket(input *s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error) + PutBucketVersioning(input *s3.PutBucketVersioningInput) (*s3.PutBucketVersioningOutput, error) } func (s *COSSession) CheckBucketAccess(bucket string) error { @@ -182,6 +185,22 @@ func (s *COSSession) DeleteBucket(bucket string) error { return err } +func (s *COSSession) EnableBucketVersioning(bucket string) error { + s.logger.Info("Enabling versioning for bucket", zap.String("bucket", bucket)) + _, err := s.svc.PutBucketVersioning(&s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String("Enabled"), + }, + }) + if err != nil { + s.logger.Error("Failed to enable versioning", zap.String("bucket", bucket), zap.Error(err)) + return fmt.Errorf("failed to enable versioning for bucket '%s': %v", bucket, err) + } + s.logger.Info("Versioning enabled successfully for bucket", zap.String("bucket", bucket)) + return nil +} + func NewS3Client(lgr *zap.Logger) (ObjectStorageSession, error) { cosSession := new(COSSession) cosSession.logger = lgr diff --git a/pkg/s3client/s3client_test.go b/pkg/s3client/s3client_test.go index 450e05fb..a8af01e1 100644 --- a/pkg/s3client/s3client_test.go +++ b/pkg/s3client/s3client_test.go @@ -12,13 +12,14 @@ import ( ) type fakeS3API struct { - ErrHeadBucket error - ErrCreateBucket error - ErrListObjects error - ErrListObjectsV2 error - ErrDeleteObject error - ErrDeleteBucket error - ObjectPath string + ErrHeadBucket error + ErrCreateBucket error + ErrListObjects error + ErrListObjectsV2 error + ErrDeleteObject error + ErrDeleteBucket error + ObjectPath string + ErrPutBucketVersioning error } const ( @@ -49,6 +50,10 @@ func (a *fakeS3API) CreateBucket(input *s3.CreateBucketInput) (*s3.CreateBucketO return nil, a.ErrCreateBucket } +func (a *fakeS3API) PutBucketVersioning(input *s3.PutBucketVersioningInput) (*s3.PutBucketVersioningOutput, error) { + return &s3.PutBucketVersioningOutput{}, a.ErrPutBucketVersioning +} + func (a *fakeS3API) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { return &s3.ListObjectsOutput{ Contents: []*s3.Object{{Key: &testObject}}, @@ -156,6 +161,20 @@ func Test_CreateBucket_Positive(t *testing.T) { assert.NoError(t, err) } +func Test_EnableBucketVersioning_Positive(t *testing.T) { + sess := getSession(&fakeS3API{}) + err := sess.EnableBucketVersioning(testBucket) + assert.NoError(t, err) +} + +func Test_EnableBucketVersioning_Error(t *testing.T) { + sess := getSession(&fakeS3API{ErrPutBucketVersioning: errFoo}) + err := sess.EnableBucketVersioning(testBucket) + if assert.Error(t, err) { + assert.EqualError(t, err, "failed to enable versioning for bucket 'test-bucket': foo") + } +} + func Test_DeleteBucket_BucketAlreadyDeleted_Positive(t *testing.T) { sess := getSession(&fakeS3API{ErrListObjects: awserr.New("NoSuchBucket", "", errFoo)}) err := sess.DeleteBucket(testBucket) diff --git a/tests/sanity/sanity_test.go b/tests/sanity/sanity_test.go index e7e42fa2..1655c982 100644 --- a/tests/sanity/sanity_test.go +++ b/tests/sanity/sanity_test.go @@ -151,6 +151,10 @@ func (s *fakeObjectStorageSession) CheckBucketAccess(bucket string) error { return nil } +func (s *fakeObjectStorageSession) EnableBucketVersioning(bucketName string) error { + return nil +} + func (s *fakeObjectStorageSession) CheckObjectPathExistence(bucket, objectpath string) (bool, error) { return true, nil } From 7af7fdaa15646628b71436ee5bba0f9d2fd36c5b Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Thu, 8 May 2025 15:29:13 +0530 Subject: [PATCH 2/8] Fix small issues From 0b6da410d5d6a07bee5dafbf0f538ae99e4101c6 Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Thu, 8 May 2025 15:29:47 +0530 Subject: [PATCH 3/8] Added versioning configuration for COS buckets in CSI driver --- examples/kubernetes/cos-csi-app-s3fs.yaml | 2 +- examples/kubernetes/cos-s3-csi-s3fs-secret.yaml | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/kubernetes/cos-csi-app-s3fs.yaml b/examples/kubernetes/cos-csi-app-s3fs.yaml index 9d3068b3..8bd72a82 100644 --- a/examples/kubernetes/cos-csi-app-s3fs.yaml +++ b/examples/kubernetes/cos-csi-app-s3fs.yaml @@ -13,4 +13,4 @@ spec: volumes: - name: cos-csi-volume persistentVolumeClaim: - claimName: cos-s3fs-pvc1 + claimName: cos-s3fs-pvc diff --git a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml index 013e144b..e86d601e 100644 --- a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml +++ b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml @@ -5,18 +5,17 @@ metadata: name: cos-s3fs-secret namespace: default data: - # Bucket name: echo -n "psacode-versiontest03" | base64 - bucketName: cHNhY29kZS12ZXJzaW9udGVzdDAz + # Bucket name: echo -n "nkcode-devtest01" | base64 + bucketName: YmhhdGVzdHRvcG9sb2d5 # apiKey: # base64 encoded IAM API Key # serviceId: # base64 encoded IAM Service Instance ID # HMAC Access Key and Secret Key # echo -n "" | base64 # echo -n "" | base64 - accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= - secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm + accessKey: bXktYWNjZXNzLWtleQ== + secretKey: bXktc2VjcmV0LWtleQ== # echo -n | base64 # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN stringData: # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. mountOptions: | - From 164beb7a6eec862039d2aceca178d84f8bbe5232 Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Fri, 9 May 2025 17:29:28 +0530 Subject: [PATCH 4/8] Added versioning configuration for COS buckets in CSI driver --- .../kubernetes/cos-s3-csi-s3fs-secret.yaml | 39 +++++++++++++------ pkg/driver/controllerserver.go | 14 +++++-- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml index e86d601e..ff5b7a16 100644 --- a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml +++ b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml @@ -1,21 +1,36 @@ +# apiVersion: v1 +# kind: Secret +# type: cos-s3-csi-driver +# metadata: +# name: cos-s3fs-secret +# namespace: default +# data: +# # Bucket name: echo -n "nkcode-devtest01" | base64 +# bucketName: cHNhLXZlcnNpb25kcml2ZS10ZXN0 +# # apiKey: # base64 encoded IAM API Key +# # serviceId: # base64 encoded IAM Service Instance ID +# # HMAC Access Key and Secret Key +# # echo -n "" | base64 +# # echo -n "" | base64 +# accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= +# secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm +# # echo -n | base64 +# # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN +# stringData: +# # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. +# mountOptions: | + apiVersion: v1 kind: Secret type: cos-s3-csi-driver metadata: - name: cos-s3fs-secret + name: cos-s3fs-conf-secret namespace: default data: - # Bucket name: echo -n "nkcode-devtest01" | base64 - bucketName: YmhhdGVzdHRvcG9sb2d5 - # apiKey: # base64 encoded IAM API Key - # serviceId: # base64 encoded IAM Service Instance ID - # HMAC Access Key and Secret Key - # echo -n "" | base64 - # echo -n "" | base64 - accessKey: bXktYWNjZXNzLWtleQ== - secretKey: bXktc2VjcmV0LWtleQ== - # echo -n | base64 - # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN + bucketName: cHNhLWNvbmZsaWN0YnVja2V0 + accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= + secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm + versioningEnabled: ZmFsc2U= stringData: # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. mountOptions: | diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index a0102399..cf909ef8 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -166,13 +166,13 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum } // Check for versioningEnabled parameter - if val, ok := params["versioningEnabled"]; ok && strings.ToLower(val) == "true" { - versioningEnabled = true - klog.Infof("Versioning enabled for volume: %s", volumeID) - } + versioningEnabled = false // Default value if val, ok := secretMap["versioningEnabled"]; ok && strings.ToLower(val) == "true" { versioningEnabled = true klog.Infof("Versioning enabled via secret for volume: %s", volumeID) + } else if val, ok := params["versioningEnabled"]; ok && strings.ToLower(val) == "true" { + versioningEnabled = true + klog.Infof("Versioning enabled via storage class parameters for volume: %s", volumeID) } creds, err := getCredentials(secretMap) @@ -466,6 +466,7 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { iamEndpoint string cosEndpoint string locationConstraint string + versioningEnabled string ) if bytesVal, ok := secret.Data["accessKey"]; ok { @@ -504,6 +505,10 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { locationConstraint = string(bytesVal) } + if bytesVal, ok := secret.Data["versioningEnabled"]; ok { + versioningEnabled = string(bytesVal) + } + secretMapCustom["accessKey"] = accessKey secretMapCustom["secretKey"] = secretKey secretMapCustom["apiKey"] = apiKey @@ -513,6 +518,7 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { secretMapCustom["iamEndpoint"] = iamEndpoint secretMapCustom["cosEndpoint"] = cosEndpoint secretMapCustom["locationConstraint"] = locationConstraint + secretMapCustom["versioningEnabled"] = versioningEnabled return secretMapCustom } From 74059a3abb71a9021da1ce1c9ac7caa468da1caf Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Mon, 12 May 2025 15:23:26 +0530 Subject: [PATCH 5/8] Added versioning configuration for COS buckets in CSI driver --- .../kubernetes/cos-s3-csi-s3fs-secret.yaml | 37 ++++++------------- pkg/driver/controllerserver.go | 28 +++++++------- pkg/s3client/fake_s3client.go | 4 +- pkg/s3client/s3client.go | 18 +++++---- pkg/s3client/s3client_test.go | 16 +++++--- 5 files changed, 50 insertions(+), 53 deletions(-) diff --git a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml index ff5b7a16..d84d7fa7 100644 --- a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml +++ b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml @@ -1,36 +1,23 @@ -# apiVersion: v1 -# kind: Secret -# type: cos-s3-csi-driver -# metadata: -# name: cos-s3fs-secret -# namespace: default -# data: -# # Bucket name: echo -n "nkcode-devtest01" | base64 -# bucketName: cHNhLXZlcnNpb25kcml2ZS10ZXN0 -# # apiKey: # base64 encoded IAM API Key -# # serviceId: # base64 encoded IAM Service Instance ID -# # HMAC Access Key and Secret Key -# # echo -n "" | base64 -# # echo -n "" | base64 -# accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= -# secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm -# # echo -n | base64 -# # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN -# stringData: -# # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. -# mountOptions: | - apiVersion: v1 kind: Secret type: cos-s3-csi-driver metadata: - name: cos-s3fs-conf-secret + name: cos-s3fs-secret namespace: default data: - bucketName: cHNhLWNvbmZsaWN0YnVja2V0 + # Bucket name: echo -n "nkcode-devtest01" | base64 + bucketName: cHNhLXZlcnNpb25kcml2ZS10ZXN0 + # apiKey: # base64 encoded IAM API Key + # serviceId: # base64 encoded IAM Service Instance ID + # HMAC Access Key and Secret Key + # echo -n "" | base64 + # echo -n "" | base64 accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm - versioningEnabled: ZmFsc2U= + # echo -n | base64 + # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN stringData: # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. mountOptions: | + + diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index cf909ef8..35f6f0a7 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -48,7 +48,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum kpRootKeyCrn string pvcName string pvcNamespace string - versioningEnabled bool + BucketVersioning bool ) secretMapCustom := make(map[string]string) @@ -165,13 +165,13 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum bucketName = secretMapCustom["bucketName"] } - // Check for versioningEnabled parameter - versioningEnabled = false // Default value - if val, ok := secretMap["versioningEnabled"]; ok && strings.ToLower(val) == "true" { - versioningEnabled = true + // Check for BucketVersioning parameter + BucketVersioning = false // Default value + if val, ok := secretMap["BucketVersioning"]; ok && strings.ToLower(val) == "true" { + BucketVersioning = true klog.Infof("Versioning enabled via secret for volume: %s", volumeID) - } else if val, ok := params["versioningEnabled"]; ok && strings.ToLower(val) == "true" { - versioningEnabled = true + } else if val, ok := params["BucketVersioning"]; ok && strings.ToLower(val) == "true" { + BucketVersioning = true klog.Infof("Versioning enabled via storage class parameters for volume: %s", volumeID) } @@ -196,8 +196,8 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum klog.Infof("Created bucket: %s", bucketName) } // Enable versioning for existing or newly created user-provided bucket - if versioningEnabled { - if err := sess.EnableBucketVersioning(bucketName); err != nil { + if BucketVersioning { + if err := sess.SetBucketVersioning(bucketName); err != nil { klog.Errorf("Failed to enable versioning for bucket: %s, error: %v", bucketName, err) return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", bucketName, err)) } @@ -216,7 +216,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum return nil, status.Error(codes.PermissionDenied, fmt.Sprintf("%v: %v", err, tempBucketName)) } // Enable versioning for new temp bucket - if versioningEnabled { + if BucketVersioning { if err := sess.EnableBucketVersioning(tempBucketName); err != nil { klog.Errorf("Failed to enable versioning for temp bucket: %s, error: %v", tempBucketName, err) return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", tempBucketName, err)) @@ -466,7 +466,7 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { iamEndpoint string cosEndpoint string locationConstraint string - versioningEnabled string + BucketVersioning string ) if bytesVal, ok := secret.Data["accessKey"]; ok { @@ -505,8 +505,8 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { locationConstraint = string(bytesVal) } - if bytesVal, ok := secret.Data["versioningEnabled"]; ok { - versioningEnabled = string(bytesVal) + if bytesVal, ok := secret.Data["BucketVersioning"]; ok { + BucketVersioning = string(bytesVal) } secretMapCustom["accessKey"] = accessKey @@ -518,7 +518,7 @@ func parseCustomSecret(secret *v1.Secret) map[string]string { secretMapCustom["iamEndpoint"] = iamEndpoint secretMapCustom["cosEndpoint"] = cosEndpoint secretMapCustom["locationConstraint"] = locationConstraint - secretMapCustom["versioningEnabled"] = versioningEnabled + secretMapCustom["BucketVersioning"] = BucketVersioning return secretMapCustom } diff --git a/pkg/s3client/fake_s3client.go b/pkg/s3client/fake_s3client.go index 9383f8d3..d410bdf4 100644 --- a/pkg/s3client/fake_s3client.go +++ b/pkg/s3client/fake_s3client.go @@ -32,9 +32,9 @@ func (s *fakeCOSSession) CheckBucketAccess(bucket string) error { return nil } -func (s *fakeCOSSession) EnableBucketVersioning(bucketName string) error { +func (s *fakeCOSSession) SetBucketVersioning(bucket string, enable bool) error { if s.factory.FailEnableVersioning { - return errors.New("failed to enable bucket versioning") + return errors.New("failed to set bucket versioning") } return nil } diff --git a/pkg/s3client/s3client.go b/pkg/s3client/s3client.go index 41c8a0a4..cdbb4834 100644 --- a/pkg/s3client/s3client.go +++ b/pkg/s3client/s3client.go @@ -62,7 +62,7 @@ type ObjectStorageSession interface { // DeleteBucket methods deletes a bucket (with all of its objects) DeleteBucket(bucket string) error - EnableBucketVersioning(bucket string) error + SetBucketVersioning(bucket string, enable bool) error } // COSSessionFactory represents a COS (S3) session factory @@ -185,19 +185,23 @@ func (s *COSSession) DeleteBucket(bucket string) error { return err } -func (s *COSSession) EnableBucketVersioning(bucket string) error { - s.logger.Info("Enabling versioning for bucket", zap.String("bucket", bucket)) +func (s *COSSession) SetBucketVersioning(bucket string, enable bool) error { + status := "Suspended" + if enable { + status = "Enabled" + } + s.logger.Info("Setting versioning for bucket", zap.String("bucket", bucket), zap.Bool("enable", enable)) _, err := s.svc.PutBucketVersioning(&s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), VersioningConfiguration: &s3.VersioningConfiguration{ - Status: aws.String("Enabled"), + Status: aws.String(status), }, }) if err != nil { - s.logger.Error("Failed to enable versioning", zap.String("bucket", bucket), zap.Error(err)) - return fmt.Errorf("failed to enable versioning for bucket '%s': %v", bucket, err) + s.logger.Error("Failed to set versioning", zap.String("bucket", bucket), zap.Bool("enable", enable), zap.Error(err)) + return fmt.Errorf("failed to set versioning to %v for bucket '%s': %v", enable, bucket, err) } - s.logger.Info("Versioning enabled successfully for bucket", zap.String("bucket", bucket)) + s.logger.Info("Versioning set successfully for bucket", zap.String("bucket", bucket), zap.Bool("enable", enable)) return nil } diff --git a/pkg/s3client/s3client_test.go b/pkg/s3client/s3client_test.go index a8af01e1..af96b03f 100644 --- a/pkg/s3client/s3client_test.go +++ b/pkg/s3client/s3client_test.go @@ -161,17 +161,23 @@ func Test_CreateBucket_Positive(t *testing.T) { assert.NoError(t, err) } -func Test_EnableBucketVersioning_Positive(t *testing.T) { +func Test_SetBucketVersioning_True_Positive(t *testing.T) { sess := getSession(&fakeS3API{}) - err := sess.EnableBucketVersioning(testBucket) + err := sess.SetBucketVersioning(testBucket, true) assert.NoError(t, err) } -func Test_EnableBucketVersioning_Error(t *testing.T) { +func Test_SetBucketVersioning_False_Positive(t *testing.T) { + sess := getSession(&fakeS3API{}) + err := sess.SetBucketVersioning(testBucket, false) + assert.NoError(t, err) +} + +func Test_SetBucketVersioning_Error(t *testing.T) { sess := getSession(&fakeS3API{ErrPutBucketVersioning: errFoo}) - err := sess.EnableBucketVersioning(testBucket) + err := sess.SetBucketVersioning(testBucket, true) if assert.Error(t, err) { - assert.EqualError(t, err, "failed to enable versioning for bucket 'test-bucket': foo") + assert.EqualError(t, err, "failed to set versioning to true for bucket 'test-bucket': foo") } } From 2a1ee4379a397559839a331c24bdbd888efcfb91 Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Mon, 12 May 2025 17:10:37 +0530 Subject: [PATCH 6/8] Added versioning configuration for COS buckets in CSI driver --- deploy/ibmCloud/kustomization.yaml | 4 +- pkg/driver/controllerserver.go | 59 ++++++++++++++++++++++-------- tests/sanity/sanity_test.go | 3 +- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/deploy/ibmCloud/kustomization.yaml b/deploy/ibmCloud/kustomization.yaml index 0681b935..0109da16 100644 --- a/deploy/ibmCloud/kustomization.yaml +++ b/deploy/ibmCloud/kustomization.yaml @@ -19,8 +19,8 @@ images: newName: k8s.gcr.io/sig-storage/csi-provisioner newTag: v5.1.0 - name: cos-driver-image - newName: docker.io/prachi102/ibm-object-csi-driver - newTag: v0.1.20 + newName: icr.io/ibm/ibm-object-csi-driver + newTag: v0.1.16 - name: driver-registrar-image newName: k8s.gcr.io/sig-storage/csi-node-driver-registrar newTag: v2.12.0 diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index 35f6f0a7..23adf669 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -48,7 +48,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum kpRootKeyCrn string pvcName string pvcNamespace string - BucketVersioning bool + BucketVersioning string ) secretMapCustom := make(map[string]string) @@ -166,13 +166,29 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum } // Check for BucketVersioning parameter - BucketVersioning = false // Default value - if val, ok := secretMap["BucketVersioning"]; ok && strings.ToLower(val) == "true" { - BucketVersioning = true - klog.Infof("Versioning enabled via secret for volume: %s", volumeID) - } else if val, ok := params["BucketVersioning"]; ok && strings.ToLower(val) == "true" { - BucketVersioning = true - klog.Infof("Versioning enabled via storage class parameters for volume: %s", volumeID) + // BucketVersioning = false // Default value + // if val, ok := secretMap["BucketVersioning"]; ok && strings.ToLower(val) == "true" { + // BucketVersioning = true + // klog.Infof("Versioning enabled via secret for volume: %s", volumeID) + // } else if val, ok := params["BucketVersioning"]; ok && strings.ToLower(val) == "true" { + // BucketVersioning = true + // klog.Infof("Versioning enabled via storage class parameters for volume: %s", volumeID) + // } + if val, ok := secretMap["BucketVersioning"]; ok && val != "" { + enable := strings.ToLower(strings.TrimSpace(val)) + if enable != "true" && enable != "false" { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid BucketVersioning value in secret: %s. Must be 'true' or 'false'", val)) + } + BucketVersioning = enable + klog.Infof("BucketVersioning set via secret: %s", BucketVersioning) + } else if val, ok := params["BucketVersioning"]; ok && val != "" { + enable := strings.ToLower(strings.TrimSpace(val)) + if enable != "true" && enable != "false" { + return nil, status.Error(codes.InvalidArgument, + fmt.Sprintf("Invalid BucketVersioning value in storage class: %s. Must be 'true' or 'false'", val)) + } + BucketVersioning = enable + klog.Infof("BucketVersioning set via storage class: %s", BucketVersioning) } creds, err := getCredentials(secretMap) @@ -196,12 +212,21 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum klog.Infof("Created bucket: %s", bucketName) } // Enable versioning for existing or newly created user-provided bucket - if BucketVersioning { - if err := sess.SetBucketVersioning(bucketName); err != nil { - klog.Errorf("Failed to enable versioning for bucket: %s, error: %v", bucketName, err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", bucketName, err)) + // if BucketVersioning { + // if err := sess.SetBucketVersioning(bucketName); err != nil { + // klog.Errorf("Failed to enable versioning for bucket: %s, error: %v", bucketName, err) + // return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", bucketName, err)) + // } + // } + if BucketVersioning != "" { + enable := BucketVersioning == "true" + if err := sess.SetBucketVersioning(bucketName, enable); err != nil { + klog.Errorf("Failed to set versioning for bucket: %s, error: %v", bucketName, err) + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to set versioning for bucket %s: %v", bucketName, err)) } + klog.Infof("Bucket versioning set to %t for bucket %s", enable, bucketName) } + params["bucketName"] = bucketName } else { // Generate random temp bucket name based on volume id @@ -216,11 +241,13 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum return nil, status.Error(codes.PermissionDenied, fmt.Sprintf("%v: %v", err, tempBucketName)) } // Enable versioning for new temp bucket - if BucketVersioning { - if err := sess.EnableBucketVersioning(tempBucketName); err != nil { - klog.Errorf("Failed to enable versioning for temp bucket: %s, error: %v", tempBucketName, err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", tempBucketName, err)) + if BucketVersioning != "" { + enable := BucketVersioning == "true" + if err := sess.SetBucketVersioning(tempBucketName, enable); err != nil { + klog.Errorf("Failed to set versioning for temp bucket: %s, error: %v", tempBucketName, err) + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to set versioning for bucket %s: %v", tempBucketName, err)) } + klog.Infof("Bucket versioning set to %t for temp bucket %s", enable, tempBucketName) } klog.Infof("Created temp bucket: %s", tempBucketName) params["userProvidedBucket"] = "false" diff --git a/tests/sanity/sanity_test.go b/tests/sanity/sanity_test.go index 1655c982..884ffd53 100644 --- a/tests/sanity/sanity_test.go +++ b/tests/sanity/sanity_test.go @@ -151,7 +151,8 @@ func (s *fakeObjectStorageSession) CheckBucketAccess(bucket string) error { return nil } -func (s *fakeObjectStorageSession) EnableBucketVersioning(bucketName string) error { +func (s *fakeObjectStorageSession) SetBucketVersioning(bucketName string, enable bool) error { + s.logger.Info(fmt.Sprintf("Fake SetBucketVersioning called for bucket %s with enable=%t", bucketName, enable)) return nil } From 842b4de61cc5397e60312be7c0e63faaa4919bc7 Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Mon, 12 May 2025 17:14:49 +0530 Subject: [PATCH 7/8] Added versioning configuration for COS buckets in CSI driver --- deploy/ibmCloud/cos-s3fs-standard-sc.yaml | 13 ++++++------- examples/kubernetes/cos-s3-csi-s3fs-secret.yaml | 8 +++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/deploy/ibmCloud/cos-s3fs-standard-sc.yaml b/deploy/ibmCloud/cos-s3fs-standard-sc.yaml index 20141a02..d6aa6cfc 100644 --- a/deploy/ibmCloud/cos-s3fs-standard-sc.yaml +++ b/deploy/ibmCloud/cos-s3fs-standard-sc.yaml @@ -7,18 +7,17 @@ metadata: ibm.cos.storageclass/tier: standard provisioner: cos.s3.csi.ibm.io mountOptions: - - "multipart_size=62" - - "max_dirty_data=51200" - - "parallel_count=8" - - "max_stat_cache_size=100000" - - "retries=5" - - "kernel_cache" + - "multipart_size=62" + - "max_dirty_data=51200" + - "parallel_count=8" + - "max_stat_cache_size=100000" + - "retries=5" + - "kernel_cache" parameters: mounter: "s3fs" client: "awss3" cosEndpoint: "https://s3.direct.us-west.cloud-object-storage.appdomain.cloud" locationConstraint: "us-west-standard" - versioningEnabled: "true" csi.storage.k8s.io/node-publish-secret-name: ${pvc.annotations['cos.csi.driver/secret']} csi.storage.k8s.io/node-publish-secret-namespace: ${pvc.namespace} reclaimPolicy: Delete diff --git a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml index d84d7fa7..e86d601e 100644 --- a/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml +++ b/examples/kubernetes/cos-s3-csi-s3fs-secret.yaml @@ -6,18 +6,16 @@ metadata: namespace: default data: # Bucket name: echo -n "nkcode-devtest01" | base64 - bucketName: cHNhLXZlcnNpb25kcml2ZS10ZXN0 + bucketName: YmhhdGVzdHRvcG9sb2d5 # apiKey: # base64 encoded IAM API Key # serviceId: # base64 encoded IAM Service Instance ID # HMAC Access Key and Secret Key # echo -n "" | base64 # echo -n "" | base64 - accessKey: YjM2NTNjNmVmMzY2NDE0ZGFiOTgzNTU4MTgxYTY1MzQ= - secretKey: ZDJiOTMzYWMzMTg4Mzc2YWRlMzE5OGZlMjZhZTIxYTI1ODJmNWMwYzg5MGJlNGVm + accessKey: bXktYWNjZXNzLWtleQ== + secretKey: bXktc2VjcmV0LWtleQ== # echo -n | base64 # kpRootKeyCRN: # base64 encoded Key Protect Root key CRN stringData: # uid: "3000" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. mountOptions: | - - From 2f00c67d105192a18fe3b57404cb0df29f7ca703 Mon Sep 17 00:00:00 2001 From: Prachi03510 Date: Mon, 12 May 2025 17:17:11 +0530 Subject: [PATCH 8/8] Added versioning configuration for COS buckets in CSI driver --- pkg/driver/controllerserver.go | 16 +--------------- pkg/s3client/fake_s3client.go | 4 ++-- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index 23adf669..aec784e0 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -166,14 +166,6 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum } // Check for BucketVersioning parameter - // BucketVersioning = false // Default value - // if val, ok := secretMap["BucketVersioning"]; ok && strings.ToLower(val) == "true" { - // BucketVersioning = true - // klog.Infof("Versioning enabled via secret for volume: %s", volumeID) - // } else if val, ok := params["BucketVersioning"]; ok && strings.ToLower(val) == "true" { - // BucketVersioning = true - // klog.Infof("Versioning enabled via storage class parameters for volume: %s", volumeID) - // } if val, ok := secretMap["BucketVersioning"]; ok && val != "" { enable := strings.ToLower(strings.TrimSpace(val)) if enable != "true" && enable != "false" { @@ -211,13 +203,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum params["userProvidedBucket"] = "false" klog.Infof("Created bucket: %s", bucketName) } - // Enable versioning for existing or newly created user-provided bucket - // if BucketVersioning { - // if err := sess.SetBucketVersioning(bucketName); err != nil { - // klog.Errorf("Failed to enable versioning for bucket: %s, error: %v", bucketName, err) - // return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to enable versioning for bucket %s: %v", bucketName, err)) - // } - // } + if BucketVersioning != "" { enable := BucketVersioning == "true" if err := sess.SetBucketVersioning(bucketName, enable); err != nil { diff --git a/pkg/s3client/fake_s3client.go b/pkg/s3client/fake_s3client.go index d410bdf4..7636690a 100644 --- a/pkg/s3client/fake_s3client.go +++ b/pkg/s3client/fake_s3client.go @@ -11,7 +11,7 @@ type FakeCOSSessionFactory struct { FailCheckBucketAccess bool FailCreateBucket bool FailDeleteBucket bool - FailEnableVersioning bool + FailBucketVersioning bool } type fakeCOSSession struct { @@ -33,7 +33,7 @@ func (s *fakeCOSSession) CheckBucketAccess(bucket string) error { } func (s *fakeCOSSession) SetBucketVersioning(bucket string, enable bool) error { - if s.factory.FailEnableVersioning { + if s.factory.FailBucketVersioning { return errors.New("failed to set bucket versioning") } return nil