feat: added configuration support for S3_SHARDED storage type

Author: pratushnyi

charts/testops/CHANGELOG.md

@@ -10,6 +10,10 @@ Entries are ordered by priority:
 4. `[BUGFIX]` Fixes for defects
 5. `[DOCS]` Update of the documentation in `values.yaml`
 
+## 5.27.0
+
+- [FEATURE] Added configuration support for `S3_SHARDED` storage type, allowing users to configure multiple S3 storages and map them to specific projects.
+
 ## 5.26.2
 
 - [BUGFIX] Fixed scientific notation in the application's thread pool configuration.

charts/testops/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: testops
-version: 5.26.2
+version: 5.27.0
 appVersion: 26.1.1
 kubeVersion: '>= 1.20.0-0'
 

charts/testops/templates/_helpers.tpl

@@ -619,7 +619,65 @@
     value: {{ .Values.storage.s3.serverSideEncryption.keyId | quote }}
 {{- end }}
 {{- end }}
+
+{{- if eq .Values.storage.type "S3_SHARDED" }}
+{{- range $index, $storage := .Values.storage.s3.additionalStorages }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_NAME
+    value: {{ $storage.name | quote }}
+{{- if $storage.endpoint }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_ENDPOINT
+    value: {{ $storage.endpoint | quote }}
+{{- end }}
+{{- if $storage.bucket }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_BUCKET
+    value: {{ $storage.bucket | quote }}
+{{- end }}
+{{- if $storage.region }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_REGION
+    value: {{ $storage.region | quote }}
+{{- end }}
+{{- if $storage.accessKey }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_ACCESSKEY
+    value: {{ $storage.accessKey | quote }}
+{{- end }}
+{{- if $storage.secretKey }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_SECRETKEY
+    value: {{ $storage.secretKey | quote }}
+{{- end }}
+{{- if and $storage.serverSideEncryption $storage.serverSideEncryption.enabled }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_SERVERSIDEENCRYPTION
+    value: {{ $storage.serverSideEncryption.type | quote }}
+{{- if $storage.serverSideEncryption.keyId }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_STORAGES_{{ $index }}_KMSKEYID
+    value: {{ $storage.serverSideEncryption.keyId | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- range $projectId, $config := .Values.storage.s3.projects }}
+{{- if $config.storage }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_PROJECTS_{{ $projectId }}_STORAGE
+    value: {{ $config.storage | quote }}
+{{- end }}
+{{- if $config.bucket }}
+  - name: ALLURE_BLOBSTORAGE_S3SHARDED_PROJECTS_{{ $projectId }}_BUCKET
+    value: {{ $config.bucket | quote }}
 {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+
+
+
+        private ServerSideEncryption serverSideEncryption;
+        private String kmsKeyId;
+
+        private boolean pathStyleAccess;
+        private boolean chunkedEncoding;
+
+        private Duration connectionTimeout = Duration.ofMinutes(1);
+        private Duration connectionMaxIdleTime = Duration.ofMinutes(2);*/}}
 
 {{- define "renderFSEnvs" }}
   - name: ALLURE_BLOBSTORAGE_TYPE

charts/testops/values.yaml

@@ -366,7 +366,9 @@ smtp:
 # 2) S3_ASYNC - recommended for AWS S3, this option supports parameters described in .Values.storage.s3.advancedS3SDK
 # S3_ASYNC allows usage of optimised commands towards S3 solution (e.g. delete objects). Consult your solution
 # documentation before using this option
-# 3) CSI
+# 3) S3_SHARDED - allows configuring multiple S3 storages
+# and mapping them to specific projects via .Values.storage.s3.storages and .Values.storage.s3.projects
+# 4) CSI
 # For CSI refer to https://kubernetes-csi.github.io/docs/drivers.html
 # S3 is preferable. The best option is using SaaS S3 AWS or other S3 comparable services.
 # It's highly recommended disabling versioning for S3 from the very start
@@ -377,6 +379,7 @@ storage:
   # Options are:
   # - S3
   # - S3_ASYNC
+  # - S3_SHARDED
   # - CSI
   type: "S3"
   s3:
@@ -423,6 +426,31 @@ storage:
       # The ARN format is: arn:aws:kms:<region>:<account-id>:key/<key-id>
       # Ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html#aws-managed-customer-managed-keys
       keyId:
+    ## @param storage.s3.additionalStorages defines an array of additional S3 storages for S3_SHARDED mode.
+    ## Each storage requires a unique 'name' and can override default S3 parameters like endpoint, bucket, region, accessKey, secretKey, and serverSideEncryption.
+    ## e.g:
+    ## additionalStorages:
+    ##   - name: "eu-storage"
+    ##     endpoint: "https://s3.eu-central-1.amazonaws.com"
+    ##     bucket: "allure-eu-bucket"
+    ##     region: "eu-central-1"
+    ##     serverSideEncryption:
+    ##       enabled: true
+    ##       type: "AWS_KMS"
+    ##       keyId: "arn:aws:kms:eu-central-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+    ##
+    additionalStorages: []
+    ## @param storage.s3.projects defines a map of project-specific S3 configurations for S3_SHARDED mode.
+    ## The key is the project ID. You can specify a custom 'bucket' and/or a 'storage' name (referencing an entry from additional-storages).
+    ## If a property is not set, the default S3 configuration will be used.
+    ## e.g:
+    ## projects:
+    ##   12:
+    ##     storage: "eu-storage"
+    ##   45:
+    ##     bucket: "custom-us-bucket"
+    ##
+    projects: []
   csi:
     storageClass: ""
     existingVolumeName: ""