Merge pull request #46 from glassflow/sudopower/etl-625-encrypt-postgres-and-kafka-credentials-in-postgres-taggrs

ETL-625: create / mount secret to api deployment to secure credentials

Author: sudopower

charts/glassflow-etl/Chart.yaml

@@ -15,17 +15,17 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.2
+version: 0.5.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.6.2"
+appVersion: "2.7.0"
 
 dependencies:
   - name: glassflow-operator
-    version: "0.6.9"
+    version: "0.7.0"
     repository: https://glassflow.github.io/glassflow-etl-k8s-operator
   - name: nats
     version: "1.3.6"

charts/glassflow-etl/templates/api-secret-role.yaml

@@ -0,0 +1,36 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "glassflow-etl.fullname" . }}-secret-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "glassflow-etl.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - get
+  - update
+  - patch
+  - delete
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "glassflow-etl.fullname" . }}-secret-manager-binding
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "glassflow-etl.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "glassflow-etl.fullname" . }}-secret-manager
+subjects:
+- kind: ServiceAccount
+  name: {{ include "glassflow-etl.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

charts/glassflow-etl/templates/deployment.yaml

@@ -435,6 +435,14 @@ spec:
               value: {{ .value | quote }}
             {{- end }}
             {{- end }}
+          volumeMounts:
+            - name: logs
+              mountPath: /tmp/logs/glassflow
+            {{- if and .Values.global.encryption.enabled (or .Values.global.encryption.existingSecret.name .Values.global.encryption.createSecret) }}
+            - name: encryption-key
+              mountPath: /etc/glassflow/secrets
+              readOnly: true
+            {{- end }}
           {{- with .Values.api.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}
@@ -444,11 +452,17 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           securityContext: {{- toYaml .Values.securityContext | nindent 12 }}
-          volumeMounts:
-            - name: logs
-              mountPath: /tmp/logs/glassflow
       serviceAccountName: {{ include "glassflow-etl.serviceAccountName" . }}
       securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }}
       volumes:
         - name: logs
           emptyDir: {}
+        {{- if and .Values.global.encryption.enabled (or .Values.global.encryption.existingSecret.name .Values.global.encryption.createSecret) }}
+        - name: encryption-key
+          secret:
+            secretName: {{ if .Values.global.encryption.existingSecret.name }}{{ .Values.global.encryption.existingSecret.name }}{{ else }}{{ .Values.global.encryption.secretName | default (printf "%s-encryption-key" .Release.Name) }}{{ end }}
+            defaultMode: 0444
+            items:
+              - key: {{ .Values.global.encryption.existingSecret.key | default "encryption-key" }}
+                path: encryption-key
+        {{- end }}

charts/glassflow-etl/templates/encryption-secret.yaml

@@ -0,0 +1,13 @@
+{{- if and .Values.global.encryption.enabled (not .Values.global.encryption.existingSecret.name) .Values.global.encryption.createSecret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.global.encryption.secretName | default (printf "%s-encryption-key" .Release.Name) }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "glassflow-etl.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+type: Opaque
+stringData:
+  encryption-key: {{ randAlphaNum 32 | quote }}
+{{- end }}

charts/glassflow-etl/values.yaml

@@ -46,6 +46,18 @@ global:
     secret:
       name: ""
       key: "connection_url"
+
+  # Encryption configuration for connection credentials
+  encryption:
+    # Enable credential encryption (encrypts Kafka and ClickHouse connection details)
+    enabled: true
+    # Use existing secret - Your own secret (recommended for production)
+    existingSecret:
+      name: ""  # e.g., "glassflow-encryption-key"
+      key: "encryption-key"  # Key name in secret
+    # Or let Helm generate a secret (for development/testing)
+    createSecret: true
+    secretName: ""  # Defaults to "{{ .Release.Name }}-encryption-key" if empty
 # =============================================================================
 # API COMPONENT CONFIGURATION
 # =============================================================================
@@ -55,7 +67,7 @@ api:
   logLevel: "info"
   image:
     repository: glassflow-etl-be
-    tag: v2.6.2
+    tag: v2.7.0
     pullPolicy: IfNotPresent
   resources:
     requests:
@@ -89,7 +101,7 @@ ui:
   replicas: 1
   image:
     repository: glassflow-etl-fe
-    tag: v2.6.2
+    tag: v2.7.0
     pullPolicy: IfNotPresent
   logLevel: "info"
   resources:
@@ -165,7 +177,7 @@ glassflow-operator:
     manager:
       image:
         repository: glassflow-etl-k8s-operator
-        tag: v1.4.3
+        tag: v1.5.0
         pullPolicy: IfNotPresent
       resources:
         limits:
@@ -181,7 +193,7 @@ glassflow-operator:
     ingestor:
       image:
         repository: glassflow-etl-ingestor
-        tag: v2.6.2
+        tag: v2.7.0
         pullPolicy: IfNotPresent
       logLevel: "INFO"
       resources:
@@ -196,7 +208,7 @@ glassflow-operator:
     join:
       image:
         repository: glassflow-etl-join
-        tag: v2.6.2
+        tag: v2.7.0
         pullPolicy: IfNotPresent
       logLevel: "INFO"
       resources:
@@ -211,7 +223,7 @@ glassflow-operator:
     sink:
       image:
         repository: glassflow-etl-sink
-        tag: v2.6.2
+        tag: v2.7.0
         pullPolicy: IfNotPresent
       logLevel: "INFO"
       resources:
@@ -226,7 +238,7 @@ glassflow-operator:
     dedup:
       image:
         repository: glassflow-etl-dedup
-        tag: v2.6.2
+        tag: v2.7.0
         pullPolicy: IfNotPresent
       logLevel: "INFO"
       resources: