move secrets to k8 secrets, add logs from tier-2

MJarmo · MJarmo · commit 719ab9778a41 · 2026-04-07T12:01:01.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,11 @@
 # environment file with secret password
 **/.env
 
+# local kubectl secrets (copy from showroom/kubectl/secrets.example.yaml)
+showroom/kubectl/secrets.local.yaml
+showroom/kubectl/secrets.env
+showroom/kubectl/htpasswd.secret
+
 # potentially generated file
 /src/file-sink/received-logs.txt
 
diff --git a/showroom/kubectl/secrets.example.yaml b/showroom/kubectl/secrets.example.yaml
@@ -0,0 +1,55 @@
+# Preferred: use secrets.env (see secrets.env.example) and run:
+#   task secrets:apply
+# or:  ./kubectl/apply-secrets.sh   (Linux/macOS/Git Bash)
+# or:  ./kubectl/apply-secrets.ps1  (Windows PowerShell)
+#
+# Secrets are applied with kubectl create secret ... | kubectl apply (nothing committed).
+#
+# Cross-check with tier-1.yaml / tier-2.yaml:
+# - tier-1 init: secret dice-java-openbao-token key token
+# - tier-1 otel sidecar: secret otel-otlp-client-auth keys username, password -> OTEL_OTLP_EXPORTER_*
+# - tier-2 otel: secret otel-otlp-ingest-htpasswd key htpasswd -> file /etc/otelcol-secrets/htpasswd
+#   The htpasswd line MUST authenticate the same user/password as otel-otlp-client-auth (tier-1 exporter).
+# - tier-2 otel OpenSearch: secret opensearch-credentials keys username, password -> OPENSEARCH_*
+#
+# Legacy (optional): copy this file to secrets.local.yaml (gitignored), replace REPLACE_ME, then:
+#   kubectl apply -f secrets.local.yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dice-java-openbao-token
+  namespace: tier-1
+type: Opaque
+stringData:
+  token: REPLACE_ME
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: otel-otlp-client-auth
+  namespace: tier-1
+type: Opaque
+stringData:
+  username: REPLACE_ME
+  password: REPLACE_ME
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: otel-otlp-ingest-htpasswd
+  namespace: tier-2
+type: Opaque
+stringData:
+  htpasswd: |
+    REPLACE_ME_USERNAME:REPLACE_ME_PASSWORD_OR_BCRYPT_HASH
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-credentials
+  namespace: tier-2
+type: Opaque
+stringData:
+  username: REPLACE_ME
+  password: REPLACE_ME
diff --git a/showroom/kubectl/tier-1.yaml b/showroom/kubectl/tier-1.yaml
@@ -2,6 +2,10 @@
 # - Deployment with 3 replicas
 # - Each pod runs the application container and an otelcol sidecar
 # - ConfigMap contains the otelcol config
+#
+# Apply namespace secrets before this manifest: task secrets:apply (kubectl/secrets.env), or see secrets.example.yaml.
+#   dice-java-openbao-token (key: token)
+#   otel-otlp-client-auth (keys: username, password) — must match tier-2 otel-otlp-ingest-htpasswd line
 
 apiVersion: v1
 kind: ConfigMap
@@ -12,10 +16,8 @@ data:
     extensions:
       basicauth/client:
         client_auth:
-          # TEST-ONLY: plaintext credentials in a ConfigMap are NOT production-ready.
-          # For production use a Kubernetes Secret and/or external secret store, and rotate credentials.
-          username: "##########"
-          password: "##########"
+          username: ${env:OTEL_OTLP_EXPORTER_USERNAME}
+          password: ${env:OTEL_OTLP_EXPORTER_PASSWORD}
 
     receivers:
       otlp:
@@ -26,6 +28,11 @@ data:
             endpoint: 0.0.0.0:4318
 
     processors:
+      transform/fixlogtime:
+        log_statements:
+          - context: log
+            statements:
+              - set(time_unix_nano, observed_time_unix_nano) where time_unix_nano == 0
       certificatehash:
         hash_algorithm: SHA256
         sign_content: body
@@ -52,7 +59,7 @@ data:
       pipelines:
         logs:
           receivers: [otlp]
-          processors: [certificatehash, batch]
+          processors: [transform/fixlogtime, certificatehash, batch]
           exporters: [otlphttp, debug]
         metrics:
           receivers: [otlp]
@@ -154,7 +161,10 @@ spec:
             - name: OPENBAO_CERT_PATH
               value: "kv/data/secret"
             - name: OPENBAO_TOKEN
-              value: "##########"
+              valueFrom:
+                secretKeyRef:
+                  name: dice-java-openbao-token
+                  key: token
           volumeMounts:
             - name: dice-java-certs
               mountPath: /etc/dice-java/certs
@@ -203,6 +213,17 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - "--config=/etc/otelcol-contrib/config.yaml"
+          env:
+            - name: OTEL_OTLP_EXPORTER_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: otel-otlp-client-auth
+                  key: username
+            - name: OTEL_OTLP_EXPORTER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: otel-otlp-client-auth
+                  key: password
           ports:
             - name: otlp-grpc
               containerPort: 4317
diff --git a/showroom/kubectl/tier-2.yaml b/showroom/kubectl/tier-2.yaml
@@ -1,4 +1,7 @@
 # ServiceAccount for the OTel Collector Agent
+# Apply namespace secrets before this manifest: task secrets:apply (kubectl/secrets.env), or see secrets.example.yaml.
+#   otel-otlp-ingest-htpasswd (key: htpasswd) -> /etc/otelcol-secrets/htpasswd
+#   opensearch-credentials (keys: username, password) -> OPENSEARCH_USERNAME / OPENSEARCH_PASSWORD
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -19,11 +22,7 @@ data:
         endpoint: ${env:MY_POD_IP}:13133
       basicauth/server:
         htpasswd:
-          inline: |
-            # TEST-ONLY: plaintext htpasswd entries are NOT production-ready.
-            # For production generate a bcrypt htpasswd line (e.g. `htpasswd -nbBC 10 USER PASS`)
-            # and store it in a Kubernetes Secret (or external secret store).
-            ##########:##########
+          file: /etc/otelcol-secrets/htpasswd
       basicauth/client:
         client_auth:
           username: ${env:OPENSEARCH_USERNAME}
@@ -40,8 +39,20 @@ data:
             endpoint: 0.0.0.0:4318
             auth:
               authenticator: basicauth/server
+      filelog/tier2:
+        include:
+          - /var/log/pods/tier-2_*/*/*.log
+        start_at: end
+        include_file_path: true
+        operators:
+          - type: container
 
     processors:
+      transform/fixlogtime:
+        log_statements:
+          - context: log
+            statements:
+              - set(time_unix_nano, observed_time_unix_nano) where time_unix_nano == 0
       certificatelogverify:
         hash_algorithm: SHA256
         sign_content: body
@@ -69,6 +80,18 @@ data:
         sending_queue:
           enabled: true
           storage: file_storage
+      opensearch/tier2self:
+        http:
+          endpoint: https://opensearch-cluster-master.tier-3.svc.cluster.local:9200
+          tls:
+            insecure_skip_verify: true
+          auth:
+            authenticator: basicauth/client
+        logs_index: otel-logs-tier2
+        traces_index: otel-traces
+        sending_queue:
+          enabled: true
+          storage: file_storage
       debug/logs:
         verbosity: detailed
       debug/metrics:
@@ -79,8 +102,12 @@ data:
       pipelines:
         logs:
           receivers: [otlp]
-          processors: [certificatelogverify, batch]
+          processors: [transform/fixlogtime, certificatelogverify, batch]
           exporters: [opensearch, debug/logs]
+        logs/tier2self:
+          receivers: [filelog/tier2]
+          processors: [transform/fixlogtime, batch]
+          exporters: [opensearch/tier2self]
         metrics:
           receivers: [otlp]
           processors: [batch]
@@ -182,13 +209,19 @@ spec:
               cpu: 200m
             requests:
               memory: 128Mi
-              cpu: 100m
+              cpu: 50m
           volumeMounts:
             - name: config
               mountPath: /etc/otelcol-contrib
               readOnly: true
             - name: storage
               mountPath: /var/lib/otelcol/storage
+            - name: pod-logs
+              mountPath: /var/log/pods
+              readOnly: true
+            - name: otlp-ingest-htpasswd
+              mountPath: /etc/otelcol-secrets
+              readOnly: true
           livenessProbe:
             httpGet:
               path: /
@@ -219,6 +252,14 @@ spec:
           hostPath:
             path: /var/lib/otelcol/storage
             type: DirectoryOrCreate
+        - name: otlp-ingest-htpasswd
+          secret:
+            secretName: otel-otlp-ingest-htpasswd
+            defaultMode: 288
+        - name: pod-logs
+          hostPath:
+            path: /var/log/pods
+            type: Directory
 ---
 # Service for the OTel Collector Agent
 apiVersion: v1
diff --git a/showroom/taskfile.yaml b/showroom/taskfile.yaml
@@ -34,6 +34,10 @@ includes:
     desc: Install and manage the sample application (dice-java).
     taskfile: ./tasks/tier-1.yaml
 
+  secrets:
+    desc: Apply workload secrets from kubectl/secrets.env.
+    taskfile: ./tasks/secrets.yaml
+
 tasks:
   default:
     desc: Create and start local kubernetes cluster and install otel-demo.
@@ -52,6 +56,7 @@ tasks:
     cmds:
       - task: garden:set-kubeconfig
       - task: cluster:install
+      - task: secrets:apply
       - task: monitoring:install
       - task: otelcol-agent:install
       - task: dice:install
