Merge pull request #64 from scalyr/raw_api_key_value

Prototype for using raw SCALYR_API_KEY environment variable value as-is (e.g. kube-secrets-init use case)

Author: tomaz-s1

.github/workflows/end_to_end_tests.yaml

@@ -226,6 +226,173 @@ jobs:
           steps: ${{ toJson(steps) }}
           channel: '#cloud-tech'
 
+  # Here we test the scenario where an raw apiKey chart value is used for SCALYR_API_KEY pod
+  # environment variable. This is to support the use cases like kube-secrets-init where
+  # the raw value contains reference to the secret which is replaced by external operator.
+  # Keep in mind that we only store raw api key itself in the secret for testing purposes,
+  # this is a bad practice and should never be used in real life.
+  daemonset_controller_type_raw_api_key_secret_value:
+    name: Daemonset - raw secret - k8s ${{ matrix.k8s_version }} - ${{ matrix.image_type }}
+    runs-on: ubuntu-latest
+
+    needs: pre_job
+    timeout-minutes: 15
+    # NOTE: We always want to run job on main branch
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s_version:
+          - 'v1.24.3'
+        image_type:
+          - "buster"
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Set up Chart Testing Environment and Kubernetes Cluster
+        uses: ./.github/actions/setup-chart-testing-environment/
+        with:
+          k8s_version: "${{ matrix.k8s_version }}"
+          github_token: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Install Scalyr Tool
+        uses: ./.github/actions/install-scalyr-tool/
+
+      - name: Install Helm Chart
+        uses: ./.github/actions/install-helm-chart
+        with:
+          scalyr_api_key: "${{ secrets.SCALYR_WRITE_API_KEY_US }}"
+          values_file_path: "ci/daemonset-agent-values-raw-secret.yaml"
+          image_type: "${{ matrix.image_type }}"
+
+      - name: Describe Pod
+        run: |
+          set -e
+
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME}
+
+          # Verify test volume defined using chart volumes and volumeMounts value is there
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME} | grep "/test-volume from test-volume (rw)"
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME} | grep "test-volume:"
+
+          # Verify that secret hasn't been created since useRawApiKeyEnvValue is used
+          kubectl get secret 
+          kubectl get secret | grep -vz scalyr-api-key
+
+      - name: Verify Agent Logs are Ingested
+        env:
+          scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
+          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
+        run: |
+          # Verify agent and kubernetes monitor has been started
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "Starting scalyr agent..."'
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "No checkpoints were found. All logs will be copied starting at their current end"'
+
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "Cluster name detected, enabling k8s metric reporting"'
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "kubernetes_monitor parameters: "'
+
+          # Verify Kubernetes metrics are beeing ingested
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_monitor.log" "k8s-daemon-set=\"scalyr-agent\""'
+
+      - name: Notify Slack on Failure
+        # NOTE: github.ref is set to pr ref (and not branch name, e.g. refs/pull/28/merge) for pull
+        # requests and that's why we need this special conditional and check for github.head_ref in
+        # case of PRs
+        if: ${{ failure() && (github.ref == 'refs/heads/main' || github.head_ref == 'main') }}
+        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f  # v2.0.0
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        with:
+          status: ${{ job.status }}
+          steps: ${{ toJson(steps) }}
+          channel: '#cloud-tech'
+
+  deployment_controller_type_raw_api_key_secret_value:
+    name: Deployment - raw secret - k8s ${{ matrix.k8s_version }} - ${{ matrix.image_type }}
+    runs-on: ubuntu-latest
+
+    needs: pre_job
+    timeout-minutes: 15
+    # NOTE: We always want to run job on main branch
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s_version:
+          - 'v1.24.3'
+        image_type:
+          - "buster"
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Set up Chart Testing Environment and Kubernetes Cluster
+        uses: ./.github/actions/setup-chart-testing-environment/
+        with:
+          k8s_version: "${{ matrix.k8s_version }}"
+          github_token: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Install Scalyr Tool
+        uses: ./.github/actions/install-scalyr-tool/
+
+      - name: Install Helm Chart
+        uses: ./.github/actions/install-helm-chart
+        with:
+          scalyr_api_key: "${{ secrets.SCALYR_WRITE_API_KEY_US }}"
+          values_file_path: "ci/deployment-agent-values-raw-secret.yaml"
+          image_type: "${{ matrix.image_type }}"
+
+      - name: Describe Pod
+        run: |
+          set -e
+
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME}
+
+          # Verify test volume defined using chart volumes and volumeMounts value is there
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME} | grep "/test-volume from test-volume (rw)"
+          kubectl describe pod ${SCALYR_AGENT_POD_NAME} | grep "test-volume:"
+
+          # Verify that secret hasn't been created since useRawApiKeyEnvValue is used
+          kubectl get secret 
+          kubectl get secret | grep -vz scalyr-api-key
+
+      - name: Verify Logs are Ingested
+        env:
+          scalyr_readlog_token: "${{ secrets.SCALYR_READ_API_KEY_US }}"
+          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
+        run: |
+          # Verify agent and kubernetes monitor has been started
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "Starting scalyr agent..."'
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "No checkpoints were found. All logs will be copied starting at their current end"'
+
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "Cluster name detected, enabling k8s metric reporting"'
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" attribute1="value1" "kubernetes_monitor parameters: "'
+
+          # Verify Kubernetes metrics are beeing ingested
+          ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_monitor.log" "k8s-deployment=\"scalyr-agent\""'
+
+      - name: Notify Slack on Failure
+        # NOTE: github.ref is set to pr ref (and not branch name, e.g. refs/pull/28/merge) for pull
+        # requests and that's why we need this special conditional and check for github.head_ref in
+        # case of PRs
+        if: ${{ failure() && (github.ref == 'refs/heads/main' || github.head_ref == 'main') }}
+        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f  # v2.0.0
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        with:
+          status: ${{ job.status }}
+          steps: ${{ toJson(steps) }}
+          channel: '#cloud-tech'
+
   daemonset_controller_type_k8s_explorer_no_deps:
     # In this workflow we manually install node exporter and kube state metrics exporter dependency
     name: K8s Explorer - no deps - k8s ${{ matrix.k8s_version }} - ${{ matrix.image_type }}

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 For actual scalyr agent changelog, please see https://github.com/scalyr/scalyr-agent-2/blob/release/CHANGELOG.md
 
+## 0.2.34
+
+- Add new ``useRawApiKeyEnvValue`` chart value. When this value is set to true (defaults to false),
+  ``scalyr.apiKey`` chart value is used as-is for the ``SCALYR_API_KEY`` pod environment variable.
+
+  When this value is set to true, corresponding ``Secret`` object is also not created.
+
+  This value can be used in deployments which utilize an operator / tool like kube-secrets-init
+  which directly replaces matching prefixed environment variable value with a secret value.
+
+  For more information on the use case and usage of this chart value, please refer to the
+  README.md.
+
+  #61 #64
+
 ## 0.2.33
 
 - Update chart for DataSet agent v2.2.4 release.

README.md

@@ -127,6 +127,23 @@ Example:
   ```
   This gives the pod permission to read the secret as defined in the IAM Policy. (Something in the cluster such as a MutatingWebhook will need to actually facilitate the secret lookup)
 
+## Using raw value for "SCALYR_API_KEY" pod environment variable
+
+By default, ``scalyr.apiKey`` chart value is stored in a Kubernetes Secret and then this secret is
+referenced by the ``SCALYR_API_KEY`` pod environment variable.
+
+In some situations, you may want to define a raw value for this environment variable. An example of
+that is using a tool like ``kube-secrets-init`` which relies on environment variable being set to a
+special prefixed value which will eventually get replaced with the actual secret by the tool itself.
+
+Here is an example excerpt chart configuration for such use case:
+
+```yaml
+# Values relevant to ServiceAccount
+scalyr:
+  apiKey: gcp:secretmanager:projects/$PROJECT_ID/secrets/myscalyragentapikey/versions/2
+useRawApiKeyEnvValue: true
+```
 
 ## Changelog
 

charts/scalyr-agent/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: scalyr-agent
 description: A Helm chart for deploying the Scalyr agent
 type: application
-version: 0.2.33
+version: 0.2.34
 appVersion: 2.2.4
 keywords:
   - scalyr

charts/scalyr-agent/templates/daemonset.yaml

@@ -89,6 +89,9 @@ spec:
             - name: "SCALYR_SERVER"
               value: {{ .Values.scalyr.server }}
             - name: "SCALYR_API_KEY"
+              {{- if .Values.useRawApiKeyEnvValue }}
+              value: {{ .Values.scalyr.apiKey }}
+              {{- else }}
               valueFrom:
                 secretKeyRef:
                   {{- if .Values.existingSecretRef }}
@@ -97,6 +100,7 @@ spec:
                   name: "{{ include "scalyr-helm.fullname" . }}-scalyr-api-key"
                   {{- end }}
                   key: "scalyr-api-key"
+              {{- end }}
             {{- if (or (or (.Values.scalyr.k8s.enableMetrics) (.Values.scalyr.k8s.enableLogs)) (.Values.scalyr.k8s.enableExplorer))  }}
             - name: "SCALYR_K8S_CLUSTER_NAME"
               value: "{{ .Values.scalyr.k8s.clusterName }}"

charts/scalyr-agent/templates/deployment.yaml

@@ -87,6 +87,9 @@ spec:
             - name: "SCALYR_SERVER"
               value: {{ .Values.scalyr.server }}
             - name: "SCALYR_API_KEY"
+              {{- if .Values.useRawApiKeyEnvValue }}
+              value: {{ .Values.scalyr.apiKey }}
+              {{- else }}
               valueFrom:
                 secretKeyRef:
                   {{- if .Values.existingSecretRef }}
@@ -95,6 +98,7 @@ spec:
                   name: "{{ include "scalyr-helm.fullname" . }}-scalyr-api-key"
                   {{- end }}
                   key: "scalyr-api-key"
+              {{- end }}
             {{- if (or (or (.Values.scalyr.k8s.enableMetrics) (.Values.scalyr.k8s.enableLogs)) (.Values.scalyr.k8s.enableExplorer))  }}
             - name: "SCALYR_K8S_CLUSTER_NAME"
               value: "{{ .Values.scalyr.k8s.clusterName }}"

charts/scalyr-agent/templates/secret.yaml

@@ -1,3 +1,4 @@
+{{- if not (.Values.useRawApiKeyEnvValue) }}
 {{- if eq .Values.existingSecretRef "" -}}
 {{- $name := .Values.scalyr.apiKey | required ".Values.scalyr.apiKey is required." -}}
 
@@ -9,4 +10,5 @@ metadata:
   {{- include "scalyr-helm.labels" . | nindent 4 }}
 data:
   scalyr-api-key: {{ .Values.scalyr.apiKey | b64enc }}
-{{- end }}
+{{- end }}
+{{- end }}

charts/scalyr-agent/values.yaml

@@ -28,7 +28,7 @@ volumeMounts: {}
 scalyr:
   # scalyr.server -- The Scalyr server to send logs to. Use eu.scalyr.com for EU
   server: "agent.scalyr.com"
-  # scalyr.apiKey -- The Scalyr API key to use
+  # scalyr.apiKey -- The Scalyr API key to use. Can also be used in combination with "useRawApiKeyEnvValue" when using something like kube-secrets-init. In that case, this should be a reference to the secret which will be replaced by kube-secrets-init.
   apiKey: ""
   # scalyr.debugLevel -- Set this to number between 1 and 5 (inclusive - 1 being least verbose and
   # 5 being most verbose) to enable additional debug logging into agent_debug.log file.
@@ -151,5 +151,8 @@ serviceAccount:
   # serviceAccount.annotations -- optional arbitrary service account annotations
   annotations: {}
 
+# useRawApiKeyEnvValue -- Set this to true if you want raw API key from "scalyr.apiKey" chart value to be used for the SCALYR_API_KEY pod environment variable. This comes handy in situations where you don't want to use a secret (e.g. you utilize something like kube-secrets-init which directly replaces environment variable value with the actual secret).
+useRawApiKeyEnvValue: false
+
 # existingSecretRef -- Use this value if the Scalyr API key is already stored in a Kubernetes secret that was created by an external secrets operator or similar.
 existingSecretRef: ""

ci/daemonset-agent-values-raw-secret.yaml

@@ -0,0 +1,31 @@
+# Values file used by end to end tests
+controllerType: "daemonset"
+podLabels:
+  test: "true"
+extraEnvVars:
+  - name: SCALYR_FOO_1
+    value: "foo1"
+  - name: SCALYR_BAR_2
+    value: "bar2"
+scalyr:
+  apiKey: "REPLACE_ME"
+  k8s:
+    clusterName: "k8s-explorer-e2e-tests"
+    verifyKubeletQueries: false
+  base64Config: true
+  config:
+    # ci/examples/agent.d/test-config.json
+    test-config.json: e3NlcnZlcl9hdHRyaWJ1dGVzOiB7YXR0cmlidXRlMTogInZhbHVlMSIsYXR0cmlidXRlMjogInZhbHVlMiIsfSx9
+
+useRawApiKeyEnvValue: true
+
+volumes:
+  - name: test-volume
+    emptyDir: {}
+
+volumeMounts:
+  - mountPath: /test-volume
+    name: test-volume
+
+image:
+  distro: "IMAGE_TYPE"

ci/deployment-agent-values-raw-secret.yaml

@@ -0,0 +1,23 @@
+# Values file used by end to end tests
+controllerType: "deployment"
+podLabels:
+  test: "true"
+scalyr:
+  apiKey: "REPLACE_ME"
+  k8s:
+    clusterName: "k8s-explorer-e2e-tests"
+    verifyKubeletQueries: false
+  base64Config: true
+  config:
+    # ci/examples/agent.d/test-config.json
+    test-config.json: e3NlcnZlcl9hdHRyaWJ1dGVzOiB7YXR0cmlidXRlMTogInZhbHVlMSIsYXR0cmlidXRlMjogInZhbHVlMiIsfSx9
+
+useRawApiKeyEnvValue: true
+
+volumes:
+  - name: test-volume
+    emptyDir: {}
+
+volumeMounts:
+  - mountPath: /test-volume
+    name: test-volume