Merge pull request #22 from scalyr/e2e_tests_k8s_image_cache

Various chart improvements

Author: tomaz-s1

.github/actions/install-helm-chart/action.yml

@@ -11,7 +11,7 @@ inputs:
   start_sleep_delay:
     description: "How many seconds to sleep / wait after installing the chart for the pod to start up"
     required: true
-    default: 20
+    default: 30
   ingest_sleep_delay:
     description: "How many seconds to sleep / wait after the pod has been started up for initial ingestion"
     required: true
@@ -28,7 +28,7 @@ runs:
       uses: nick-fields/retry@b4fa57557dda8c2f30bcb2d19372cc3237190f7f  # v2.8.1
       with:
         shell: bash
-        timeout_seconds: 100
+        timeout_seconds: 130
         max_attempts: 3
         command: |
           set -e
@@ -42,7 +42,9 @@ runs:
           # Install chart
           helm install --debug --wait --values "${{ inputs.values_file_path }}" scalyr-agent charts/scalyr-agent
 
-          # Verify deployment is either there or not there - depending on the controller type
+          # Verify damonset / deployment is there - it depends on the run type
+          kubectl get daemonset
+          kubectl get daemonset scalyr-agent || true
           kubectl get deployments
           kubectl get deployment scalyr-agent || true
 

.github/actions/setup-chart-testing-environment/action.yml

@@ -32,7 +32,7 @@ runs:
       id: setup-chart-testing
       uses: helm/chart-testing-action@09ed88797198755e5031f25be13da255e7e33aad  # v2.3.0
       with:
-        version: v3.5.0
+        version: v3.7.0
 
     - name: Create minikube Kubernetes ${{ inputs.k8s_version }} Cluster
       id: create-minikube-cluster
@@ -41,6 +41,7 @@ runs:
       with:
         minikube version: '${{ inputs.minikube_version }}'
         kubernetes version: '${{ inputs.k8s_version }}'
+        start args: '--wait=all'
         github token: '${{ inputs.github_token }}'
 
     - name: Print minikube environment info

.github/actions/verify-k8s-explorer-logs/action.yml

@@ -52,13 +52,15 @@ runs:
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent_debug.log" "Adding new monitor with config:"'
 
         # Assert that Kubernetes Explorer metrics which we scrape from 2 annotated pods are there
-        ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests" k8s-daemon-set="node-exporter" "level=info collector=cpu"'
+
+        ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-metrics.log" k8s-cluster="k8s-explorer-e2e-tests" "kubelet_runtime_operations_total "'
 
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests"'
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests" "container_start_time_seconds "'
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "kubernetes-api-cadvisor-metrics.log" k8s-cluster="k8s-explorer-e2e-tests" "container_spec_memory_limit_bytes "'
 
         # Verify metrics scraped from node-exporter and kube-state-metrics are available
+        ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests" k8s-daemon-set="node-exporter" "level=info collector=cpu"'
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests"'
         ./ci/scripts/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" monitor="openmetrics_monitor" logfile contains "node-exporter" k8s-cluster="k8s-explorer-e2e-tests" "process_resident_memory_bytes"'
 

.github/workflows/end_to_end_tests.yaml

@@ -2,6 +2,8 @@
 # Those tests verify that the chart can be installed and that it correctly configured agent
 # ingestion for various options (daemonset mode, deployment mode, k8s cluster metrics are ingested,
 # pod logs are ingested, etc.)
+# NOTE: To reduce the size of the build matrix and speed up test run, we only run single set of tests using both Buster
+# and Alpine Linux based Docker image. Rest of the tests only exercise single image.
 name: "End to End Tests"
 
 on:
@@ -38,6 +40,7 @@ jobs:
     runs-on: ubuntu-latest
 
     needs: pre_job
+    timeout-minutes: 15
     # NOTE: We always want to run job on main branch
     if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
 
@@ -55,7 +58,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - name: Set up Chart Testing Environment and Kubernetes Cluster
         uses: ./.github/actions/setup-chart-testing-environment/
@@ -136,6 +139,7 @@ jobs:
     runs-on: ubuntu-latest
 
     needs: pre_job
+    timeout-minutes: 15
     # NOTE: We always want to run job on main branch
     if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
 
@@ -147,13 +151,12 @@ jobs:
           - 'v1.24.3'
         image_type:
           - "buster"
-          - "alpine"
 
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - name: Set up Chart Testing Environment and Kubernetes Cluster
         uses: ./.github/actions/setup-chart-testing-environment/
@@ -207,6 +210,7 @@ jobs:
     runs-on: ubuntu-latest
 
     needs: pre_job
+    timeout-minutes: 15
     # NOTE: We always want to run job on main branch
     if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
 
@@ -215,16 +219,17 @@ jobs:
       matrix:
         k8s_version:
           - 'v1.22.2'
-          - 'v1.24.3'
+          # Started randomly failing without any changes on our side. Looks like weird DNS /
+          # networking issue inside minikube when running on GHA (runs fine locally)
+          #- 'v1.24.3'
         image_type:
           - "buster"
-          - "alpine"
 
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - name: Set up Chart Testing Environment and Kubernetes Cluster
         uses: ./.github/actions/setup-chart-testing-environment/
@@ -242,21 +247,39 @@ jobs:
           values_file_path: "ci/daemonset-agent-values-with-k8s-explorer.yaml"
           image_type: "${{ matrix.image_type }}"
 
+      - name: Checkout Agent Repository
+        uses: actions/checkout@v3
+        with:
+          repository: "scalyr/scalyr-agent-2"
+          path: "scalyr-agent-2"
+          fetch-depth: 1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Here we build the dummy Java app image which exposes JMX metrics via exporter
+      # TODO: Cache this dummy / test image for faster builds
+      - name: Build Mock Java App Docker Image
+        uses: docker/build-push-action@v3
+        with:
+          context: scalyr-agent-2/tests/e2e/k8s_om_monitor/java-hello-world/
+          tags: java-hello-world:latest
+          # load image into docker
+          load: true
+          push: false
+          #push: true
+          #tags: localhost:5000/java-hello-world:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       # Create mock pods and exporters which will be scrapped by the monitor
       # (step taken from scalyr-agent-2 repo)
       - name: Create mock pods and exporters
         run: |
           set -e
 
-          git clone --depth 1 https://github.com/scalyr/scalyr-agent-2.git
           pushd scalyr-agent-2/
 
-          # Here we build the dummy Java app image which exposes JMX metrics via exporter
-          # TODO: Cache this dummy / test image for faster builds
-          pushd tests/e2e/k8s_om_monitor/java-hello-world
-          docker build -t java-hello-world .
-          popd
-
           docker image ls
           minikube image load java-hello-world:latest
 
@@ -305,6 +328,7 @@ jobs:
     runs-on: ubuntu-latest
 
     needs: pre_job
+    timeout-minutes: 15
     # NOTE: We always want to run job on main branch
     if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/main' }}
 
@@ -313,16 +337,17 @@ jobs:
       matrix:
         k8s_version:
           - 'v1.22.2'
-          - 'v1.24.3'
+          # Started randomly failing without any changes on our side. Looks like weird DNS /
+          # networking issue inside minikube when running on GHA (runs fine locally)
+          #- 'v1.24.3'
         image_type:
           - "buster"
-          - "alpine"
 
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - name: Set up Chart Testing Environment and Kubernetes Cluster
         uses: ./.github/actions/setup-chart-testing-environment/
@@ -340,30 +365,42 @@ jobs:
           values_file_path: "ci/daemonset-agent-values-with-k8s-explorer-with-deps.yaml"
           image_type: "${{ matrix.image_type }}"
 
+      - name: Checkout Agent Repository
+        uses: actions/checkout@v3
+        with:
+          repository: "scalyr/scalyr-agent-2"
+          path: "scalyr-agent-2"
+          fetch-depth: 1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Here we build the dummy Java app image which exposes JMX metrics via exporter
+      # TODO: Cache this dummy / test image for faster builds
+      - name: Build Mock Java App Docker Image
+        uses: docker/build-push-action@v3
+        with:
+          context: scalyr-agent-2/tests/e2e/k8s_om_monitor/java-hello-world/
+          tags: java-hello-world:latest
+          # load image into docker
+          load: true
+          push: false
+          #push: true
+          #tags: localhost:5000/java-hello-world:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Create mock pods and exporters
         run: |
           set -e
 
-          git clone --depth 1 https://github.com/scalyr/scalyr-agent-2.git
           pushd scalyr-agent-2/
 
-          # Here we build the dummy Java app image which exposes JMX metrics via exporter
-          # TODO: Cache this dummy / test image for faster builds
-          pushd tests/e2e/k8s_om_monitor/java-hello-world
-          docker build -t java-hello-world .
-          popd
-
           docker image ls
           minikube image load java-hello-world:latest
 
           kubectl create namespace monitoring
 
-          # 1. node exporter pod
-          kubectl apply -f tests/e2e/k8s_om_monitor/node_exporter.yaml
-
-          # 2. kube state metrics deployment
-          kubectl apply -k tests/e2e/k8s_om_monitor/kube-state-metrics/
-
           # 3. Install dummy java app container with jmx exporter side
           kubectl apply -f tests/e2e/k8s_om_monitor/java_app_deployment.yaml
           popd
@@ -372,7 +409,7 @@ jobs:
           kubectl get pods -A
 
           # Verify dependencies have been installed and are running
-            echo "Verifying node-exporter and kube-state-metrics pods are running"
+          echo "Verifying node-exporter and kube-state-metrics pods are running"
           kubectl get pods -A | grep node-exporter
           kubectl get pods -A | grep kube-state-metrics
 

.github/workflows/lint_tests.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - name: Set up Chart Testing Environment
         uses: ./.github/actions/setup-chart-testing-environment/
@@ -104,7 +104,7 @@ jobs:
         run: |
           # Write test values to a file which is used by ct install
           mkdir -p charts/scalyr-agent/ci/
-          echo -e 'controllerType: "daemonset"\nscalyr:\n  apiKey: "${API_KEY}"' > charts/scalyr-agent/ci/test-values.yaml
+          echo -e 'controllerType: "daemonset"\nscalyr:\n  apiKey: "${API_KEY}"\n  k8s:\n    clusterName: "k8s-explorer-e2e-tests"' > charts/scalyr-agent/ci/test-values.yaml
 
           ct install --debug --config ci/ct.yaml
 
@@ -116,7 +116,7 @@ jobs:
         run: |
           # Write test values to a file which is used by ct install
           mkdir -p charts/scalyr-agent/ci/
-          echo -e 'controllerType: "deployment"\nscalyr:\n  apiKey: "${API_KEY}"' > charts/scalyr-agent/ci/test-values.yaml
+          echo -e 'controllerType: "deployment"\nscalyr:\n  apiKey: "${API_KEY}"\n  k8s:\n    clusterName: "k8s-explorer-e2e-tests"' > charts/scalyr-agent/ci/test-values.yaml
 
           ct install --debug --config ci/ct.yaml
 

.github/workflows/release.yml

@@ -31,6 +31,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
+          # here we want to check out all the branches
           fetch-depth: 0
 
       - name: Configure Git

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 For actual scalyr agent changelog, please see https://github.com/scalyr/scalyr-agent-2/blob/release/CHANGELOG.md
 
+## 0.2.19
+
+- Update chart to throw an error if required ``scalyr.k8s.clusterName`` value is not specified.
+
+- Update Kubernetes Explorer config to make sure we also scrape Kubernetes API metrics.
+
 ## 0.2.18
 
 - Add new ``scalyr.k8s.installExplorerDependencies`` chart config option.

README.md

@@ -16,7 +16,7 @@ database service etc.).
 Use
 
 ```bash
-helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/
+helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.apiKey="<your write logs api key>" --set scalyr.k8s.clusterName="<your-k8s-cluster-name>"
 ```
 
 to install this chart.
@@ -85,7 +85,7 @@ This chart also supports configuring the agent for the Kubernetes Explorer funct
 To install the chart enabling Kubernetes Explorer functionality, you can use the following command:
 
 ```bash
-helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.k8s.enableExplorer=true
+helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.apiKey="<your write logs api key>" --set scalyr.k8s.clusterName="<your-k8s-cluster-name>" --set scalyr.k8s.enableExplorer=true
 ```
 
 This command will enable Kubernetes Explorer functionality, but it won't install additional
@@ -101,7 +101,7 @@ components for you.
 You can do that using the command below:
 
 ```bash
-helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.k8s.enableExplorer=true --set scalyr.k8s.installExplorerDependencies=true
+helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.apiKey="<your write logs api key>" --set scalyr.k8s.clusterName="<your-k8s-cluster-name>" --set scalyr.k8s.enableExplorer=true --set scalyr.k8s.installExplorerDependencies=true
 ```
 
 This functionality is primarily meant to be used for Kubernetes Explorer evaluation purposes when
@@ -118,7 +118,7 @@ very good reason for it, you should never disable certificate validation in prod
 For example:
 
 ```bash
-helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.k8s.enableExplorer=true --set scalyr.k8s.verifyKubeletQueries=false --set scalyr.k8s.eventsIgnoreMaster=false
+helm install <name of release> scalyr-agent --repo https://scalyr.github.io/helm-scalyr/ --set scalyr.apiKey="<your write logs api key>" --set scalyr.k8s.clusterName="<your-k8s-cluster-name>" --set scalyr.k8s.enableExplorer=true --set scalyr.k8s.verifyKubeletQueries=false --set scalyr.k8s.eventsIgnoreMaster=false
 ```
 
 This command also sets ``scalyr.k8s.eventsIgnoreMaster`` option to ``false``. This is needed when
@@ -130,6 +130,8 @@ default.
 
 For chart changelog, please see <https://github.com/scalyr/helm-scalyr/blob/main/CHANGELOG.md>.
 
+For agent changelog, please see <https://github.com/scalyr/scalyr-agent-2/blob/release/CHANGELOG.md>.
+
 ## Maintainers
 
 | Name | Email | Url |
@@ -256,7 +258,7 @@ minikube start
 ct lint --debug --config ci/ct.yaml
 
 # To use valid API key
-echo -e 'scalyr:\n  apiKey: "SCALYR_TEST_WRITE_API_KEY"' > charts/scalyr-agent/ci//test-values.yaml
+echo -e 'scalyr:\n  apiKey: "SCALYR_TEST_WRITE_API_KEY"' > charts/scalyr-agent/ci/test-values.yaml
 
 ct install --debug --config ci/ct.yaml
 ```

README.md.gotmpl

@@ -100,7 +100,7 @@ minikube start
 ct lint --debug --config ci/ct.yaml
 
 # To use valid API key
-echo -e 'scalyr:\n  apiKey: "SCALYR_TEST_WRITE_API_KEY"' > charts/scalyr-agent/ci//test-values.yaml
+echo -e 'scalyr:\n  apiKey: "SCALYR_TEST_WRITE_API_KEY"' > charts/scalyr-agent/ci/test-values.yaml
 
 ct install --debug
 ```