operator-integration-test.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

name: Operator Integration Test
on:
  push:
    branches:
      - main
  pull_request:
    types: [ opened, reopened, synchronize, ready_for_review ]
    branches:
      - main
  workflow_dispatch:
concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read

jobs:
  InstrumentationTest:
    name: InstrumentationTest
    needs: [BuildAndCacheImage]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5
      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          sleep 10
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar


      - name: Deploy operator to minikube
        run: |
          make deploy
          kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch


      - name: Test case for cloudwatch agent pod creation
        run: |
          kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset.yaml -n amazon-cloudwatch
          sleep 20
          kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch

          pod_name="$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/component=amazon-cloudwatch-agent,app.kubernetes.io/instance=amazon-cloudwatch.cloudwatch-agent -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"
          if [ -z "$pod_name" ]; then
            echo "Pod $pod_name is not created. Exiting with ERROR."
            echo "Pod is not created test case: FAIL"
          else
            echo "Pod $pod_name is created."
            echo "Pod is created test case: PASS"
          fi

      - name: Test for default instrumentation resources for Java
        run: |
          kubectl apply -f integration-tests/java/sample-deployment-java.yaml
          sleep 15
          kubectl wait --for=condition=Ready pod --all -n default
          kubectl get pods -A
          kubectl describe pods -n default
          kubectl logs deployment/cloudwatch-controller-manager --all-containers=true -n amazon-cloudwatch
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json app_signals

      - name: Test for defined instrumentation resources for Java
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 15
          kubectl wait --for=condition=Ready pod --all -n default

          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test for default instrumentation resources for python
        run: |
          kubectl apply -f integration-tests/python/sample-deployment-python.yaml
          sleep 15
          kubectl wait --for=condition=Ready pod --all -n default --timeout=120s
          kubectl get pods -A
          kubectl describe pods -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python/default_instrumentation_python_env_variables.json app_signals

      - name: Test for defined instrumentation resources for python
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 15
          kubectl wait --for=condition=Ready pod --all -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test for default instrumentation resources for dotnet
        run: |
          cat integration-tests/dotnet/sample-deployment-dotnet.yaml
          kubectl apply -f integration-tests/dotnet/sample-deployment-dotnet.yaml
          sleep 15
          kubectl wait --for=condition=Available deployment/nginx -n default
          kubectl get pods -A
          kubectl describe pods -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/dotnet/default_instrumentation_dotnet_env_variables.json app_signals

      - name: Test for defined instrumentation resources for dotnet
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 15
          kubectl wait --for=condition=Ready pod --all -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test for default instrumentation resources for nodejs
        run: |
          kubectl delete pods --all -n default
          sleep 15
          cat integration-tests/nodejs/sample-deployment-nodejs.yaml
          kubectl apply -f integration-tests/nodejs/sample-deployment-nodejs.yaml
          sleep 15
          kubectl wait --for=condition=Available deployment/nginx -n default
          kubectl get pods -A
          kubectl describe pods -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/nodejs/default_instrumentation_nodejs_env_variables.json app_signals

      - name: Test for defined instrumentation resources for nodejs
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 15
          kubectl wait --for=condition=Available deployment/nginx -n default
          sleep 15
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test for default instrumentation resources for all languages
        run: |
          kubectl apply -f integration-tests/all-languages/sample-deployment-all-languages.yaml
          sleep 15
          kubectl wait --for=condition=Available deployment/nginx -n default
          sleep 15
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/all-languages/default_instrumentation_all-languages_env_variables.json app_signals

      - name: Test for defined instrumentation resources for all languages
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 15
          kubectl wait --for=condition=Available deployment/nginx -n default
          sleep 15
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test for default instrumentation resources for jmx
        run: |
          kubectl delete pods --all -n default
          sleep 5
          cat integration-tests/jmx/sample-deployment-jmx.yaml
          kubectl apply -f integration-tests/jmx/sample-deployment-jmx.yaml
          sleep 5
          kubectl wait --for=condition=Available deployment/nginx -n default
          kubectl get pods -A
          kubectl describe pods -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/jmx/default_instrumentation_jmx_env_variables.json app_signals

      - name: Test for defined instrumentation resources for jmx
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 5
          kubectl wait --for=condition=Available deployment/nginx -n default
          sleep 5
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

      - name: Test case for cloudwatch agent without app signals pod creation
        run: |
          kubectl delete daemonset cloudwatch-agent -n amazon-cloudwatch
          sleep 5
          kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset-no-app-signals.yaml -n amazon-cloudwatch
          sleep 20
          kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch

          pod_name="$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/component=amazon-cloudwatch-agent,app.kubernetes.io/instance=amazon-cloudwatch.cloudwatch-agent -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"
          if [ -z "$pod_name" ]; then
            echo "Pod $pod_name is not created. Exiting with ERROR."
            echo "Pod is not created test case: FAIL"
          else
            echo "Pod $pod_name is created."
            echo "Pod is created test case: PASS"
          fi

      - name: Test for default instrumentation resources for jmx without app signals
        run: |
          kubectl delete pods --all -n default
          sleep 5
          cat integration-tests/jmx/sample-deployment-jmx.yaml
          kubectl apply -f integration-tests/jmx/sample-deployment-jmx.yaml
          sleep 5
          kubectl wait --for=condition=Available deployment/nginx -n default
          kubectl get pods -A
          kubectl describe pods -n default
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/jmx/default_instrumentation_jmx_env_variables_no_app_signals.json no_app_signals

      - name: Test for defined instrumentation resources for jmx without app signals
        run: |
          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
          kubectl delete pods --all -n default
          sleep 5
          kubectl wait --for=condition=Available deployment/nginx -n default
          sleep 5
          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
          kubectl delete instrumentation sample-instrumentation

  DeploymentAnnotationsTest:
    name: DeploymentAnnotationsTest
    needs: [BuildAndCacheImage]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5

      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          sleep 10
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar

      - name: Deploy operator to minikube
        run: |
          make deploy

      - name: Test Annotations
        run: |
          kubectl get pods -A
          kubectl describe pods -n default
          sleep 10
          go test -v -run TestAllLanguagesDeployment ./integration-tests/manifests/annotations -timeout 30m
          kubectl get pods -A
          kubectl describe pods -n default
          sleep 5
          go test -v -run TestJavaOnlyDeployment ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestPythonOnlyDeployment ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestDotNetOnlyDeployment ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestNodeJSOnlyDeployment ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestAnnotationsOnMultipleResources ./integration-tests/manifests/annotations -timeout 30m
  DefineAutoMonitorTestMatrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v5
      - name: Generate test list
        id: tests
        run: |
          cd integration-tests/manifests/automonitor/ && \
          echo "tests<<EOF" >> $GITHUB_OUTPUT
          go test -list . | sed -e '$d' >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT
      - name: Create test matrix
        id: set-matrix
        run: |
          TESTS=$(echo "${{ steps.tests.outputs.tests }}" | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "matrix={\"test\":${TESTS}}" >> $GITHUB_OUTPUT
  BuildAndCacheImage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5
      - name: Build image
        run: |
          make container
          docker save $(docker images --format '{{.Repository}}:{{.Tag}}' | grep operator | head -n 1) > image.tar

      - name: Cache Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

  AutoMonitorTest:
    name: AutoMonitorTest
    runs-on: ubuntu-latest
    needs: [DefineAutoMonitorTestMatrix, BuildAndCacheImage]
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix: ${{fromJson(needs.DefineAutoMonitorTestMatrix.outputs.matrix)}}
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5

      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          sleep 10
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar

      - name: Deploy operator to minikube
        run: |
          make deploy

      - name: Test AutoMonitor-created annotations
        run: |
          kubectl get pods -A
          kubectl describe pods -n default
          go test -v ./integration-tests/manifests/automonitor -run ^${{ matrix.test }}$ -timeout 60m -controllerManagerName 'cloudwatch-controller-manager'

  DaemonsetAnnotationsTest:
    name: DaemonsetAnnotationsTest
    needs: [BuildAndCacheImage]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5

      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          sleep 10
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar


      - name: Deploy operator to minikube
        run: |
          make deploy

      - name: Test Annotations
        run: |
          sleep 5
          kubectl get pods -A
          kubectl describe pods -n default
          go test -v -run TestAllLanguagesDaemonSet ./integration-tests/manifests/annotations -timeout 30m
          kubectl get pods -A
          kubectl describe pods -n default
          sleep 5
          go test -v -run TestJavaOnlyDaemonSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestPythonOnlyDaemonSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestDotNetOnlyDaemonSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestNodeJSOnlyDaemonSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestAutoAnnotationForManualAnnotationRemoval ./integration-tests/manifests/annotations -timeout 30m

  StatefulsetAnnotationsTest:
    name: StatefulsetAnnotationsTest
    needs: [BuildAndCacheImage]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5

      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          sleep 10
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar


      - name: Deploy operator to minikube
        run: |
          make deploy

      - name: Test Annotations
        run: |
          kubectl get pods -A
          kubectl describe pods -n default
          go test -v -run TestAllLanguagesStatefulSet ./integration-tests/manifests/annotations -timeout 30m
          kubectl get pods -A
          kubectl describe pods -n default
          sleep 5
          go test -v -run TestJavaOnlyStatefulSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestPythonOnlyStatefulSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestDotNetOnlyStatefulSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestNodeJSOnlyStatefulSet ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestOnlyNonAnnotatedAppsShouldBeRestarted ./integration-tests/manifests/annotations -timeout 30m



  NamespaceAnnotationsTest:
    name: NamespaceAnnotationsTest
    needs: [BuildAndCacheImage]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5

      - name: Start minikube
        uses: medyagh/setup-minikube@master

      - name: Deploy cert-manager to minikube
        run:
          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml

      - name: Verify minikube and cert-manager
        run: |
          kubectl get pods -A

      - name: Restore cached Docker image
        uses: actions/cache@v3
        with:
          path: image.tar
          key: docker-image-${{ github.sha }}

      - name: Load Docker image
        run: |
          eval $(minikube docker-env)
          docker load < image.tar


      - name: Deploy operator to minikube
        run: |
          make deploy

      - name: Test Annotations
        run: |
          kubectl get pods -A
          kubectl describe pods -n default
          go test -v -run TestAllLanguagesNamespace ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestJavaOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestPythonOnlyNamespace ./integration-tests/manifests/annotations -timeout 45m
          sleep 5
          go test -v -run TestDotNetOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestNodeJSOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m
          sleep 5
          go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m