tests.yaml

name: tests
run-name: tests, branch:${{ github.ref_name }}, triggered by @${{ github.actor }}

concurrency:
  # Run only for most recent commit in PRs but for all tags and commits on main
  # Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency
  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

on:
  merge_group:
    branches:
      - 'main'
  pull_request:
    branches:
      - '**'
      - 'release/*'
  push:
    branches:
      - 'main'
      - 'release/*'
    tags:
      - '*'
  workflow_dispatch: {}

# NOTE: Some jobs add GITHUB_API_TOKEN env so that some mise plugins which
# download GitHub release assets can access GitHub's API authenticated and thus
# not get rate-limited (which causes failures).

# NOTE: Some jobs require GITHUB_TOKEN env which is used by mise itself to access
# GitHub's API authenticated and thus not get rate-limited (which causes failures).
# Ref: https://mise.jdx.dev/getting-started.html#github-api-rate-limiting.
#
# Recent versions of mise started to also accept MISE_GITHUB_TOKEN as an alternative to
# GITHUB_TOKEN. That env var is set by the mise-action automatically.

permissions:
  contents: read
  attestations: read

env:
  MISE_VERBOSE: 1

jobs:
  build-e2e-image:
    runs-on: ubuntu-latest
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    env:
      IMG: kong-operator
      TAG: e2e-${{ github.sha }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit
      - name: checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: setup golang
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version-file: go.mod

      - name: build docker image
        run: make docker.build

      - name: save docker image as artifact
        run: docker save ${{ env.IMG }}:${{ env.TAG }} -o kong-operator_e2e.tar

      - name: upload docker image artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: kong-operator-e2e-image-${{ github.run_id }}
          path: kong-operator_e2e.tar
          retention-days: 1
          if-no-files-found: error

  check-docs-only:
    runs-on: ubuntu-latest
    outputs:
      docs_only: ${{ steps.check-files.outputs.docs_only }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - id: check-files
        name: Check if only documentation files changed
        run: ./scripts/check-docs-only-changes.sh ${{ github.event.pull_request.base.sha || 'HEAD~1' }} ${{ github.sha }}

  ensure-actions-sha-pin:
    runs-on: ubuntu-latest
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - uses: zgosalvez/github-actions-ensure-sha-pinned-actions@70c4af2ed5282c51ba40566d026d6647852ffa3e # v5.0.1

  ossf-scorecard:
    runs-on: ubuntu-latest
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    # Ref: https://github.com/ossf/scorecard
    - run: |
        GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
          REPO=github.com/${{ github.repository }} \
          COMMIT=${{ github.sha }} \
          make docker.run.openssf

  lint:
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__lint.yaml
    secrets: inherit

  lint-markdownlint:
    runs-on: ubuntu-latest
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false

    - name: run lint.markdownlint
      run: make lint.markdownlint

  govulncheck:
    runs-on: ubuntu-latest
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Setup go
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod
    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false
    - run: make govulncheck

  verify:
    runs-on: ubuntu-latest
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Setup go
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod
        cache: false

    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false

    - run: echo "GO_MOD_CACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV
    - run: echo "GO_BUILD_CACHE=$(go env GOCACHE)" >> $GITHUB_ENV

    - run: echo "GO_CACHE_KEY=go-build-${{ runner.os }}-${{ runner.arch }}-go-${{ hashFiles('**/go.mod') }}" >> $GITHUB_ENV
    - run: echo "GO_CACHE_KEY_2=go-build-${{ runner.os }}-${{ runner.arch }}-go-" >> $GITHUB_ENV

    - name: Go cache (restore only from build cache)
      uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306
      with:
        path: |
          ${{ env.GO_MOD_CACHE }}
          ${{ env.GO_BUILD_CACHE }}
        key: ${{ env.GO_CACHE_KEY }}
        restore-keys: |
          ${{ env.GO_CACHE_KEY }}
          ${{ env.GO_CACHE_KEY_2 }}

    - name: Verify go fix
      run: make verify.go-fix

    - name: Verify manifests consistency
      run: make verify.manifests

    - name: Verify generators consistency
      uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
      with:
        timeout_minutes: 8
        max_attempts: 3
        retry_wait_seconds: 30
        command: make verify.generators

  matrix_k8s_node_versions:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      latest: ${{ steps.set-latest.outputs.latest }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - id: set-matrix
        run: |
          (
            echo 'matrix<<EOF'
            yq eval -o=json '.' .github/supported_k8s_node_versions.yaml
            echo 'EOF'
          ) >> "${GITHUB_OUTPUT}"
      - id: set-latest
        run: |
          (
            echo 'latest<<EOF'
            yq eval -r -o=json '. | sort | reverse | .[0]' .github/supported_k8s_node_versions.yaml
            echo 'EOF'
          ) >> "${GITHUB_OUTPUT}"

  samples:
    runs-on: ubuntu-latest
    needs:
    - check-docs-only
    - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Setup go
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod

    - name: Create k8s KinD Cluster
      uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
      with:
        # NOTE: default is 0.29.0 https://github.com/helm/kind-action/blob/a1b0e391336a6ee6713a0583f8c6240d70863de3/kind.sh#L21
        # so bump this manually
        version: v0.30.0
        node_image: kindest/node:${{ needs.matrix_k8s_node_versions.outputs.latest }}

    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false

    # We use install.all to install all CRDs and resources also the ones that are not bundled
    # in base kustomization (e.g. currently AIGateway) but which have samples defined.
    - name: Verify installing CRDs via kustomize works
      run: make install.all

    - name: Install and delete each sample one by one
      run: make test.samples

    - name: Verify that uninstalling operator CRDs via kustomize works
      run: make ignore-not-found=true uninstall.all

  install-with-kustomize:
    runs-on: ubuntu-latest
    needs:
    - check-docs-only
    - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    env:
      IMG: kong-operator
      TAG: e2e-${{ github.sha }}
      CLUSTER_NAME: install-with-kustomize
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - name: Checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Setup go
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod

    - name: build docker image
      run: make docker.build

    - name: Create k8s KinD Cluster
      uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
      with:
        cluster_name: ${{ env.CLUSTER_NAME }}
        ignore_failed_clean: true
        node_image: kindest/node:${{ needs.matrix_k8s_node_versions.outputs.latest }}

    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false

    - name: Verify installing CRDs via kustomize works
      run: make install

    - name: Load image to kind cluster
      run: kind load docker-image kong-operator:e2e-${{ github.sha }} --name $CLUSTER_NAME

    - name: Verify deploying operator via kustomize works
      env:
        IMG: kong-operator
        VERSION: e2e-${{ github.sha }}
      run: make deploy

    - name: Verify that undeploying operator via kustomize works
      run: make undeploy

    - name: Verify that uninstalling operator CRDs via kustomize works
      run: make ignore-not-found=true uninstall

  build:
    runs-on: ubuntu-latest
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: setup golang
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod
        cache: false

    - run: echo "GO_MOD_CACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV
    - run: echo "GO_BUILD_CACHE=$(go env GOCACHE)" >> $GITHUB_ENV

    - run: echo "GO_CACHE_KEY=go-build-${{ runner.os }}-${{ runner.arch }}-go-${{ hashFiles('**/go.mod') }}" >> $GITHUB_ENV
    - run: echo "GO_CACHE_KEY_2=go-build-${{ runner.os }}-${{ runner.arch }}-go-" >> $GITHUB_ENV

    - name: Go cache main branch
      if: ${{ github.ref == 'refs/heads/main' }}
      uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306
      with:
        path: |
          ${{ env.GO_MOD_CACHE }}
          ${{ env.GO_BUILD_CACHE }}
        key: ${{ env.GO_CACHE_KEY }}
        restore-keys: |
          ${{ env.GO_CACHE_KEY }}
          ${{ env.GO_CACHE_KEY_2 }}

    - name: Go cache (restore only for non-main branches)
      if: ${{ github.ref != 'refs/heads/main' }}
      uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306
      with:
        path: |
          ${{ env.GO_MOD_CACHE }}
          ${{ env.GO_BUILD_CACHE }}
        key: ${{ env.GO_CACHE_KEY }}
        restore-keys: |
          ${{ env.GO_CACHE_KEY }}
          ${{ env.GO_CACHE_KEY_2 }}

    - run: make build.operator

    - name: Test the binary by running it with -version flag
      run: ./bin/manager -version | ./scripts/verify-version.sh ${{ github.repository }}

  unit-tests:
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__unit_tests.yaml
    secrets: inherit

  CRDs:
    needs:
    - check-docs-only
    - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__crdvalidation_tests.yaml
    secrets: inherit
    with:
      cluster_versions: ${{ needs.matrix_k8s_node_versions.outputs.matrix }}  

  envtest-tests:
    needs:
    - check-docs-only
    - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__envtest_tests.yaml
    secrets: inherit
    with:
      cluster_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }}
  
  kongintegration-tests:
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__kongintegration_tests.yaml
    secrets: inherit

  conformance-tests:
    needs: [check-docs-only]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__conformance_tests.yaml
    secrets: inherit

  integration-tests:
    needs:
      - check-docs-only
      - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__integration_tests.yaml
    secrets: inherit
    with:
      cluster_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }}

  e2e-tests:
    runs-on: ubuntu-latest
    needs: [check-docs-only, build-e2e-image]
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
      with:
        egress-policy: audit
    - name: checkout repository
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        fetch-depth: 0

    - name: setup golang
      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
      with:
        go-version-file: go.mod

    - name: download e2e image artifact
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        name: kong-operator-e2e-image-${{ github.run_id }}

    - name: load docker image
      run: docker load -i kong-operator_e2e.tar

    - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3
      with:
        install: false

    - name: run e2e tests
      run: make test.e2e
      env:
        KONG_TEST_KONG_OPERATOR_IMAGE_LOAD: kong-operator:e2e-${{ github.sha }}
        GOTESTSUM_JUNITFILE: "e2e-tests.xml"

    - name: upload diagnostics
      if: always()
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: diagnostics-e2e
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: tests-report-e2e
        path: e2e-tests.xml

  e2e-tests-chainsaw:
    needs:
      - check-docs-only
      - build-e2e-image
      - matrix_k8s_node_versions
    if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
    uses: ./.github/workflows/__e2e_chainsaw_tests.yaml
    secrets: inherit
    with:
      node_image_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }}
      artifacts_run_id: ${{ github.run_id }}

  buildpulse-report:
    needs:
      - check-docs-only
      - unit-tests
      - envtest-tests
      - integration-tests
      - conformance-tests
      - e2e-tests
    if: ${{ always() && needs.check-docs-only.outputs.docs_only != 'true' }}
    runs-on: ubuntu-latest
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit

      - name: checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: download tests report
        id: download-coverage
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          pattern: tests-report*
          path: report
          merge-multiple: true

      - name: Upload test results to BuildPulse for flaky test detection
        if: ${{ !cancelled() }}
        uses: buildpulse/buildpulse-action@d4d8e00c645a2e3db0419a43664bbcf868080234 # v0.12.0
        with:
          account: 962416
          repository: 477814940
          path: report/*.xml
          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

  # We need this step to fail the workflow if any of the previous steps failed or were cancelled.
  # It allows to use this particular job as a required check for PRs.
  # Ref: https://github.com/orgs/community/discussions/26822#discussioncomment-3305794
  passed:
    runs-on: ubuntu-latest
    needs:
      - check-docs-only
      - ensure-actions-sha-pin
      - ossf-scorecard
      - lint
      - lint-markdownlint
      - govulncheck
      - verify
      - samples
      - install-with-kustomize
      - build
      - build-e2e-image
      - CRDs
      - unit-tests
      - envtest-tests
      - kongintegration-tests
      - conformance-tests
      - integration-tests
      - e2e-tests
      # TODO: reenable when chainsaw e2e tests are stable
      # https://github.com/Kong/kong-operator/issues/2960
      # - e2e-tests-chainsaw
      - buildpulse-report
    if: always()
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
        with:
          egress-policy: audit
      - name: Check if docs-only changes
        run: |
          if [[ "${{ needs.check-docs-only.outputs.docs_only }}" == "true" ]]; then
            echo "Only documentation files were changed, skipping tests"
            exit 0
          fi

      - name: Check test results
        if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }}
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "Some jobs failed or were cancelled."
            exit 1
          fi