nightly-tests.yaml

name: Nightly Tests

env:
  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # necessary to pass upgrade tests
  ENVOYINIT_CACHE_REF: ghcr.io/${{ github.repository_owner }}/envoy-wrapper-cache
  CONTROLLER_CACHE_REF: ghcr.io/${{ github.repository_owner }}/kgateway-cache
  SDS_CACHE_REF: ghcr.io/${{ github.repository_owner }}/sds-cache

on:
  schedule:
    - cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against main and supported LTS branches
  workflow_dispatch:
    inputs:
      target_ref:
        description: "The Git ref to run tests against. Empty uses the workflow run commit."
        type: string
        required: false
        default: ""
      run-conformance:
        description: "Run conformance tests"
        type: boolean
        default: false
      run-load-tests:
        description: "Run load testing suite"
        type: boolean
        default: false
      run-e2e-tests:
        description: "Run e2e tests"
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ inputs.target_ref || github.ref_name }}
  cancel-in-progress: true

jobs:
  determine_refs:
    name: Determine Refs
    runs-on: ubuntu-22.04
    outputs:
      refs: ${{ steps.determine.outputs.refs }}
    steps:
      - id: determine
        name: Determine Refs
        shell: bash
        env:
          INPUT_TARGET_REF: ${{ inputs.target_ref }}
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then
            refs='[
              {"checkout_ref":"main","display_ref":"main"},
              {"checkout_ref":"v2.2.x","display_ref":"v2.2.x"},
              {"checkout_ref":"v2.3.x","display_ref":"v2.3.x"}
            ]'
          else
            checkout_ref="${INPUT_TARGET_REF:-}"
            display_ref="${INPUT_TARGET_REF:-}"
            if [[ -z "${checkout_ref}" ]]; then
              checkout_ref="${GITHUB_SHA}"
              display_ref="${GITHUB_REF_NAME}@${GITHUB_SHA::7}"
            fi
            refs="$(jq -cn --arg checkout_ref "${checkout_ref}" --arg display_ref "${display_ref}" \
              '[{checkout_ref:$checkout_ref,display_ref:$display_ref}]')"
          fi
          refs="$(jq -c . <<< "${refs}")"
          echo "refs=${refs}" >> "$GITHUB_OUTPUT"

  kube_gateway_api_conformance_tests:
    name: Conformance (ref=${{ matrix.ref.display_ref }}, type=Kubernetes Gateway API, k8s=${{ matrix.env-versions.label }}, gw-api=${{matrix.gateway-api-channel}})
    needs: determine_refs
    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance ) || github.event.schedule == '0 5 * * *' }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ref: ${{ fromJson(needs.determine_refs.outputs.refs) }}
        env-versions:
          - file: "./.github/workflows/.env/nightly-tests/max_versions.env"
            label: max_versions
          - file: "./.github/workflows/.env/nightly-tests/min_versions.env"
            label: min_versions
        gateway-api-channel: ['experimental', 'standard']
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          ref: ${{ matrix.ref.checkout_ref }}
      - name: Dotenv Action
        uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4
        id: dotenv
        with:
          path: ${{ matrix.env-versions.file }}
          log-variables: true
      - uses: ./.github/actions/kube-conformance-tests
        with:
          test-type: gateway-api
          api-channel: ${{ matrix.gateway-api-channel }}
          kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
          kind-node-version: ${{ steps.dotenv.outputs.node_version }}
          summarize-report: true

  kube_gateway_api_load_tests:
    name: Load Tests (ref=${{ matrix.ref.display_ref }}, type=Kubernetes Gateway API, k8s=${{ matrix.env-versions.label }})
    needs: determine_refs
    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-load-tests ) || github.event.schedule == '0 5 * * *' }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ref: ${{ fromJson(needs.determine_refs.outputs.refs) }}
        env-versions:
          - file: "./.github/workflows/.env/nightly-tests/max_versions.env"
            label: max_versions
          - file: "./.github/workflows/.env/nightly-tests/min_versions.env"
            label: min_versions
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          ref: ${{ matrix.ref.checkout_ref }}
      - name: Dotenv Action
        uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4
        id: dotenv
        with:
          path: ${{ matrix.env-versions.file }}
          log-variables: true
      - uses: ./.github/actions/kube-gateway-api-load-tests
        with:
          cluster-node-version: ${{ steps.dotenv.outputs.node_version }}

  kgateway_e2e_tests_for_gateway_api_versions:
    name: E2E (ref=${{ matrix.ref.display_ref }}, k8s=${{ matrix.env-versions.label }}, gw-api=${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }})
    needs: determine_refs
    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-e2e-tests ) || github.event.schedule == '0 5 * * *' }}
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        ref: ${{ fromJson(needs.determine_refs.outputs.refs) }}
        env-versions:
          - file: "./.github/workflows/.env/nightly-tests/max_versions.env"
            label: max
          - file: "./.github/workflows/.env/nightly-tests/min_versions.env"
            label: min
        gateway-api-version:
          - version: 'v1.6.1'
            channel: experimental
          - version: 'v1.6.1'
            channel: standard
          - version: 'v1.5.1'
            channel: experimental
          - version: 'v1.5.1'
            channel: standard
          - version: 'v1.4.1'
            channel: experimental
          - version: 'v1.4.1'
            channel: standard
          - version: 'v1.3.0'
            channel: experimental
          - version: 'v1.3.0'
            channel: standard
        controllers:
          - regex: "^TestKgateway|^TestAPIValidation|^TestListenerSet|^TestCustomGWP$$|^TestRouteReplacement$$|^TestZeroDowntimeRollout$$/^TestZeroDowntimeRollout$$|^TestControlPlaneTLS$$"
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          ref: ${{ matrix.ref.checkout_ref }}
      # Runs before the guard step below so the guard can key its Kubernetes-version check off
      # the real kubectl_version output instead of guessing from the min/max label.
      - name: Dotenv Action
        uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4
        id: dotenv
        with:
          path: ${{ matrix.env-versions.file }}
          log-variables: true
      # v2.2.x is built against Gateway API 1.4 and refuses to start on 1.5+ (#13872), so the
      # v2.2.x x v1.5+ lanes need special handling: skip the standard channel entirely, and run
      # the experimental channel with the version check bypassed to keep forward-compat signal.
      # This is done with step guards rather than a matrix `exclude:` because every dimension
      # here is object-valued and object-valued excludes are unreliable (actions/runner#1512).
      - name: Determine run plan
        id: guard
        env:
          REF: ${{ matrix.ref.checkout_ref }}
          GW_API_VERSION: ${{ matrix.gateway-api-version.version }}
          GW_API_CHANNEL: ${{ matrix.gateway-api-version.channel }}
          VERSIONS_LABEL: ${{ matrix.env-versions.label }}
          KUBE_VERSION: ${{ steps.dotenv.outputs.kubectl_version }}
          BASE_REGEX: ${{ matrix.controllers.regex }}
        shell: bash
        run: |
          run=true
          bypass_version_check=false
          run_regex="${BASE_REGEX}"
          if [[ "${REF}" == "v2.2.x" && "${GW_API_VERSION}" == "v1.6.1" ]]; then
            case "${GW_API_CHANNEL}" in
              standard)     run=false ;;
              experimental) bypass_version_check=true ;;
            esac
          fi
          if [[ "${REF}" == "v2.2.x" && "${GW_API_VERSION}" == "v1.5.1" ]]; then
            case "${GW_API_CHANNEL}" in
              standard)     run=false ;;
              experimental) bypass_version_check=true ;;
            esac
          fi
          # v2.2.x's chart predates ListenerSet support, so TestListenerSet fails at apply time
          # ("the server could not find the requested resource") even with the version check
          # bypassed. Drop it from the regex on v2.2.x while keeping the rest of the
          # forward-compat signal. (ListenerSet only runs on the experimental channel; the
          # standard v2.2.x lanes are skipped above.)
          if [[ "${REF}" == "v2.2.x" ]]; then
            run_regex="${run_regex//^TestListenerSet|/}"
          fi
          # The Gateway API experimental-channel CRDs rely on CEL library functions that
          # Kubernetes only ships starting at specific minor versions: the 1.5+ CRDs use
          # isIP() (added in k8s 1.31), and the 1.6+ CRDs additionally use the `format`
          # library, e.g. format.dns1123Label() (only reliably available from k8s 1.33).
          # Skip experimental-channel lanes whose Kubernetes version predates what the CRDs
          # need, based on the actual version pin rather than a hardcoded ref/label -- a
          # hardcoded list silently goes stale as per-ref version pins drift (this is what
          # let the v2.2.x min lane start failing once its pin moved under a 1.6.1 CRD without
          # anyone updating this guard).
          if [[ "${GW_API_CHANNEL}" == "experimental" ]]; then
            gwapi_minor="${GW_API_VERSION#v}"
            gwapi_minor="${gwapi_minor#*.}"
            gwapi_minor="${gwapi_minor%%.*}"
            kube_minor="${KUBE_VERSION#v}"
            kube_minor="${kube_minor#*.}"
            kube_minor="${kube_minor%%.*}"
            if (( gwapi_minor >= 6 && kube_minor < 33 )); then
              run=false
            elif (( gwapi_minor == 5 && kube_minor < 31 )); then
              run=false
            fi
          fi
          {
            echo "run=${run}"
            echo "bypass_version_check=${bypass_version_check}"
            echo "run_regex=${run_regex}"
          } >> "${GITHUB_OUTPUT}"
          echo "ref=${REF} gw-api=${GW_API_VERSION}-${GW_API_CHANNEL} versions=${VERSIONS_LABEL} k8s=${KUBE_VERSION} -> run=${run} bypass_version_check=${bypass_version_check} run_regex=${run_regex}"
      # Bake KGW_SKIP_GATEWAY_API_VERSION_CHECK into the chart before setup-kind-cluster
      # packages it (hack/kind/setup-kind.sh runs `make package-kgateway-charts` from source),
      # so the controller starts despite the unsupported Gateway API version.
      - name: Bypass Gateway API version check
        if: ${{ steps.guard.outputs.bypass_version_check == 'true' }}
        shell: bash
        run: |
          yq -i '.controller.extraEnv.KGW_SKIP_GATEWAY_API_VERSION_CHECK = "true"' \
            install/helm/kgateway/values.yaml
          echo "Set controller.extraEnv.KGW_SKIP_GATEWAY_API_VERSION_CHECK=true in install/helm/kgateway/values.yaml"
      - name: Prep Go Runner
        uses: ./.github/actions/prep-go-runner
      - name: Setup KinD Cluster
        if: ${{ steps.dotenv.outputs.cluster_type != 'k3d' && steps.guard.outputs.run == 'true' }}
        uses: ./.github/actions/setup-kind-cluster
        with:
          gateway-api-version: ${{ matrix.gateway-api-version.version }}
          gateway-api-channel: ${{ matrix.gateway-api-version.channel }}
          cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}"
          kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
          istio-version: ${{ steps.dotenv.outputs.istio_version }}
          kind-node-version: ${{ steps.dotenv.outputs.node_version }}
          localstack: "true"
      - name: Setup k3d Cluster
        if: ${{ steps.dotenv.outputs.cluster_type == 'k3d' && steps.guard.outputs.run == 'true' }}
        uses: ./.github/actions/setup-k3d-cluster
        with:
          gateway-api-version: ${{ matrix.gateway-api-version.version }}
          gateway-api-channel: ${{ matrix.gateway-api-version.channel }}
          cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}"
          kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
          istio-version: ${{ steps.dotenv.outputs.istio_version }}
          node-version: ${{ steps.dotenv.outputs.node_version }}
          localstack: "true"
          github-token: ${{ github.token }}
      - id: run-tests
        if: ${{ steps.guard.outputs.run == 'true' }}
        uses: ./.github/actions/kubernetes-e2e-tests
        env:
          VERSION: 'v1.0.0-ci1'
          GITHUB_TOKEN: ${{ github.token }}
          GO_TEST_RETRIES: '2' # There are so many tests and we don't want to rerun, but some failures are persistent
          # A single test failure counts as a failure for each of its parent suites, and these tests run for a long time, so use a higher max failure count
          GOTESTSUM_ARGS: '--format=standard-verbose --rerun-fails-max-failures 25'
        with:
          cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}"
          cluster-type: ${{ steps.dotenv.outputs.cluster_type || 'kind' }}
          test-args: '-timeout=120m'
          run-regex: ${{ steps.guard.outputs.run_regex }}
          istio-version: ${{ steps.dotenv.outputs.istio_version }}
          matrix-label: "nightly-kgw-api-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}"
          envtest-k8s-version: ${{ steps.dotenv.outputs.envtest_k8s_version }}