Nightly Tests #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # necessary to pass upgrade tests | |
| ENVOYINIT_CACHE_REF: ghcr.io/${{ github.repository_owner }}/envoy-wrapper-cache | |
| CONTROLLER_CACHE_REF: ghcr.io/${{ github.repository_owner }}/kgateway-cache | |
| SDS_CACHE_REF: ghcr.io/${{ github.repository_owner }}/sds-cache | |
| on: | |
| schedule: | |
| - cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against main and supported LTS branches | |
| workflow_dispatch: | |
| inputs: | |
| target_ref: | |
| description: "The Git ref to run tests against. Empty uses the workflow run commit." | |
| type: string | |
| required: false | |
| default: "" | |
| run-conformance: | |
| description: "Run conformance tests" | |
| type: boolean | |
| default: false | |
| run-load-tests: | |
| description: "Run load testing suite" | |
| type: boolean | |
| default: false | |
| run-e2e-tests: | |
| description: "Run e2e tests" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ inputs.target_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| determine_refs: | |
| name: Determine Refs | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| refs: ${{ steps.determine.outputs.refs }} | |
| steps: | |
| - id: determine | |
| name: Determine Refs | |
| shell: bash | |
| env: | |
| INPUT_TARGET_REF: ${{ inputs.target_ref }} | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then | |
| refs='[ | |
| {"checkout_ref":"main","display_ref":"main"}, | |
| {"checkout_ref":"v2.2.x","display_ref":"v2.2.x"}, | |
| {"checkout_ref":"v2.3.x","display_ref":"v2.3.x"} | |
| ]' | |
| else | |
| checkout_ref="${INPUT_TARGET_REF:-}" | |
| display_ref="${INPUT_TARGET_REF:-}" | |
| if [[ -z "${checkout_ref}" ]]; then | |
| checkout_ref="${GITHUB_SHA}" | |
| display_ref="${GITHUB_REF_NAME}@${GITHUB_SHA::7}" | |
| fi | |
| refs="$(jq -cn --arg checkout_ref "${checkout_ref}" --arg display_ref "${display_ref}" \ | |
| '[{checkout_ref:$checkout_ref,display_ref:$display_ref}]')" | |
| fi | |
| refs="$(jq -c . <<< "${refs}")" | |
| echo "refs=${refs}" >> "$GITHUB_OUTPUT" | |
| kube_gateway_api_conformance_tests: | |
| name: Conformance (ref=${{ matrix.ref.display_ref }}, type=Kubernetes Gateway API, k8s=${{ matrix.env-versions.label }}, gw-api=${{matrix.gateway-api-channel}}) | |
| needs: determine_refs | |
| if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance ) || github.event.schedule == '0 5 * * *' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ref: ${{ fromJson(needs.determine_refs.outputs.refs) }} | |
| env-versions: | |
| - file: "./.github/workflows/.env/nightly-tests/max_versions.env" | |
| label: max_versions | |
| - file: "./.github/workflows/.env/nightly-tests/min_versions.env" | |
| label: min_versions | |
| gateway-api-channel: ['experimental', 'standard'] | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ matrix.ref.checkout_ref }} | |
| - name: Dotenv Action | |
| uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4 | |
| id: dotenv | |
| with: | |
| path: ${{ matrix.env-versions.file }} | |
| log-variables: true | |
| - uses: ./.github/actions/kube-conformance-tests | |
| with: | |
| test-type: gateway-api | |
| api-channel: ${{ matrix.gateway-api-channel }} | |
| kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} | |
| kind-node-version: ${{ steps.dotenv.outputs.node_version }} | |
| summarize-report: true | |
| kube_gateway_api_load_tests: | |
| name: Load Tests (ref=${{ matrix.ref.display_ref }}, type=Kubernetes Gateway API, k8s=${{ matrix.env-versions.label }}) | |
| needs: determine_refs | |
| if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-load-tests ) || github.event.schedule == '0 5 * * *' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ref: ${{ fromJson(needs.determine_refs.outputs.refs) }} | |
| env-versions: | |
| - file: "./.github/workflows/.env/nightly-tests/max_versions.env" | |
| label: max_versions | |
| - file: "./.github/workflows/.env/nightly-tests/min_versions.env" | |
| label: min_versions | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ matrix.ref.checkout_ref }} | |
| - name: Dotenv Action | |
| uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4 | |
| id: dotenv | |
| with: | |
| path: ${{ matrix.env-versions.file }} | |
| log-variables: true | |
| - uses: ./.github/actions/kube-gateway-api-load-tests | |
| with: | |
| cluster-node-version: ${{ steps.dotenv.outputs.node_version }} | |
| kgateway_e2e_tests_for_gateway_api_versions: | |
| name: E2E (ref=${{ matrix.ref.display_ref }}, k8s=${{ matrix.env-versions.label }}, gw-api=${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}) | |
| needs: determine_refs | |
| if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-e2e-tests ) || github.event.schedule == '0 5 * * *' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ref: ${{ fromJson(needs.determine_refs.outputs.refs) }} | |
| env-versions: | |
| - file: "./.github/workflows/.env/nightly-tests/max_versions.env" | |
| label: max | |
| - file: "./.github/workflows/.env/nightly-tests/min_versions.env" | |
| label: min | |
| gateway-api-version: | |
| - version: 'v1.6.1' | |
| channel: experimental | |
| - version: 'v1.6.1' | |
| channel: standard | |
| - version: 'v1.5.1' | |
| channel: experimental | |
| - version: 'v1.5.1' | |
| channel: standard | |
| - version: 'v1.4.1' | |
| channel: experimental | |
| - version: 'v1.4.1' | |
| channel: standard | |
| - version: 'v1.3.0' | |
| channel: experimental | |
| - version: 'v1.3.0' | |
| channel: standard | |
| controllers: | |
| - regex: "^TestKgateway|^TestAPIValidation|^TestListenerSet|^TestCustomGWP$$|^TestRouteReplacement$$|^TestZeroDowntimeRollout$$/^TestZeroDowntimeRollout$$|^TestControlPlaneTLS$$" | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: ${{ matrix.ref.checkout_ref }} | |
| # Runs before the guard step below so the guard can key its Kubernetes-version check off | |
| # the real kubectl_version output instead of guessing from the min/max label. | |
| - name: Dotenv Action | |
| uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 # v1.1.4 | |
| id: dotenv | |
| with: | |
| path: ${{ matrix.env-versions.file }} | |
| log-variables: true | |
| # v2.2.x is built against Gateway API 1.4 and refuses to start on 1.5+ (#13872), so the | |
| # v2.2.x x v1.5+ lanes need special handling: skip the standard channel entirely, and run | |
| # the experimental channel with the version check bypassed to keep forward-compat signal. | |
| # This is done with step guards rather than a matrix `exclude:` because every dimension | |
| # here is object-valued and object-valued excludes are unreliable (actions/runner#1512). | |
| - name: Determine run plan | |
| id: guard | |
| env: | |
| REF: ${{ matrix.ref.checkout_ref }} | |
| GW_API_VERSION: ${{ matrix.gateway-api-version.version }} | |
| GW_API_CHANNEL: ${{ matrix.gateway-api-version.channel }} | |
| VERSIONS_LABEL: ${{ matrix.env-versions.label }} | |
| KUBE_VERSION: ${{ steps.dotenv.outputs.kubectl_version }} | |
| BASE_REGEX: ${{ matrix.controllers.regex }} | |
| shell: bash | |
| run: | | |
| run=true | |
| bypass_version_check=false | |
| run_regex="${BASE_REGEX}" | |
| if [[ "${REF}" == "v2.2.x" && "${GW_API_VERSION}" == "v1.6.1" ]]; then | |
| case "${GW_API_CHANNEL}" in | |
| standard) run=false ;; | |
| experimental) bypass_version_check=true ;; | |
| esac | |
| fi | |
| if [[ "${REF}" == "v2.2.x" && "${GW_API_VERSION}" == "v1.5.1" ]]; then | |
| case "${GW_API_CHANNEL}" in | |
| standard) run=false ;; | |
| experimental) bypass_version_check=true ;; | |
| esac | |
| fi | |
| # v2.2.x's chart predates ListenerSet support, so TestListenerSet fails at apply time | |
| # ("the server could not find the requested resource") even with the version check | |
| # bypassed. Drop it from the regex on v2.2.x while keeping the rest of the | |
| # forward-compat signal. (ListenerSet only runs on the experimental channel; the | |
| # standard v2.2.x lanes are skipped above.) | |
| if [[ "${REF}" == "v2.2.x" ]]; then | |
| run_regex="${run_regex//^TestListenerSet|/}" | |
| fi | |
| # The Gateway API experimental-channel CRDs rely on CEL library functions that | |
| # Kubernetes only ships starting at specific minor versions: the 1.5+ CRDs use | |
| # isIP() (added in k8s 1.31), and the 1.6+ CRDs additionally use the `format` | |
| # library, e.g. format.dns1123Label() (only reliably available from k8s 1.33). | |
| # Skip experimental-channel lanes whose Kubernetes version predates what the CRDs | |
| # need, based on the actual version pin rather than a hardcoded ref/label -- a | |
| # hardcoded list silently goes stale as per-ref version pins drift (this is what | |
| # let the v2.2.x min lane start failing once its pin moved under a 1.6.1 CRD without | |
| # anyone updating this guard). | |
| if [[ "${GW_API_CHANNEL}" == "experimental" ]]; then | |
| gwapi_minor="${GW_API_VERSION#v}" | |
| gwapi_minor="${gwapi_minor#*.}" | |
| gwapi_minor="${gwapi_minor%%.*}" | |
| kube_minor="${KUBE_VERSION#v}" | |
| kube_minor="${kube_minor#*.}" | |
| kube_minor="${kube_minor%%.*}" | |
| if (( gwapi_minor >= 6 && kube_minor < 33 )); then | |
| run=false | |
| elif (( gwapi_minor == 5 && kube_minor < 31 )); then | |
| run=false | |
| fi | |
| fi | |
| { | |
| echo "run=${run}" | |
| echo "bypass_version_check=${bypass_version_check}" | |
| echo "run_regex=${run_regex}" | |
| } >> "${GITHUB_OUTPUT}" | |
| echo "ref=${REF} gw-api=${GW_API_VERSION}-${GW_API_CHANNEL} versions=${VERSIONS_LABEL} k8s=${KUBE_VERSION} -> run=${run} bypass_version_check=${bypass_version_check} run_regex=${run_regex}" | |
| # Bake KGW_SKIP_GATEWAY_API_VERSION_CHECK into the chart before setup-kind-cluster | |
| # packages it (hack/kind/setup-kind.sh runs `make package-kgateway-charts` from source), | |
| # so the controller starts despite the unsupported Gateway API version. | |
| - name: Bypass Gateway API version check | |
| if: ${{ steps.guard.outputs.bypass_version_check == 'true' }} | |
| shell: bash | |
| run: | | |
| yq -i '.controller.extraEnv.KGW_SKIP_GATEWAY_API_VERSION_CHECK = "true"' \ | |
| install/helm/kgateway/values.yaml | |
| echo "Set controller.extraEnv.KGW_SKIP_GATEWAY_API_VERSION_CHECK=true in install/helm/kgateway/values.yaml" | |
| - name: Prep Go Runner | |
| uses: ./.github/actions/prep-go-runner | |
| - name: Setup KinD Cluster | |
| if: ${{ steps.dotenv.outputs.cluster_type != 'k3d' && steps.guard.outputs.run == 'true' }} | |
| uses: ./.github/actions/setup-kind-cluster | |
| with: | |
| gateway-api-version: ${{ matrix.gateway-api-version.version }} | |
| gateway-api-channel: ${{ matrix.gateway-api-version.channel }} | |
| cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}" | |
| kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} | |
| istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| kind-node-version: ${{ steps.dotenv.outputs.node_version }} | |
| localstack: "true" | |
| - name: Setup k3d Cluster | |
| if: ${{ steps.dotenv.outputs.cluster_type == 'k3d' && steps.guard.outputs.run == 'true' }} | |
| uses: ./.github/actions/setup-k3d-cluster | |
| with: | |
| gateway-api-version: ${{ matrix.gateway-api-version.version }} | |
| gateway-api-channel: ${{ matrix.gateway-api-version.channel }} | |
| cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}" | |
| kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} | |
| istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| node-version: ${{ steps.dotenv.outputs.node_version }} | |
| localstack: "true" | |
| github-token: ${{ github.token }} | |
| - id: run-tests | |
| if: ${{ steps.guard.outputs.run == 'true' }} | |
| uses: ./.github/actions/kubernetes-e2e-tests | |
| env: | |
| VERSION: 'v1.0.0-ci1' | |
| GITHUB_TOKEN: ${{ github.token }} | |
| GO_TEST_RETRIES: '2' # There are so many tests and we don't want to rerun, but some failures are persistent | |
| # A single test failure counts as a failure for each of its parent suites, and these tests run for a long time, so use a higher max failure count | |
| GOTESTSUM_ARGS: '--format=standard-verbose --rerun-fails-max-failures 25' | |
| with: | |
| cluster-name: "kgw-api-e2e-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}" | |
| cluster-type: ${{ steps.dotenv.outputs.cluster_type || 'kind' }} | |
| test-args: '-timeout=120m' | |
| run-regex: ${{ steps.guard.outputs.run_regex }} | |
| istio-version: ${{ steps.dotenv.outputs.istio_version }} | |
| matrix-label: "nightly-kgw-api-${{ matrix.env-versions.label }}-${{ matrix.gateway-api-version.version }}-${{ matrix.gateway-api-version.channel }}" | |
| envtest-k8s-version: ${{ steps.dotenv.outputs.envtest_k8s_version }} |