tests, branch:3575/merge, triggered by @tao12345666333 #15604
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| run-name: tests, branch:${{ github.ref_name }}, triggered by @${{ github.actor }} | |
| concurrency: | |
| # Run only for most recent commit in PRs but for all tags and commits on main | |
| # Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| branches: | |
| - 'main' | |
| pull_request: | |
| branches: | |
| - '**' | |
| - 'release/*' | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release/*' | |
| tags: | |
| - '*' | |
| workflow_dispatch: {} | |
| # NOTE: Some jobs add GITHUB_API_TOKEN env so that some mise plugins which | |
| # download GitHub release assets can access GitHub's API authenticated and thus | |
| # not get rate-limited (which causes failures). | |
| # NOTE: Some jobs require GITHUB_TOKEN env which is used by mise itself to access | |
| # GitHub's API authenticated and thus not get rate-limited (which causes failures). | |
| # Ref: https://mise.jdx.dev/getting-started.html#github-api-rate-limiting. | |
| # | |
| # Recent versions of mise started to also accept MISE_GITHUB_TOKEN as an alternative to | |
| # GITHUB_TOKEN. That env var is set by the mise-action automatically. | |
| permissions: | |
| contents: read | |
| attestations: read | |
| env: | |
| MISE_VERBOSE: 1 | |
| jobs: | |
| build-e2e-image: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| env: | |
| IMG: kong-operator | |
| TAG: e2e-${{ github.sha }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: setup golang | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: build docker image | |
| run: make docker.build | |
| - name: save docker image as artifact | |
| run: docker save ${{ env.IMG }}:${{ env.TAG }} -o kong-operator_e2e.tar | |
| - name: upload docker image artifact | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: kong-operator-e2e-image-${{ github.run_id }} | |
| path: kong-operator_e2e.tar | |
| retention-days: 1 | |
| if-no-files-found: error | |
| check-docs-only: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| docs_only: ${{ steps.check-files.outputs.docs_only }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - id: check-files | |
| name: Check if only documentation files changed | |
| run: ./scripts/check-docs-only-changes.sh ${{ github.event.pull_request.base.sha || 'HEAD~1' }} ${{ github.sha }} | |
| ensure-actions-sha-pin: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: zgosalvez/github-actions-ensure-sha-pinned-actions@70c4af2ed5282c51ba40566d026d6647852ffa3e # v5.0.1 | |
| ossf-scorecard: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| # Ref: https://github.com/ossf/scorecard | |
| - run: | | |
| GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ | |
| REPO=github.com/${{ github.repository }} \ | |
| COMMIT=${{ github.sha }} \ | |
| make docker.run.openssf | |
| lint: | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__lint.yaml | |
| secrets: inherit | |
| lint-markdownlint: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| - name: run lint.markdownlint | |
| run: make lint.markdownlint | |
| govulncheck: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup go | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| - run: make govulncheck | |
| verify: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup go | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| - run: echo "GO_MOD_CACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV | |
| - run: echo "GO_BUILD_CACHE=$(go env GOCACHE)" >> $GITHUB_ENV | |
| - run: echo "GO_CACHE_KEY=go-build-${{ runner.os }}-${{ runner.arch }}-go-${{ hashFiles('**/go.mod') }}" >> $GITHUB_ENV | |
| - run: echo "GO_CACHE_KEY_2=go-build-${{ runner.os }}-${{ runner.arch }}-go-" >> $GITHUB_ENV | |
| - name: Go cache (restore only from build cache) | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 | |
| with: | |
| path: | | |
| ${{ env.GO_MOD_CACHE }} | |
| ${{ env.GO_BUILD_CACHE }} | |
| key: ${{ env.GO_CACHE_KEY }} | |
| restore-keys: | | |
| ${{ env.GO_CACHE_KEY }} | |
| ${{ env.GO_CACHE_KEY_2 }} | |
| - name: Verify go fix | |
| run: make verify.go-fix | |
| - name: Verify manifests consistency | |
| run: make verify.manifests | |
| - name: Verify generators consistency | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 | |
| with: | |
| timeout_minutes: 8 | |
| max_attempts: 3 | |
| retry_wait_seconds: 30 | |
| command: make verify.generators | |
| matrix_k8s_node_versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| latest: ${{ steps.set-latest.outputs.latest }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - id: set-matrix | |
| run: | | |
| ( | |
| echo 'matrix<<EOF' | |
| yq eval -o=json '.' .github/supported_k8s_node_versions.yaml | |
| echo 'EOF' | |
| ) >> "${GITHUB_OUTPUT}" | |
| - id: set-latest | |
| run: | | |
| ( | |
| echo 'latest<<EOF' | |
| yq eval -r -o=json '. | sort | reverse | .[0]' .github/supported_k8s_node_versions.yaml | |
| echo 'EOF' | |
| ) >> "${GITHUB_OUTPUT}" | |
| samples: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check-docs-only | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup go | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Create k8s KinD Cluster | |
| uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 | |
| with: | |
| # NOTE: default is 0.29.0 https://github.com/helm/kind-action/blob/a1b0e391336a6ee6713a0583f8c6240d70863de3/kind.sh#L21 | |
| # so bump this manually | |
| version: v0.30.0 | |
| node_image: kindest/node:${{ needs.matrix_k8s_node_versions.outputs.latest }} | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| # We use install.all to install all CRDs and resources also the ones that are not bundled | |
| # in base kustomization (e.g. currently AIGateway) but which have samples defined. | |
| - name: Verify installing CRDs via kustomize works | |
| run: make install.all | |
| - name: Install and delete each sample one by one | |
| run: make test.samples | |
| - name: Verify that uninstalling operator CRDs via kustomize works | |
| run: make ignore-not-found=true uninstall.all | |
| install-with-kustomize: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check-docs-only | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| env: | |
| IMG: kong-operator | |
| TAG: e2e-${{ github.sha }} | |
| CLUSTER_NAME: install-with-kustomize | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup go | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: build docker image | |
| run: make docker.build | |
| - name: Create k8s KinD Cluster | |
| uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 | |
| with: | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| ignore_failed_clean: true | |
| node_image: kindest/node:${{ needs.matrix_k8s_node_versions.outputs.latest }} | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| - name: Verify installing CRDs via kustomize works | |
| run: make install | |
| - name: Load image to kind cluster | |
| run: kind load docker-image kong-operator:e2e-${{ github.sha }} --name $CLUSTER_NAME | |
| - name: Verify deploying operator via kustomize works | |
| env: | |
| IMG: kong-operator | |
| VERSION: e2e-${{ github.sha }} | |
| run: make deploy | |
| - name: Verify that undeploying operator via kustomize works | |
| run: make undeploy | |
| - name: Verify that uninstalling operator CRDs via kustomize works | |
| run: make ignore-not-found=true uninstall | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: setup golang | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - run: echo "GO_MOD_CACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV | |
| - run: echo "GO_BUILD_CACHE=$(go env GOCACHE)" >> $GITHUB_ENV | |
| - run: echo "GO_CACHE_KEY=go-build-${{ runner.os }}-${{ runner.arch }}-go-${{ hashFiles('**/go.mod') }}" >> $GITHUB_ENV | |
| - run: echo "GO_CACHE_KEY_2=go-build-${{ runner.os }}-${{ runner.arch }}-go-" >> $GITHUB_ENV | |
| - name: Go cache main branch | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 | |
| with: | |
| path: | | |
| ${{ env.GO_MOD_CACHE }} | |
| ${{ env.GO_BUILD_CACHE }} | |
| key: ${{ env.GO_CACHE_KEY }} | |
| restore-keys: | | |
| ${{ env.GO_CACHE_KEY }} | |
| ${{ env.GO_CACHE_KEY_2 }} | |
| - name: Go cache (restore only for non-main branches) | |
| if: ${{ github.ref != 'refs/heads/main' }} | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 | |
| with: | |
| path: | | |
| ${{ env.GO_MOD_CACHE }} | |
| ${{ env.GO_BUILD_CACHE }} | |
| key: ${{ env.GO_CACHE_KEY }} | |
| restore-keys: | | |
| ${{ env.GO_CACHE_KEY }} | |
| ${{ env.GO_CACHE_KEY_2 }} | |
| - run: make build.operator | |
| - name: Test the binary by running it with -version flag | |
| run: ./bin/manager -version | ./scripts/verify-version.sh ${{ github.repository }} | |
| unit-tests: | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__unit_tests.yaml | |
| secrets: inherit | |
| CRDs: | |
| needs: | |
| - check-docs-only | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__crdvalidation_tests.yaml | |
| secrets: inherit | |
| with: | |
| cluster_versions: ${{ needs.matrix_k8s_node_versions.outputs.matrix }} | |
| envtest-tests: | |
| needs: | |
| - check-docs-only | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__envtest_tests.yaml | |
| secrets: inherit | |
| with: | |
| cluster_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }} | |
| kongintegration-tests: | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__kongintegration_tests.yaml | |
| secrets: inherit | |
| conformance-tests: | |
| needs: [check-docs-only] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__conformance_tests.yaml | |
| secrets: inherit | |
| integration-tests: | |
| needs: | |
| - check-docs-only | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__integration_tests.yaml | |
| secrets: inherit | |
| with: | |
| cluster_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }} | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-docs-only, build-e2e-image] | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - name: setup golang | |
| uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: download e2e image artifact | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: kong-operator-e2e-image-${{ github.run_id }} | |
| - name: load docker image | |
| run: docker load -i kong-operator_e2e.tar | |
| - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 | |
| with: | |
| install: false | |
| - name: run e2e tests | |
| run: make test.e2e | |
| env: | |
| KONG_TEST_KONG_OPERATOR_IMAGE_LOAD: kong-operator:e2e-${{ github.sha }} | |
| GOTESTSUM_JUNITFILE: "e2e-tests.xml" | |
| - name: upload diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: diagnostics-e2e | |
| path: /tmp/ktf-diag* | |
| if-no-files-found: ignore | |
| - name: collect test report | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: tests-report-e2e | |
| path: e2e-tests.xml | |
| e2e-tests-chainsaw: | |
| needs: | |
| - check-docs-only | |
| - build-e2e-image | |
| - matrix_k8s_node_versions | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| uses: ./.github/workflows/__e2e_chainsaw_tests.yaml | |
| secrets: inherit | |
| with: | |
| node_image_version: ${{ needs.matrix_k8s_node_versions.outputs.latest }} | |
| artifacts_run_id: ${{ github.run_id }} | |
| buildpulse-report: | |
| needs: | |
| - check-docs-only | |
| - unit-tests | |
| - envtest-tests | |
| - integration-tests | |
| - conformance-tests | |
| - e2e-tests | |
| if: ${{ always() && needs.check-docs-only.outputs.docs_only != 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: download tests report | |
| id: download-coverage | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| pattern: tests-report* | |
| path: report | |
| merge-multiple: true | |
| - name: Upload test results to BuildPulse for flaky test detection | |
| if: ${{ !cancelled() }} | |
| uses: buildpulse/buildpulse-action@d4d8e00c645a2e3db0419a43664bbcf868080234 # v0.12.0 | |
| with: | |
| account: 962416 | |
| repository: 477814940 | |
| path: report/*.xml | |
| key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} | |
| secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} | |
| # We need this step to fail the workflow if any of the previous steps failed or were cancelled. | |
| # It allows to use this particular job as a required check for PRs. | |
| # Ref: https://github.com/orgs/community/discussions/26822#discussioncomment-3305794 | |
| passed: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check-docs-only | |
| - ensure-actions-sha-pin | |
| - ossf-scorecard | |
| - lint | |
| - lint-markdownlint | |
| - govulncheck | |
| - verify | |
| - samples | |
| - install-with-kustomize | |
| - build | |
| - build-e2e-image | |
| - CRDs | |
| - unit-tests | |
| - envtest-tests | |
| - kongintegration-tests | |
| - conformance-tests | |
| - integration-tests | |
| - e2e-tests | |
| # TODO: reenable when chainsaw e2e tests are stable | |
| # https://github.com/Kong/kong-operator/issues/2960 | |
| # - e2e-tests-chainsaw | |
| - buildpulse-report | |
| if: always() | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0 | |
| with: | |
| egress-policy: audit | |
| - name: Check if docs-only changes | |
| run: | | |
| if [[ "${{ needs.check-docs-only.outputs.docs_only }}" == "true" ]]; then | |
| echo "Only documentation files were changed, skipping tests" | |
| exit 0 | |
| fi | |
| - name: Check test results | |
| if: ${{ needs.check-docs-only.outputs.docs_only != 'true' }} | |
| run: | | |
| if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then | |
| echo "Some jobs failed or were cancelled." | |
| exit 1 | |
| fi |