BATS #554
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BATS | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| owner: | |
| description: Override owner (e.g. rancher-sandbox) | |
| type: string | |
| repo: | |
| description: Override repository (e.g. rancher-desktop) | |
| type: string | |
| branch: | |
| description: Override branch (e.g. main, or PR#) | |
| type: string | |
| tests: | |
| description: 'Tests (in the tests/ directory, e.g. "containers")' | |
| default: '*' | |
| type: string | |
| platforms: | |
| description: Platforms to run | |
| default: 'linux mac win' | |
| type: string | |
| engines: | |
| description: Container engines to run | |
| default: 'containerd moby' | |
| type: string | |
| kubernetes-version: | |
| description: Primary Kubernetes version to test | |
| default: '1.22.7' # Must also change in calculate step | |
| type: string | |
| kubernetes-alt-version: | |
| description: Secondary Kubernetes version to test (e.g. for upgrades) | |
| default: '1.28.11' # Must also change in calculate step | |
| type: string | |
| package-id: | |
| description: Package run ID override; leave empty to use latest. | |
| default: '' | |
| type: string | |
| experimental: | |
| description: Run with experimental settings (WSL) | |
| default: false | |
| type: boolean | |
| schedule: | |
| - cron: '0 8 * * 1-5' # 8AM UTC weekdays as a baseline | |
| permissions: | |
| contents: read | |
| env: | |
| GH_OWNER: ${{ github.repository_owner }} | |
| GH_REPOSITORY: ${{ github.repository }} | |
| GH_REF_NAME: ${{ github.ref_name }} | |
| jobs: | |
| get-tests: | |
| name: Calculate tests to run | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch install script | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| scripts/install-latest-ci.sh | |
| .github/workflows/bats/get-tests.py | |
| - id: repo | |
| name: Calculate short repository | |
| run: echo "repo=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_OUTPUT" | |
| - name: Fetch tests | |
| run: | | |
| : ${OWNER:=$GH_OWNER} | |
| : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}} | |
| : ${BRANCH:=$GH_REF_NAME} | |
| # If BRANCH is a number, assume it is supposed to be a PR | |
| [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH | |
| "scripts/install-latest-ci.sh" | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ inputs.owner || github.repository_owner }} | |
| REPO: ${{ inputs.repo || steps.repo.outputs.repo }} | |
| BRANCH: ${{ inputs.branch || github.ref_name }} | |
| ID: ${{ inputs.package-id }} | |
| BATS_DIR: ${{ github.workspace }}/bats | |
| INSTALL_MODE: skip | |
| - name: Calculate tests | |
| id: calculate | |
| # This script is not inline to make local testing easier | |
| run: python3 ${{ github.workspace }}/.github/workflows/bats/get-tests.py | |
| env: | |
| TESTS: ${{ inputs.tests }} | |
| PLATFORMS: ${{ inputs.platforms }} | |
| ENGINES: ${{ inputs.engines }} | |
| KUBERNETES_VERSION: ${{ inputs.kubernetes-version || '1.22.7' }} | |
| # rancher/rancher helm chart 2.8.5 supports up to 1.28.* | |
| KUBERNETES_ALT_VERSION: ${{ inputs.kubernetes-alt-version || '1.28.11' }} | |
| working-directory: bats/tests | |
| outputs: | |
| repo: ${{ steps.repo.outputs.repo }} | |
| tests: ${{ steps.calculate.outputs.tests }} | |
| bats: | |
| needs: get-tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.get-tests.outputs.tests )}} | |
| runs-on: ${{ matrix.host }} | |
| steps: | |
| - name: Fetch install script | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| scripts/install-latest-ci.sh | |
| .github/actions/setup-environment/action.yaml | |
| .github/workflows/bats/sanitize-artifact-name.sh | |
| - name: Install latest CI build | |
| run: | | |
| : ${OWNER:=$GH_OWNER} | |
| : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}} | |
| : ${BRANCH:=$GH_REF_NAME} | |
| # If BRANCH is a number, assume it is supposed to be a PR | |
| [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH | |
| scripts/install-latest-ci.sh | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ inputs.owner || github.repository_owner }} | |
| REPO: ${{ inputs.repo || needs.get-tests.outputs.repo }} | |
| BRANCH: ${{ inputs.branch || github.ref_name }} | |
| ID: ${{ inputs.package-id }} | |
| BATS_DIR: ${{ github.workspace }}/bats | |
| INSTALL_MODE: installer | |
| ZIP_NAME: ${{ github.workspace }}/version.txt | |
| RD_LOCATION: system | |
| - name: Set up environment | |
| uses: ./.github/actions/setup-environment | |
| - name: "Linux: Install prerequisites" | |
| if: runner.os == 'Linux' | |
| run: >- | |
| sudo DEBIAN_FRONTEND=noninteractive | |
| apt-get install coreutils | |
| - name: "macOS: Install prerequisites" | |
| if: runner.os == 'macOS' | |
| shell: bash | |
| run: brew install --force bash coreutils | |
| - name: "Windows: Install WSL2 Distribution" | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| # Install a "modern" WSL distribution | |
| wsl --install --no-launch --web-download --name openSUSE openSUSE-Leap-15.6 | |
| # Prevent first-boot from running | |
| wsl --distribution openSUSE --user root --exec sed -i -e '/^command/d' /etc/wsl-distribution.conf | |
| # Create the initial user | |
| wsl --distribution openSUSE --user root --exec /usr/sbin/useradd --create-home --uid 1000 user | |
| - name: "Windows: Install prerequisites in WSL" | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: >- | |
| wsl.exe --distribution openSUSE --user root --exec | |
| zypper --non-interactive install | |
| curl | |
| util-linux | |
| - name: "Windows: Enable experimental WSL settings" | |
| if: runner.os == 'Windows' && inputs.experimental | |
| shell: pwsh | |
| run: | | |
| Set-Content -Encoding UTF8NoBOM -Path "${HOME}/.wslconfig" -Value @" | |
| ; Note that not all settings here make sense together. | |
| [wsl] | |
| dnsProxy=false | |
| ; networkingMode=mirrored ; https://github.com/rancher-sandbox/rancher-desktop/issues/6665 | |
| firewall=true | |
| dnsTunneling=true | |
| autoProxy=true | |
| [experimental] | |
| autoMemoryReclaim=gradual | |
| sparseVhd=true | |
| useWindowsDnsCache=true | |
| bestEffortDnsParsing=true | |
| hostAddressLoopback=true | |
| "@ | |
| - name: Set log directory | |
| shell: bash | |
| run: | | |
| echo "LOGS_DIR=$(pwd)/logs" >> "$GITHUB_ENV" | |
| mkdir logs | |
| - name: "Windows: Override log directory" | |
| if: runner.os == 'Windows' | |
| shell: powershell | |
| run: >- | |
| wsl.exe --distribution openSUSE -- echo 'LOGS_DIR=$(pwd)' | |
| | Out-File -Encoding ASCII -Append "$ENV:GITHUB_ENV" | |
| working-directory: logs | |
| - name: Normalize test name | |
| id: normalize | |
| shell: bash | |
| run: | | |
| t="${{ matrix.name }}" | |
| if [[ ! -r "tests/$t" ]] && [[ -r "tests/${t}.bats" ]]; then | |
| t="${t}.bats" | |
| fi | |
| echo "test=$t" >> "$GITHUB_OUTPUT" | |
| working-directory: bats | |
| - name: "macOS: Set startup command" | |
| if: runner.os == 'macOS' | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: exec | |
| - name: "Linux: Set startup command" | |
| if: runner.os == 'Linux' | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: >- | |
| exec xvfb-run --auto-servernum | |
| --server-args='-screen 0 1280x960x24' | |
| - name: "Windows: Set startup command" | |
| if: runner.os == 'Windows' | |
| shell: bash | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: wsl.exe --distribution openSUSE --exec | |
| - name: Run BATS | |
| # We use ${{ env.BATS_COMMAND }} instead of ${BATS_COMMAND} to let the | |
| # shell parse the command, instead of doing it via expansion which is then | |
| # parsed differently (--server-args isn't kept as one word). Also, we | |
| # need to use the env.* form because PowerShell uses ${ENV:VAR} instead. | |
| run: >- | |
| ${{ env.BATS_COMMAND }} | |
| ./bats-core/bin/bats | |
| --gather-test-outputs-in '${{ env.LOGS_DIR }}' | |
| --print-output-on-failure | |
| --filter-tags '!ci-skip' | |
| --formatter cat | |
| --report-formatter tap | |
| 'tests/${{ steps.normalize.outputs.test }}' | |
| env: | |
| BATS_COMMAND: ${{ env.BATS_COMMAND }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| LOGS_DIR: ${{ env.LOGS_DIR }} | |
| RD_CAPTURE_LOGS: "true" | |
| RD_CONTAINER_ENGINE: ${{ matrix.engine }} | |
| RD_KUBERNETES_VERSION: ${{ matrix.k3sVersion }} | |
| RD_KUBERNETES_ALT_VERSION: ${{ matrix.k3sAltVersion }} | |
| RD_TAKE_SCREENSHOTS: "true" | |
| RD_TRACE: "true" | |
| RD_USE_GHCR_IMAGES: "true" | |
| RD_USE_RAMDISK: "true" | |
| RD_USE_WINDOWS_EXE: "${{ runner.os == 'Windows' }}" | |
| WSLENV: "\ | |
| GITHUB_TOKEN:\ | |
| RD_CAPTURE_LOGS:\ | |
| RD_CONTAINER_ENGINE:\ | |
| RD_KUBERNETES_VERSION:\ | |
| RD_KUBERNETES_ALT_VERSION:\ | |
| RD_TAKE_SCREENSHOTS:\ | |
| RD_TRACE:\ | |
| RD_USE_GHCR_IMAGES:\ | |
| RD_USE_RAMDISK:\ | |
| RD_USE_WINDOWS_EXE:\ | |
| " | |
| working-directory: bats | |
| timeout-minutes: 120 | |
| - name: Calculate log name | |
| id: log_name | |
| if: ${{ !cancelled() }} | |
| run: | | |
| name="$(.github/workflows/bats/sanitize-artifact-name.sh <<< "$name")" | |
| # For the artifact name, backslash and forward slash are also invalid. | |
| name=${name//\\/%3C} | |
| name=${name//\//%2F} | |
| echo "name=$name" >>"$GITHUB_OUTPUT" | |
| shell: bash | |
| env: | |
| name: ${{ matrix.host }}-${{ matrix.engine }}-${{ matrix.name }}.logs | |
| - name: Consolidate logs | |
| if: ${{ !cancelled() }} | |
| run: | | |
| # bats/logs may not exist if the workflow is being tested with e.g. tests/helpers/utils.bats | |
| if [ -d "bats/logs" ]; then | |
| cp -R "bats/logs/" logs | |
| fi | |
| cp "bats/report.tap" logs | |
| .github/workflows/bats/sanitize-artifact-name.sh logs | |
| echo "$NAME" > logs/name.txt | |
| echo "$OS" > logs/os.txt | |
| echo "$ENGINE" > logs/engine.txt | |
| echo "$LOG_NAME" > logs/log-name.txt | |
| mv version.txt logs/ | |
| shell: bash | |
| env: | |
| NAME: ${{ matrix.name }} | |
| OS: ${{ matrix.host }} | |
| ENGINE: ${{ matrix.engine }} | |
| LOG_NAME: ${{ steps.log_name.outputs.name }} | |
| - name: Upload logs | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: ${{ steps.log_name.outputs.name }} | |
| path: logs/ | |
| if-no-files-found: error | |
| summarize: | |
| name: Summarize output | |
| needs: [ get-tests, bats ] | |
| if: ${{ !cancelled() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch summarizer script | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| package.json | |
| .github/workflows/bats/summarize.mjs | |
| - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: package.json | |
| - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| pattern: "*.logs" | |
| - run: node .github/workflows/bats/summarize.mjs | |
| env: | |
| EXPECTED_TESTS: ${{ needs.get-tests.outputs.tests }} |