[cov,ci] Collect test coverage on CI #18833
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright lowRISC contributors (OpenTitan project). | |
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # FIXME: Copy this file as coverage.yml and restore ci.yml | |
| name: Coverage | |
| on: | |
| # FIXME: Remove pull request trigger | |
| pull_request: | |
| push: | |
| branches-ignore: | |
| - "backport-*" | |
| tags: | |
| - "*" | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: 'Branch to run the coverage tests' | |
| required: true | |
| type: string | |
| quick: | |
| description: 'Whether to run quick tests only' | |
| default: 'true' | |
| type: string | |
| workflow_call: | |
| inputs: | |
| quick: | |
| description: 'Whether to run quick tests only' | |
| default: 'true' | |
| type: string | |
| report-artifact-name: | |
| description: 'Name of the artifact to upload' | |
| default: 'coverage-report' | |
| type: string | |
| permissions: | |
| contents: read | |
| # Needed for workload identity federation | |
| id-token: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| VIVADO_VERSION: "2021.1" | |
| # Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases | |
| TOOLCHAIN_VERSION: 20220210-1 | |
| jobs: | |
| # FIXME: Remove lint jobs | |
| quick_lint: | |
| name: Lint (quick) | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Required so we can lint commit messages. | |
| - name: Prepare environment | |
| uses: ./.github/actions/prepare-env | |
| with: | |
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | |
| - name: Show environment | |
| run: ./ci/scripts/show-env.sh | |
| - name: Commit metadata | |
| run: ./ci/scripts/lint-commits.sh "$GITHUB_BASE_REF" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| - name: License headers | |
| run: ./ci/scripts/check-licence-headers.sh "$GITHUB_BASE_REF" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| - name: Executable bits | |
| run: ./ci/scripts/exec-check.sh | |
| - name: Non-ASCII characters | |
| run: ./ci/scripts/check-ascii.sh | |
| - name: Python (flake8) | |
| run: ./ci/scripts/python-lint.sh "$GITHUB_BASE_REF" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| - name: Python (mypy) | |
| run: ./ci/scripts/mypy.sh | |
| - name: Validate testplans with schema | |
| run: ./ci/scripts/validate_testplans.sh | |
| - name: C/C++ formatting | |
| run: ./bazelisk.sh test //quality:clang_format_check | |
| - name: Rust formatting | |
| run: ./bazelisk.sh test //quality:rustfmt_check | |
| - name: Shellcheck | |
| run: ./bazelisk.sh test //quality:shellcheck_check | |
| - name: ASM instrumentation | |
| run: ./util/coverage/asm/run_instrument.sh --check | |
| - name: Header guards | |
| run: ./ci/scripts/include-guard.sh "$GITHUB_BASE_REF" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| - name: Trailing whitespace | |
| run: ./ci/scripts/whitespace.sh "$GITHUB_BASE_REF" | |
| if: ${{ github.event_name == 'pull_request' }} | |
| - name: Broken links | |
| run: ./ci/scripts/check-links.sh | |
| - name: Generated documentation | |
| run: ./ci/scripts/check-cmdgen.sh | |
| slow_lint: | |
| name: Lint (slow) | |
| runs-on: ubuntu-22.04 | |
| needs: quick_lint | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Bitstream cache requires all commits. | |
| - name: Prepare environment | |
| uses: ./.github/actions/prepare-env | |
| with: | |
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | |
| - name: Countermeasures implemented (earlgrey) | |
| run: ./ci/scripts/check-countermeasures.sh earlgrey | |
| continue-on-error: true | |
| - name: Countermeasures implemented (englishbreakfast) | |
| run: ./ci/scripts/check-countermeasures.sh englishbreakfast | |
| continue-on-error: true | |
| - name: Bazel test suite tags | |
| run: ./ci/scripts/check_bazel_test_suites.py | |
| continue-on-error: true | |
| # See #21973: disabled until Verilator tags are fixed. | |
| # - name: Check Bazel tags | |
| # run: ./ci/scripts/check-bazel-tags.sh | |
| # continue-on-error: true | |
| - name: Banned Bazel rules | |
| run: ./ci/scripts/check-bazel-banned-rules.sh | |
| - name: Bazel target names | |
| run: ./ci/scripts/check_bazel_target_names.py | |
| continue-on-error: true | |
| - name: DV software images | |
| run: ./ci/scripts/check_dv_sw_images.sh | |
| continue-on-error: true | |
| - name: Generated files | |
| run: ./ci/scripts/check-generated.sh | |
| env: | |
| OT_DESTRUCTIVE: 1 # Required by the script to clean up. | |
| - name: Buildifier | |
| run: ./bazelisk.sh test //quality:buildifier_check | |
| - name: Vendored files | |
| run: ./ci/scripts/check-vendoring.sh | |
| - name: Verible RTL | |
| run: ./ci/scripts/verible-lint.sh rtl | |
| - name: Verible DV | |
| run: ./ci/scripts/verible-lint.sh dv | |
| - name: Verible FPV | |
| run: ./ci/scripts/verible-lint.sh fpv | |
| otbn_crypto_tests_coverage: | |
| name: OTBN Crypto Tests Coverage | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Bitstream cache requires all commits. | |
| - name: Prepare environment | |
| uses: ./.github/actions/prepare-env | |
| with: | |
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | |
| - name: Execute tests | |
| run: | | |
| ./bazelisk.sh coverage --config=ot_coverage --test_tag_filters=-nightly \ | |
| //sw/otbn/crypto/... | |
| - name: Publish Bazel test results | |
| uses: ./.github/actions/publish-bazel-test-results | |
| if: ${{ !cancelled() }} | |
| with: | |
| artifact-name: otbn-test-results | |
| chip_earlgrey_cw310: | |
| name: Earl Grey for CW310 | |
| if: ${{ inputs.quick != 'true' }} | |
| uses: ./.github/workflows/bitstream.yml | |
| secrets: inherit | |
| with: | |
| top_name: earlgrey | |
| design_suffix: cw310 | |
| chip_earlgrey_cw310_hyperdebug: | |
| name: Earl Grey for CW310 Hyperdebug | |
| if: ${{ inputs.quick != 'true' }} | |
| uses: ./.github/workflows/bitstream.yml | |
| secrets: inherit | |
| with: | |
| top_name: earlgrey | |
| design_suffix: cw310_hyperdebug | |
| chip_earlgrey_cw340: | |
| name: Earl Grey for CW340 | |
| if: ${{ inputs.quick != 'true' }} | |
| uses: ./.github/workflows/bitstream.yml | |
| secrets: inherit | |
| with: | |
| top_name: earlgrey | |
| design_suffix: cw340 | |
| # CW310 FPGA jobs. | |
| execute_test_rom_fpga_tests_cw310_coverage: | |
| name: CW310 Test ROM Tests Coverage | |
| needs: chip_earlgrey_cw310 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_test_rom_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310 | |
| board: cw310 | |
| interface: cw310 | |
| tag_filters: cw310_test_rom | |
| mode: coverage | |
| execute_rom_fpga_tests_cw310_coverage: | |
| name: CW310 ROM Tests Coverage | |
| needs: chip_earlgrey_cw310 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_rom_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310 | |
| board: cw310 | |
| interface: cw310 | |
| tag_filters: "cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf" | |
| timeout: 90 | |
| mode: coverage | |
| execute_rom_ext_fpga_tests_cw310_coverage: | |
| name: CW310 ROM_EXT Tests Coverage | |
| needs: chip_earlgrey_cw310 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_rom_ext_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310 | |
| board: cw310 | |
| interface: cw310 | |
| tag_filters: cw310_rom_ext | |
| mode: coverage | |
| execute_sival_fpga_tests_cw310_coverage: | |
| name: CW310 SiVal Tests Coverage | |
| needs: chip_earlgrey_cw310_hyperdebug | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_sival_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310_hyperdebug | |
| board: cw310 | |
| interface: hyper310 | |
| tag_filters: "cw310_sival,-manuf" | |
| mode: coverage | |
| execute_sival_rom_ext_fpga_tests_cw310_coverage: | |
| name: CW310 SiVal ROM_EXT Tests Coverage | |
| needs: chip_earlgrey_cw310_hyperdebug | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_sival_rom_ext_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310_hyperdebug | |
| board: cw310 | |
| interface: hyper310 | |
| tag_filters: cw310_sival_rom_ext | |
| mode: coverage | |
| execute_manuf_fpga_tests_cw310_coverage: | |
| name: CW310 Manufacturing Tests Coverage | |
| needs: chip_earlgrey_cw310_hyperdebug | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_manuf_fpga_tests_cw310 | |
| bitstream: chip_earlgrey_cw310_hyperdebug | |
| board: cw310 | |
| interface: hyper310 | |
| tag_filters: "manuf,-cw340,-qemu" | |
| mode: coverage | |
| execute_fpga_rom_ext_tests_hyper310_coverage: | |
| name: Hyper310 ROM_EXT Tests Coverage | |
| needs: chip_earlgrey_cw310_hyperdebug | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_fpga_rom_ext_tests_hyper310 | |
| bitstream: chip_earlgrey_cw310_hyperdebug | |
| board: cw310 | |
| interface: hyper310 | |
| tag_filters: hyper310_rom_ext | |
| mode: coverage | |
| # CW340 FPGA jobs | |
| execute_test_rom_fpga_tests_cw340_coverage: | |
| name: CW340 Test ROM Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_test_rom_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| tag_filters: cw340_test_rom | |
| mode: coverage | |
| execute_rom_fpga_tests_cw340_coverage: | |
| name: CW340 ROM Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_rom_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| tag_filters: "cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf" | |
| mode: coverage | |
| execute_rom_ext_fpga_tests_cw340_coverage: | |
| name: CW340 ROM_EXT Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_rom_ext_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| tag_filters: cw340_rom_ext | |
| mode: coverage | |
| execute_sival_fpga_tests_cw340_coverage: | |
| name: CW340 SiVal Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_sival_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| tag_filters: "cw340_sival,-manuf" | |
| mode: coverage | |
| execute_sival_rom_ext_fpga_tests_cw340_coverage: | |
| name: CW340 SiVal ROM_EXT Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_sival_rom_ext_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| # FIXME: enable pentest tests after buildable | |
| tag_filters: "cw340_sival_rom_ext,-pentest" | |
| mode: coverage | |
| execute_manuf_fpga_tests_cw340_coverage: | |
| name: CW340 Manufacturing Tests Coverage | |
| needs: chip_earlgrey_cw340 | |
| uses: ./.github/workflows/fpga.yml | |
| secrets: inherit | |
| with: | |
| job_name: execute_manuf_fpga_tests_cw340 | |
| bitstream: chip_earlgrey_cw340 | |
| board: cw340 | |
| interface: cw340 | |
| tag_filters: "manuf,-hyper310,-qemu" | |
| mode: coverage | |
| sw_build_test_coverage: | |
| name: Unit Test Coverage | |
| runs-on: ubuntu-22.04-vivado | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Required for bitstream cache to work. | |
| - name: Prepare environment | |
| uses: ./.github/actions/prepare-env | |
| with: | |
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | |
| - name: Check Bazel build graph | |
| run: | | |
| # Test the graph with both an empty and filled bitstream cache. | |
| ./ci/scripts/test-empty-bitstream-cache.sh | |
| ./bazelisk.sh build --nobuild //... | |
| - name: Select software targets | |
| run: | | |
| target_pattern_file="$(mktemp)" | |
| echo "target_pattern_file=${target_pattern_file}" >> "$GITHUB_ENV" | |
| # Start with building the whole graph. | |
| echo '//sw/...' > "$target_pattern_file" | |
| # Exclude some targets: | |
| # | |
| # 1. `//hw/...` is out of scope. | |
| # 2. `//quality/...` is tested by the lint jobs. | |
| # 3. `//sw/otbn/crypto/...` is tested by the OTBN job. | |
| # 4. `//third_party/...` which is not our code. | |
| # 5. `//sw/host/...` FIXME: rust coverage are broken on CI. | |
| printf "%s\n" \ | |
| "-//hw/..." \ | |
| "-//quality/..." \ | |
| "-//sw/otbn/crypto/..." \ | |
| "-//third_party/..." \ | |
| "-//sw/host/..." \ | |
| >> "$target_pattern_file" | |
| # Exclude anything that requires a bitstream splice. | |
| ./bazelisk.sh cquery \ | |
| --noinclude_aspects \ | |
| --output=starlark \ | |
| --starlark:expr='"-{}".format(target.label)' \ | |
| --define DISABLE_VERILATOR_BUILD=true \ | |
| -- "rdeps(//..., kind(bitstream_splice, //...))" \ | |
| >> "$target_pattern_file" | |
| - name: Run software unit tests coverage | |
| run: | | |
| ./bazelisk.sh coverage \ | |
| --build_tests_only \ | |
| --keep_going \ | |
| --config=ot_coverage \ | |
| --test_output=errors \ | |
| --define DISABLE_VERILATOR_BUILD=true \ | |
| --target_pattern_file="$target_pattern_file" \ | |
| --test_tag_filters=-broken,-coverage_broken,-cw310,-verilator,-dv,-silicon,-qemu | |
| - name: Publish Bazel test results | |
| uses: ./.github/actions/publish-bazel-test-results | |
| if: ${{ !cancelled() }} | |
| with: | |
| artifact-name: sw_build_test-test-results | |
| - name: Check for unrunnable tests | |
| run: ./ci/scripts/check-unrunnable-tests.sh | |
| continue-on-error: true | |
| publish_coverage_report: | |
| name: Publish Coverage Report | |
| needs: | |
| - otbn_crypto_tests_coverage | |
| - execute_test_rom_fpga_tests_cw310_coverage | |
| - execute_rom_fpga_tests_cw310_coverage | |
| - execute_rom_ext_fpga_tests_cw310_coverage | |
| - execute_sival_fpga_tests_cw310_coverage | |
| - execute_sival_rom_ext_fpga_tests_cw310_coverage | |
| - execute_manuf_fpga_tests_cw310_coverage | |
| - execute_fpga_rom_ext_tests_hyper310_coverage | |
| - execute_test_rom_fpga_tests_cw340_coverage | |
| - execute_rom_fpga_tests_cw340_coverage | |
| - execute_rom_ext_fpga_tests_cw340_coverage | |
| - execute_sival_fpga_tests_cw340_coverage | |
| - execute_sival_rom_ext_fpga_tests_cw340_coverage | |
| - execute_manuf_fpga_tests_cw340_coverage | |
| - sw_build_test_coverage | |
| if: success() || failure() | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare environment | |
| uses: ./.github/actions/prepare-env | |
| with: | |
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | |
| - name: Download coverage artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: "*-coverage" | |
| path: /tmp/job_reports | |
| - name: Merge Coverage Report | |
| shell: bash | |
| run: | | |
| ci/scripts/merge-coverage-report.sh /tmp/job_reports /tmp/report | |
| - name: Upload report as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ inputs.report-artifact-name || 'coverage-report' }} | |
| path: /tmp/report | |
| overwrite: true | |
| - name: Upload report to Google Cloud | |
| if: ${{ github.event_name != 'pull_request' }} | |
| shell: bash | |
| run: | | |
| BUCKET="sw-coverage" | |
| JOB="${{ github.job }}" | |
| BRANCH="${{ inputs.branch || 'earlgrey_1.0.0'}}" | |
| OUTPUT="job/$JOB/branch/$BRANCH/$(date +%Y-%m-%d-%H%M%S)_coverage_report" | |
| URL="https://storage.googleapis.com/$BUCKET/$OUTPUT/html_report/index-sort-l.html" | |
| gcloud storage cp -r /tmp/report "gs://$BUCKET/$OUTPUT" | |
| echo "Coverage report at $URL" > "$GITHUB_STEP_SUMMARY" |