[cov,ci] Collect test coverage on CI #18844
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright lowRISC contributors (OpenTitan project). | ||
|
Check failure on line 1 in .github/workflows/ci.yml
|
||
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # FIXME: Copy this file as coverage-ci.yml and restore ci.yml | ||
| name: Coverage | ||
| on: | ||
| # FIXME: Remove pull request trigger | ||
| pull_request: | ||
| push: | ||
| branches-ignore: | ||
| - "backport-*" | ||
| tags: | ||
| - "*" | ||
| workflow_dispatch: | ||
| inputs: | ||
| branch: | ||
| description: 'Branch to run the coverage tests' | ||
| required: true | ||
| type: string | ||
| permissions: | ||
| contents: read | ||
| # Needed for workload identity federation | ||
| id-token: write | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | ||
| env: | ||
| VIVADO_VERSION: "2021.1" | ||
| # Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases | ||
| TOOLCHAIN_VERSION: 20220210-1 | ||
| jobs: | ||
| # FIXME: Remove lint jobs in coverage-ci.yml | ||
| quick_lint: | ||
| name: Lint (quick) | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 # Required so we can lint commit messages. | ||
| # - name: Prepare environment | ||
| # uses: ./.github/actions/prepare-env | ||
| # with: | ||
| # service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | ||
| # - name: Show environment | ||
| # run: ./ci/scripts/show-env.sh | ||
| # - name: Commit metadata | ||
| # run: ./ci/scripts/lint-commits.sh "$GITHUB_BASE_REF" | ||
| # if: ${{ github.event_name == 'pull_request' }} | ||
| # - name: License headers | ||
| # run: ./ci/scripts/check-licence-headers.sh "$GITHUB_BASE_REF" | ||
| # if: ${{ github.event_name == 'pull_request' }} | ||
| # - name: Executable bits | ||
| # run: ./ci/scripts/exec-check.sh | ||
| # - name: Non-ASCII characters | ||
| # run: ./ci/scripts/check-ascii.sh | ||
| # - name: Python (flake8) | ||
| # run: ./ci/scripts/python-lint.sh "$GITHUB_BASE_REF" | ||
| # if: ${{ github.event_name == 'pull_request' }} | ||
| # - name: Python (mypy) | ||
| # run: ./ci/scripts/mypy.sh | ||
| # - name: Validate testplans with schema | ||
| # run: ./ci/scripts/validate_testplans.sh | ||
| # - name: C/C++ formatting | ||
| # run: ./bazelisk.sh test //quality:clang_format_check | ||
| # - name: Rust formatting | ||
| # run: ./bazelisk.sh test //quality:rustfmt_check | ||
| # - name: Shellcheck | ||
| # run: ./bazelisk.sh test //quality:shellcheck_check | ||
| # - name: ASM instrumentation | ||
| # run: ./util/coverage/asm/run_instrument.sh --check | ||
| # - name: Header guards | ||
| # run: ./ci/scripts/include-guard.sh "$GITHUB_BASE_REF" | ||
| # if: ${{ github.event_name == 'pull_request' }} | ||
| # - name: Trailing whitespace | ||
| # run: ./ci/scripts/whitespace.sh "$GITHUB_BASE_REF" | ||
| # if: ${{ github.event_name == 'pull_request' }} | ||
| # - name: Broken links | ||
| # run: ./ci/scripts/check-links.sh | ||
| # - name: Generated documentation | ||
| # run: ./ci/scripts/check-cmdgen.sh | ||
| slow_lint: | ||
| name: Lint (slow) | ||
| runs-on: ubuntu-22.04 | ||
| needs: quick_lint | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 # Bitstream cache requires all commits. | ||
| - name: Prepare environment | ||
| uses: ./.github/actions/prepare-env | ||
| with: | ||
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | ||
| - name: Countermeasures implemented (earlgrey) | ||
| run: ./ci/scripts/check-countermeasures.sh earlgrey | ||
| continue-on-error: true | ||
| - name: Countermeasures implemented (englishbreakfast) | ||
| run: ./ci/scripts/check-countermeasures.sh englishbreakfast | ||
| continue-on-error: true | ||
| - name: Bazel test suite tags | ||
| run: ./ci/scripts/check_bazel_test_suites.py | ||
| continue-on-error: true | ||
| # See #21973: disabled until Verilator tags are fixed. | ||
| # - name: Check Bazel tags | ||
| # run: ./ci/scripts/check-bazel-tags.sh | ||
| # continue-on-error: true | ||
| - name: Banned Bazel rules | ||
| run: ./ci/scripts/check-bazel-banned-rules.sh | ||
| - name: Bazel target names | ||
| run: ./ci/scripts/check_bazel_target_names.py | ||
| continue-on-error: true | ||
| - name: DV software images | ||
| run: ./ci/scripts/check_dv_sw_images.sh | ||
| continue-on-error: true | ||
| - name: Generated files | ||
| run: ./ci/scripts/check-generated.sh | ||
| env: | ||
| OT_DESTRUCTIVE: 1 # Required by the script to clean up. | ||
| - name: Buildifier | ||
| run: ./bazelisk.sh test //quality:buildifier_check | ||
| - name: Vendored files | ||
| run: ./ci/scripts/check-vendoring.sh | ||
| - name: Verible RTL | ||
| run: ./ci/scripts/verible-lint.sh rtl | ||
| - name: Verible DV | ||
| run: ./ci/scripts/verible-lint.sh dv | ||
| - name: Verible FPV | ||
| run: ./ci/scripts/verible-lint.sh fpv | ||
| chip_earlgrey_cw310: | ||
| name: Earl Grey for CW310 | ||
| uses: ./.github/workflows/bitstream.yml | ||
| secrets: inherit | ||
| with: | ||
| top_name: earlgrey | ||
| design_suffix: cw310 | ||
| chip_earlgrey_cw310_hyperdebug: | ||
| name: Earl Grey for CW310 Hyperdebug | ||
| uses: ./.github/workflows/bitstream.yml | ||
| secrets: inherit | ||
| with: | ||
| top_name: earlgrey | ||
| design_suffix: cw310_hyperdebug | ||
| chip_earlgrey_cw340: | ||
| name: Earl Grey for CW340 | ||
| uses: ./.github/workflows/bitstream.yml | ||
| secrets: inherit | ||
| with: | ||
| top_name: earlgrey | ||
| design_suffix: cw340 | ||
| # CW310 FPGA jobs. | ||
| execute_test_rom_fpga_tests_cw310: | ||
| name: CW310 Test ROM Tests | ||
| needs: chip_earlgrey_cw310 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_test_rom_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310 | ||
| board: cw310 | ||
| interface: cw310 | ||
| tag_filters: "cw310_test_rom,-coverage_broken" | ||
| mode: coverage | ||
| execute_rom_fpga_tests_cw310: | ||
| name: CW310 ROM Tests | ||
| needs: chip_earlgrey_cw310 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_rom_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310 | ||
| board: cw310 | ||
| interface: cw310 | ||
| tag_filters: "cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf,-coverage_broken" | ||
| timeout: 90 | ||
| mode: coverage | ||
| execute_rom_ext_fpga_tests_cw310: | ||
| name: CW310 ROM_EXT Tests | ||
| needs: chip_earlgrey_cw310 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_rom_ext_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310 | ||
| board: cw310 | ||
| interface: cw310 | ||
| tag_filters: "cw310_rom_ext,-coverage_broken" | ||
| mode: coverage | ||
| execute_sival_fpga_tests_cw310: | ||
| name: CW310 SiVal Tests | ||
| needs: chip_earlgrey_cw310_hyperdebug | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_sival_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310_hyperdebug | ||
| board: cw310 | ||
| interface: hyper310 | ||
| tag_filters: "cw310_sival,-manuf,-coverage_broken" | ||
| mode: coverage | ||
| execute_sival_rom_ext_fpga_tests_cw310: | ||
| name: CW310 SiVal ROM_EXT Tests | ||
| needs: chip_earlgrey_cw310_hyperdebug | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_sival_rom_ext_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310_hyperdebug | ||
| board: cw310 | ||
| interface: hyper310 | ||
| tag_filters: "cw310_sival_rom_ext,-coverage_broken" | ||
| mode: coverage | ||
| execute_manuf_fpga_tests_cw310: | ||
| name: CW310 Manufacturing Tests | ||
| needs: chip_earlgrey_cw310_hyperdebug | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_manuf_fpga_tests_cw310 | ||
| bitstream: chip_earlgrey_cw310_hyperdebug | ||
| board: cw310 | ||
| interface: hyper310 | ||
| tag_filters: "manuf,-cw340,-qemu,-coverage_broken" | ||
| mode: coverage | ||
| execute_fpga_rom_ext_tests_hyper310: | ||
| name: Hyper310 ROM_EXT Tests | ||
| needs: chip_earlgrey_cw310_hyperdebug | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_fpga_rom_ext_tests_hyper310 | ||
| bitstream: chip_earlgrey_cw310_hyperdebug | ||
| board: cw310 | ||
| interface: hyper310 | ||
| tag_filters: "hyper310_rom_ext,-coverage_broken" | ||
| mode: coverage | ||
| # CW340 FPGA jobs | ||
| execute_test_rom_fpga_tests_cw340: | ||
| name: CW340 Test ROM Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_test_rom_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "cw340_test_rom,-coverage_broken" | ||
| mode: coverage | ||
| execute_rom_fpga_tests_cw340: | ||
| name: CW340 ROM Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_rom_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf,-coverage_broken" | ||
| mode: coverage | ||
| execute_rom_ext_fpga_tests_cw340: | ||
| name: CW340 ROM_EXT Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_rom_ext_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "cw340_rom_ext,-coverage_broken" | ||
| mode: coverage | ||
| execute_sival_fpga_tests_cw340: | ||
| name: CW340 SiVal Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_sival_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "cw340_sival,-manuf,-coverage_broken" | ||
| mode: coverage | ||
| execute_sival_rom_ext_fpga_tests_cw340: | ||
| name: CW340 SiVal ROM_EXT Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_sival_rom_ext_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "cw340_sival_rom_ext,-coverage_broken" | ||
| mode: coverage | ||
| execute_manuf_fpga_tests_cw340: | ||
| name: CW340 Manufacturing Tests | ||
| needs: chip_earlgrey_cw340 | ||
| uses: ./.github/workflows/fpga.yml | ||
| secrets: inherit | ||
| with: | ||
| job_name: execute_manuf_fpga_tests_cw340 | ||
| bitstream: chip_earlgrey_cw340 | ||
| board: cw340 | ||
| interface: cw340 | ||
| tag_filters: "manuf,-hyper310,-qemu,-coverage_broken" | ||
| mode: coverage | ||
| # FIXME: Move this job to coverage-quick.yml | ||
| sw_build_test: | ||
| name: Unit Tests | ||
| runs-on: ubuntu-22.04-vivado | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 # Required for bitstream cache to work. | ||
| - name: Prepare environment | ||
| uses: ./.github/actions/prepare-env | ||
| with: | ||
| service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}' | ||
| - name: Check Bazel build graph | ||
| run: | | ||
| # Test the graph with both an empty and filled bitstream cache. | ||
| ./ci/scripts/test-empty-bitstream-cache.sh | ||
| ./bazelisk.sh build --nobuild //... | ||
| - name: Select software targets | ||
| run: | | ||
| target_pattern_file="$(mktemp)" | ||
| echo "target_pattern_file=${target_pattern_file}" >> "$GITHUB_ENV" | ||
| # Start with building the whole graph. | ||
| echo '//sw/...' > "$target_pattern_file" | ||
| # Exclude some targets: | ||
| # | ||
| # 1. `//hw/...` is out of scope. | ||
| # 2. `//quality/...` is tested by the lint jobs. | ||
| # 3. `//sw/otbn/crypto/...` is tested by the OTBN job. | ||
| # 4. `//third_party/...` which is not our code. | ||
| # 5. `//sw/host/...` FIXME: rust coverage are broken on CI. | ||
| printf "%s\n" \ | ||
| "-//hw/..." \ | ||
| "-//quality/..." \ | ||
| "-//sw/otbn/crypto/..." \ | ||
| "-//third_party/..." \ | ||
| "-//sw/host/..." \ | ||
| >> "$target_pattern_file" | ||
| # Exclude anything that requires a bitstream splice. | ||
| ./bazelisk.sh cquery \ | ||
| --noinclude_aspects \ | ||
| --output=starlark \ | ||
| --starlark:expr='"-{}".format(target.label)' \ | ||
| --define DISABLE_VERILATOR_BUILD=true \ | ||
| -- "rdeps(//..., kind(bitstream_splice, //...))" \ | ||
| >> "$target_pattern_file" | ||
| - name: Run software unit tests coverage | ||
| run: | | ||
| ./bazelisk.sh coverage \ | ||
| --build_tests_only \ | ||
| --keep_going \ | ||
| --config=ot_coverage \ | ||
| --test_output=errors \ | ||
| --define DISABLE_VERILATOR_BUILD=true \ | ||
| --target_pattern_file="$target_pattern_file" \ | ||
| --test_tag_filters=-broken,-coverage_broken,-cw310,-verilator,-dv,-silicon,-qemu | ||
| - name: Publish Bazel test results | ||
| uses: ./.github/actions/publish-bazel-test-results | ||
| if: ${{ !cancelled() }} | ||
| with: | ||
| artifact-name: sw_build_test-test-results | ||
| - name: Check for unrunnable tests | ||
| run: ./ci/scripts/check-unrunnable-tests.sh | ||
| continue-on-error: true | ||
| # FIXME: Copy this job to ci.yml after restore | ||
| coverage_quick: | ||
| name: Coverage (quick) | ||
| needs: quick_lint | ||
| uses: ./.github/workflows/coverage-quick.yml | ||
| secrets: inherit | ||
| # FIXME: Copy this job to ci.yml after restore | ||
| publish_coverage_report: | ||
| name: Publish Coverage Report | ||
| needs: | ||
| # - execute_test_rom_fpga_tests_cw310_coverage | ||
| # - execute_rom_fpga_tests_cw310_coverage | ||
| # - execute_rom_ext_fpga_tests_cw310_coverage | ||
| # - execute_sival_fpga_tests_cw310_coverage | ||
| # - execute_sival_rom_ext_fpga_tests_cw310_coverage | ||
| # - execute_manuf_fpga_tests_cw310_coverage | ||
| # - execute_fpga_rom_ext_tests_hyper310_coverage | ||
| # - execute_test_rom_fpga_tests_cw340_coverage | ||
| # - execute_rom_fpga_tests_cw340_coverage | ||
| # - execute_rom_ext_fpga_tests_cw340_coverage | ||
| # - execute_sival_fpga_tests_cw340_coverage | ||
| # - execute_sival_rom_ext_fpga_tests_cw340_coverage | ||
| # - execute_manuf_fpga_tests_cw340_coverage | ||
| - sw_build_test_coverage | ||
| - coverage_tests | ||
| if: success() || failure() | ||
| uses: ./.github/workflows/publish-coverage-report.yml | ||
| secrets: inherit | ||
| with: | ||
| branch: ${{ inputs.branch || 'earlgrey_1.0.0' }} | ||
