[cov,ci] Collect test coverage on CI #18837

Workflow file for this run

	# Copyright lowRISC contributors (OpenTitan project).
	# Licensed under the Apache License, Version 2.0, see LICENSE for details.
	# SPDX-License-Identifier: Apache-2.0

	# FIXME: Copy this file as coverage.yml and restore ci.yml
	name: Coverage
	on:
	# FIXME: Remove pull request trigger
	pull_request:

	push:
	branches-ignore:
	- "backport-*"
	tags:
	- "*"

	workflow_dispatch:
	inputs:
	branch:
	description: 'Branch to run the coverage tests'
	required: true
	type: string

	permissions:
	contents: read
	# Needed for workload identity federation
	id-token: write

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	VIVADO_VERSION: "2021.1"
	# Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases
	TOOLCHAIN_VERSION: 20220210-1

	jobs:
	# FIXME: Remove lint jobs
	quick_lint:
	name: Lint (quick)
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # Required so we can lint commit messages.
	- name: Prepare environment
	uses: ./.github/actions/prepare-env
	with:
	service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
	- name: Show environment
	run: ./ci/scripts/show-env.sh
	- name: Commit metadata
	run: ./ci/scripts/lint-commits.sh "$GITHUB_BASE_REF"
	if: ${{ github.event_name == 'pull_request' }}
	- name: License headers
	run: ./ci/scripts/check-licence-headers.sh "$GITHUB_BASE_REF"
	if: ${{ github.event_name == 'pull_request' }}
	- name: Executable bits
	run: ./ci/scripts/exec-check.sh
	- name: Non-ASCII characters
	run: ./ci/scripts/check-ascii.sh
	- name: Python (flake8)
	run: ./ci/scripts/python-lint.sh "$GITHUB_BASE_REF"
	if: ${{ github.event_name == 'pull_request' }}
	- name: Python (mypy)
	run: ./ci/scripts/mypy.sh
	- name: Validate testplans with schema
	run: ./ci/scripts/validate_testplans.sh
	- name: C/C++ formatting
	run: ./bazelisk.sh test //quality:clang_format_check
	- name: Rust formatting
	run: ./bazelisk.sh test //quality:rustfmt_check
	- name: Shellcheck
	run: ./bazelisk.sh test //quality:shellcheck_check
	- name: ASM instrumentation
	run: ./util/coverage/asm/run_instrument.sh --check
	- name: Header guards
	run: ./ci/scripts/include-guard.sh "$GITHUB_BASE_REF"
	if: ${{ github.event_name == 'pull_request' }}
	- name: Trailing whitespace
	run: ./ci/scripts/whitespace.sh "$GITHUB_BASE_REF"
	if: ${{ github.event_name == 'pull_request' }}
	- name: Broken links
	run: ./ci/scripts/check-links.sh
	- name: Generated documentation
	run: ./ci/scripts/check-cmdgen.sh
	slow_lint:
	name: Lint (slow)
	runs-on: ubuntu-22.04
	needs: quick_lint
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # Bitstream cache requires all commits.
	- name: Prepare environment
	uses: ./.github/actions/prepare-env
	with:
	service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
	- name: Countermeasures implemented (earlgrey)
	run: ./ci/scripts/check-countermeasures.sh earlgrey
	continue-on-error: true
	- name: Countermeasures implemented (englishbreakfast)
	run: ./ci/scripts/check-countermeasures.sh englishbreakfast
	continue-on-error: true
	- name: Bazel test suite tags
	run: ./ci/scripts/check_bazel_test_suites.py
	continue-on-error: true
	# See #21973: disabled until Verilator tags are fixed.
	# - name: Check Bazel tags
	# run: ./ci/scripts/check-bazel-tags.sh
	# continue-on-error: true
	- name: Banned Bazel rules
	run: ./ci/scripts/check-bazel-banned-rules.sh
	- name: Bazel target names
	run: ./ci/scripts/check_bazel_target_names.py
	continue-on-error: true
	- name: DV software images
	run: ./ci/scripts/check_dv_sw_images.sh
	continue-on-error: true
	- name: Generated files
	run: ./ci/scripts/check-generated.sh
	env:
	OT_DESTRUCTIVE: 1 # Required by the script to clean up.
	- name: Buildifier
	run: ./bazelisk.sh test //quality:buildifier_check
	- name: Vendored files
	run: ./ci/scripts/check-vendoring.sh
	- name: Verible RTL
	run: ./ci/scripts/verible-lint.sh rtl
	- name: Verible DV
	run: ./ci/scripts/verible-lint.sh dv
	- name: Verible FPV
	run: ./ci/scripts/verible-lint.sh fpv

	chip_earlgrey_cw310:
	name: Earl Grey for CW310
	uses: ./.github/workflows/bitstream.yml
	secrets: inherit
	with:
	top_name: earlgrey
	design_suffix: cw310

	chip_earlgrey_cw310_hyperdebug:
	name: Earl Grey for CW310 Hyperdebug
	uses: ./.github/workflows/bitstream.yml
	secrets: inherit
	with:
	top_name: earlgrey
	design_suffix: cw310_hyperdebug

	chip_earlgrey_cw340:
	name: Earl Grey for CW340
	uses: ./.github/workflows/bitstream.yml
	secrets: inherit
	with:
	top_name: earlgrey
	design_suffix: cw340

	# CW310 FPGA jobs.
	execute_test_rom_fpga_tests_cw310_coverage:
	name: CW310 Test ROM Tests Coverage
	needs: chip_earlgrey_cw310
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_test_rom_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310
	board: cw310
	interface: cw310
	tag_filters: cw310_test_rom
	mode: coverage

	execute_rom_fpga_tests_cw310_coverage:
	name: CW310 ROM Tests Coverage
	needs: chip_earlgrey_cw310
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_rom_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310
	board: cw310
	interface: cw310
	tag_filters: "cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf"
	timeout: 90
	mode: coverage

	execute_rom_ext_fpga_tests_cw310_coverage:
	name: CW310 ROM_EXT Tests Coverage
	needs: chip_earlgrey_cw310
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_rom_ext_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310
	board: cw310
	interface: cw310
	tag_filters: cw310_rom_ext
	mode: coverage

	execute_sival_fpga_tests_cw310_coverage:
	name: CW310 SiVal Tests Coverage
	needs: chip_earlgrey_cw310_hyperdebug
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_sival_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310_hyperdebug
	board: cw310
	interface: hyper310
	tag_filters: "cw310_sival,-manuf"
	mode: coverage

	execute_sival_rom_ext_fpga_tests_cw310_coverage:
	name: CW310 SiVal ROM_EXT Tests Coverage
	needs: chip_earlgrey_cw310_hyperdebug
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_sival_rom_ext_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310_hyperdebug
	board: cw310
	interface: hyper310
	tag_filters: cw310_sival_rom_ext
	mode: coverage

	execute_manuf_fpga_tests_cw310_coverage:
	name: CW310 Manufacturing Tests Coverage
	needs: chip_earlgrey_cw310_hyperdebug
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_manuf_fpga_tests_cw310
	bitstream: chip_earlgrey_cw310_hyperdebug
	board: cw310
	interface: hyper310
	tag_filters: "manuf,-cw340,-qemu"
	mode: coverage

	execute_fpga_rom_ext_tests_hyper310_coverage:
	name: Hyper310 ROM_EXT Tests Coverage
	needs: chip_earlgrey_cw310_hyperdebug
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_fpga_rom_ext_tests_hyper310
	bitstream: chip_earlgrey_cw310_hyperdebug
	board: cw310
	interface: hyper310
	tag_filters: hyper310_rom_ext
	mode: coverage

	# CW340 FPGA jobs
	execute_test_rom_fpga_tests_cw340_coverage:
	name: CW340 Test ROM Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_test_rom_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	tag_filters: cw340_test_rom
	mode: coverage

	execute_rom_fpga_tests_cw340_coverage:
	name: CW340 ROM Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_rom_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	tag_filters: "cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf"
	mode: coverage

	execute_rom_ext_fpga_tests_cw340_coverage:
	name: CW340 ROM_EXT Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_rom_ext_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	tag_filters: cw340_rom_ext
	mode: coverage

	execute_sival_fpga_tests_cw340_coverage:
	name: CW340 SiVal Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_sival_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	tag_filters: "cw340_sival,-manuf"
	mode: coverage

	execute_sival_rom_ext_fpga_tests_cw340_coverage:
	name: CW340 SiVal ROM_EXT Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_sival_rom_ext_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	# FIXME: enable pentest tests after buildable
	tag_filters: "cw340_sival_rom_ext,-pentest"
	mode: coverage

	execute_manuf_fpga_tests_cw340_coverage:
	name: CW340 Manufacturing Tests Coverage
	needs: chip_earlgrey_cw340
	uses: ./.github/workflows/fpga.yml
	secrets: inherit
	with:
	job_name: execute_manuf_fpga_tests_cw340
	bitstream: chip_earlgrey_cw340
	board: cw340
	interface: cw340
	tag_filters: "manuf,-hyper310,-qemu"
	mode: coverage

	# FIXME: Move this job to coverage-quick.yml
	sw_build_test_coverage:
	name: Unit Test Coverage
	runs-on: ubuntu-22.04-vivado
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # Required for bitstream cache to work.
	- name: Prepare environment
	uses: ./.github/actions/prepare-env
	with:
	service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
	- name: Check Bazel build graph
	run: \|
	# Test the graph with both an empty and filled bitstream cache.
	./ci/scripts/test-empty-bitstream-cache.sh
	./bazelisk.sh build --nobuild //...
	- name: Select software targets
	run: \|
	target_pattern_file="$(mktemp)"
	echo "target_pattern_file=${target_pattern_file}" >> "$GITHUB_ENV"

	# Start with building the whole graph.
	echo '//sw/...' > "$target_pattern_file"
	# Exclude some targets:
	#
	# 1. `//hw/...` is out of scope.
	# 2. `//quality/...` is tested by the lint jobs.
	# 3. `//sw/otbn/crypto/...` is tested by the OTBN job.
	# 4. `//third_party/...` which is not our code.
	# 5. `//sw/host/...` FIXME: rust coverage are broken on CI.
	printf "%s\n" \
	"-//hw/..." \
	"-//quality/..." \
	"-//sw/otbn/crypto/..." \
	"-//third_party/..." \
	"-//sw/host/..." \
	>> "$target_pattern_file"
	# Exclude anything that requires a bitstream splice.
	./bazelisk.sh cquery \
	--noinclude_aspects \
	--output=starlark \
	--starlark:expr='"-{}".format(target.label)' \
	--define DISABLE_VERILATOR_BUILD=true \
	-- "rdeps(//..., kind(bitstream_splice, //...))" \
	>> "$target_pattern_file"
	- name: Run software unit tests coverage
	run: \|
	./bazelisk.sh coverage \
	--build_tests_only \
	--keep_going \
	--config=ot_coverage \
	--test_output=errors \
	--define DISABLE_VERILATOR_BUILD=true \
	--target_pattern_file="$target_pattern_file" \
	--test_tag_filters=-broken,-coverage_broken,-cw310,-verilator,-dv,-silicon,-qemu
	- name: Publish Bazel test results
	uses: ./.github/actions/publish-bazel-test-results
	if: ${{ !cancelled() }}
	with:
	artifact-name: sw_build_test-test-results
	- name: Check for unrunnable tests
	run: ./ci/scripts/check-unrunnable-tests.sh
	continue-on-error: true

	coverage_tests:
	name: Coverage Tests
	uses: ./.github/workflows/coverage-quick.yml
	secrets: inherit

	publish_coverage_report:
	name: Publish Coverage Report
	needs:
	- execute_test_rom_fpga_tests_cw310_coverage
	- execute_rom_fpga_tests_cw310_coverage
	- execute_rom_ext_fpga_tests_cw310_coverage
	- execute_sival_fpga_tests_cw310_coverage
	- execute_sival_rom_ext_fpga_tests_cw310_coverage
	- execute_manuf_fpga_tests_cw310_coverage
	- execute_fpga_rom_ext_tests_hyper310_coverage
	- execute_test_rom_fpga_tests_cw340_coverage
	- execute_rom_fpga_tests_cw340_coverage
	- execute_rom_ext_fpga_tests_cw340_coverage
	- execute_sival_fpga_tests_cw340_coverage
	- execute_sival_rom_ext_fpga_tests_cw340_coverage
	- execute_manuf_fpga_tests_cw340_coverage
	- sw_build_test_coverage
	- coverage_tests
	if: success() \|\| failure()
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4
	- name: Prepare environment
	uses: ./.github/actions/prepare-env
	with:
	service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
	- name: Download coverage artifacts
	uses: actions/download-artifact@v4
	with:
	pattern: "*-coverage"
	path: /tmp/job_reports
	- name: Merge Coverage Report
	shell: bash
	run: \|
	ci/scripts/merge-coverage-report.sh /tmp/job_reports /tmp/report
	- name: Upload report as artifact
	uses: actions/upload-artifact@v4
	with:
	name: 'coverage-report'
	path: /tmp/report
	overwrite: true
	- name: Upload report to Google Cloud
	if: ${{ github.event_name != 'pull_request' }}
	shell: bash
	run: \|
	BUCKET="sw-coverage"
	JOB="${{ github.job }}"
	BRANCH="${{ inputs.branch \|\| 'earlgrey_1.0.0'}}"
	OUTPUT="job/$JOB/branch/$BRANCH/$(date +%Y-%m-%d-%H%M%S)_coverage_report"
	URL="https://storage.googleapis.com/$BUCKET/$OUTPUT/html_report/index-sort-l.html"
	gcloud storage cp -r /tmp/report "gs://$BUCKET/$OUTPUT"
	echo "Coverage report at $URL" > "$GITHUB_STEP_SUMMARY"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[cov,ci] Collect test coverage on CI #18837

Workflow file

[cov,ci] Collect test coverage on CI #18837

Uh oh!

Workflow file for this run