Skip to content

Code coverage - Nightly #18

Code coverage - Nightly

Code coverage - Nightly #18

Workflow file for this run

name: Code coverage
run-name: ${{ github.event_name == 'schedule' && 'Code coverage - Nightly' || 'Code coverage' }}
on:
schedule:
# run daily at 00:00 UTC
- cron: '0 0 * * *'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions: read-all
env:
PIP_CACHE_PATH: /mount/caches/pip/linux
TEST_PROFILE: cpu
ENABLE_BRANCH_COVERAGE: 'false'
LCOV_CAPTURE_TIMEOUT_SECONDS: '3600'
NODE_VERSION: '21'
NODE_EXTRA_CA_CERTS: /usr/local/share/ca-certificates/IntelProxyRootCA-Base64.crt
jobs:
Docker:
runs-on: aks-linux-4-cores-16gb-docker-build
container:
image: openvinogithubactions.azurecr.io/docker_build:0.2
volumes:
- /mount:/mount
outputs:
images: ${{ steps.handle_docker.outputs.images }}
steps:
- name: Checkout
uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
timeout-minutes: 15
- uses: ./.github/actions/handle_docker
id: handle_docker
with:
images: |
ov_build/ubuntu_22_04_x64
ov_test/ubuntu_22_04_x64
registry: 'openvinogithubactions.azurecr.io'
dockerfiles_root_dir: '.github/dockerfiles'
changed_components: '{"docker_env":false,"dockerfiles":false}'
Build:
name: Build coverage artifacts
needs: Docker
uses: ./.github/workflows/job_build_linux.yml
with:
runner: 'aks-linux-16-cores-32gb'
image: ${{ fromJSON(needs.Docker.outputs.images).ov_build.ubuntu_22_04_x64 }}
affected-components: '{"JS_API":true}'
event-name: ${{ github.event_name }}
os: 'ubuntu_22_04'
build-js: true
build-additional-python-packages: false
build-debian-packages: false
upload-coverage-notes: true
cmake-options: >-
-G 'Ninja Multi-Config'
-DENABLE_NCC_STYLE=OFF
-DENABLE_TESTS=ON
-DENABLE_FUNCTIONAL_TESTS=ON
-DENABLE_STRICT_DEPENDENCIES=OFF
-DENABLE_SYSTEM_OPENCL=ON
-DCMAKE_VERBOSE_MAKEFILE=ON
-DCPACK_GENERATOR=TGZ
-DENABLE_WHEEL=ON
-DENABLE_PYTHON=ON
-DENABLE_JS=ON
-DENABLE_COVERAGE=ON
-DENABLE_OV_ONNX_FRONTEND=ON
-DENABLE_OV_PADDLE_FRONTEND=ON
-DENABLE_OV_PYTORCH_FRONTEND=ON
-DENABLE_OV_TF_FRONTEND=ON
-DENABLE_OV_TF_LITE_FRONTEND=ON
-DENABLE_INTEL_GPU=OFF
-DENABLE_INTEL_NPU=OFF
-DENABLE_ONEDNN_FOR_GPU=OFF
CoverageCpp:
name: Coverage C++
needs: [Docker, Build]
runs-on: aks-linux-8-cores-32gb
container:
image: ${{ fromJSON(needs.Docker.outputs.images).ov_test.ubuntu_22_04_x64 }}
volumes:
- /usr/local/share/ca-certificates:/usr/local/share/ca-certificates:ro
- /mount:/mount
- ${{ github.workspace }}:${{ github.workspace }}
defaults:
run:
shell: bash
working-directory: ${{ github.workspace }}
env:
OV_WORKSPACE: ${{ github.workspace }}
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
INSTALL_PKG_DIR: ${{ github.workspace }}/install
BIN_DIR: ${{ github.workspace }}/install/tests
BUILD_DIR: ${{ github.workspace }}/openvino/openvino_build
BUILD_JS_DIR: ${{ github.workspace }}/openvino/openvino_build_js
COVERAGE_WRITE_STEP_SUMMARY: 'false'
steps:
- name: Checkout sources
uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
timeout-minutes: 15
with:
submodules: 'true'
- name: Download OpenVINO package
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_package
path: ${{ env.INSTALL_DIR }}
- name: Download OpenVINO tests
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_tests
path: ${{ env.INSTALL_DIR }}
- name: Download coverage build notes
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: coverage_build_notes
path: ${{ github.workspace }}/coverage_notes
- name: Extract OpenVINO packages
run: |
pigz -dc openvino_package.tar.gz | tar -xf - -C "${INSTALL_DIR}"
pigz -dc openvino_tests.tar.gz | tar -xf - -C "${INSTALL_DIR}"
working-directory: ${{ env.INSTALL_DIR }}
- name: Restore coverage notes
run: |
set -euo pipefail
notes_root="${{ github.workspace }}/coverage_notes"
if [[ -d "${notes_root}/coverage_build_notes" ]]; then
notes_root="${notes_root}/coverage_build_notes"
fi
rm -rf "${BUILD_DIR}" "${BUILD_JS_DIR}"
mkdir -p "${BUILD_DIR}" "${BUILD_JS_DIR}"
cp -a "${notes_root}/main-build/." "${BUILD_DIR}/"
if [[ -d "${notes_root}/js-build" ]]; then
cp -a "${notes_root}/js-build/." "${BUILD_JS_DIR}/"
fi
- name: Setup Python 3.11
uses: ./.github/actions/setup_python
with:
version: '3.11'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'
- name: Install coverage Python dependencies
run: python3 -m pip install -r .github/scripts/coverage/requirements-ci.txt
- name: Run CPU C++ coverage tests
run: python3 .github/scripts/coverage/coverage.py step run-cpp-tests
- name: Generate CPU native C/C++ coverage report
run: python3 .github/scripts/coverage/coverage.py step collect-cpp-coverage
- name: Collect C++ CPU results
id: cpu_artifact
if: ${{ always() }}
run: |
dir="${RUNNER_TEMP}/coverage-cpp-cpu"
python3 .github/scripts/coverage/ci_reports.py collect-suite-results \
--workspace "${{ env.OV_WORKSPACE }}" \
--suite cpp \
--profile cpu \
--lane cpu \
--artifact-name coverage-cpp-cpu \
--artifact-dir "${dir}"
echo "dir=${dir}" >> "$GITHUB_OUTPUT"
- name: Upload Codecov (C++ runtime, CPU)
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ env.OV_WORKSPACE }}/coverage.info
disable_search: true
plugins: noop
flags: cpp-runtime-cpp-cpu
override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
fail_ci_if_error: true
verbose: true
- name: Upload C++ CPU artifact
if: ${{ always() }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: coverage-cpp-cpu
path: ${{ steps.cpu_artifact.outputs.dir }}
if-no-files-found: warn
retention-days: 14
compression-level: 0
overwrite: true
CoveragePython:
name: Coverage Python
needs: [Docker, Build]
runs-on: aks-linux-4-cores-16gb
container:
image: ${{ fromJSON(needs.Docker.outputs.images).ov_test.ubuntu_22_04_x64 }}
volumes:
- /usr/local/share/ca-certificates:/usr/local/share/ca-certificates:ro
- /mount:/mount
- ${{ github.workspace }}:${{ github.workspace }}
defaults:
run:
shell: bash
working-directory: ${{ github.workspace }}
env:
OV_WORKSPACE: ${{ github.workspace }}
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
INSTALL_PKG_DIR: ${{ github.workspace }}/install
BUILD_DIR: ${{ github.workspace }}/openvino/openvino_build
BUILD_JS_DIR: ${{ github.workspace }}/openvino/openvino_build_js
COVERAGE_WRITE_STEP_SUMMARY: 'false'
steps:
- name: Checkout sources
uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
timeout-minutes: 15
with:
submodules: 'true'
- name: Download OpenVINO package
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_package
path: ${{ env.INSTALL_DIR }}
- name: Download OpenVINO tests
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_tests
path: ${{ env.INSTALL_DIR }}
- name: Download OpenVINO wheels
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_wheels
path: ${{ env.INSTALL_DIR }}
- name: Download coverage build notes
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: coverage_build_notes
path: ${{ github.workspace }}/coverage_notes
- name: Extract OpenVINO packages
run: |
pigz -dc openvino_package.tar.gz | tar -xf - -C "${INSTALL_DIR}"
pigz -dc openvino_tests.tar.gz | tar -xf - -C "${INSTALL_DIR}"
working-directory: ${{ env.INSTALL_DIR }}
- name: Restore coverage notes
run: |
set -euo pipefail
notes_root="${{ github.workspace }}/coverage_notes"
if [[ -d "${notes_root}/coverage_build_notes" ]]; then
notes_root="${notes_root}/coverage_build_notes"
fi
rm -rf "${BUILD_DIR}" "${BUILD_JS_DIR}"
mkdir -p "${BUILD_DIR}" "${BUILD_JS_DIR}"
cp -a "${notes_root}/main-build/." "${BUILD_DIR}/"
if [[ -d "${notes_root}/js-build" ]]; then
cp -a "${notes_root}/js-build/." "${BUILD_JS_DIR}/"
fi
- name: Setup Python 3.11
uses: ./.github/actions/setup_python
with:
version: '3.11'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'
- name: Install coverage Python dependencies
run: python3 -m pip install -r .github/scripts/coverage/requirements-ci.txt
- name: Install OpenVINO Python wheels
uses: ./.github/actions/install_ov_wheels
with:
wheels-dir-path: ${{ env.INSTALL_DIR }}
wheels-to-install: 'openvino'
- name: Install Python test dependencies
run: |
python3 -m pip install -r "${INSTALL_DIR}/tests/bindings/python/requirements_test.txt"
python3 -m pip install -r "${INSTALL_DIR}/tests/python/preprocess/torchvision/requirements.txt"
python3 -m pip install -r "${INSTALL_DIR}/tests/layer_tests/requirements.txt"
python3 -m pip install -r "${INSTALL_DIR}/tests/requirements_onnx"
python3 -m pip install -r "${INSTALL_DIR}/tests/requirements_jax"
python3 -m pip install -r "${INSTALL_DIR}/tests/requirements_pytorch"
python3 -m pip install -r "${INSTALL_DIR}/tests/requirements_tensorflow"
- name: Run CPU Python coverage tests
run: python3 .github/scripts/coverage/coverage.py step run-python-tests
- name: Generate CPU native C/C++ coverage report
run: python3 .github/scripts/coverage/coverage.py step collect-cpp-coverage
- name: Collect Python CPU results
id: cpu_artifact
if: ${{ always() }}
run: |
dir="${RUNNER_TEMP}/coverage-python-cpu"
python3 .github/scripts/coverage/ci_reports.py collect-suite-results \
--workspace "${{ env.OV_WORKSPACE }}" \
--suite python \
--profile cpu \
--lane cpu \
--artifact-name coverage-python-cpu \
--artifact-dir "${dir}"
echo "dir=${dir}" >> "$GITHUB_OUTPUT"
- name: Upload Codecov (Python API, CPU)
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ env.OV_WORKSPACE }}/python-coverage.xml
disable_search: true
plugins: noop
flags: python-api-frontend-layer-ovc-cpu
override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
fail_ci_if_error: true
verbose: true
- name: Upload Codecov (C++ runtime, Python suite, CPU)
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ env.OV_WORKSPACE }}/coverage.info
disable_search: true
plugins: noop
flags: cpp-runtime-python-cpu
override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
fail_ci_if_error: true
verbose: true
- name: Upload Python CPU artifact
if: ${{ always() }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: coverage-python-cpu
path: ${{ steps.cpu_artifact.outputs.dir }}
if-no-files-found: warn
retention-days: 14
compression-level: 0
overwrite: true
CoverageJS:
name: Coverage JS
needs: [Docker, Build]
runs-on: aks-linux-4-cores-16gb
container:
image: ${{ fromJSON(needs.Docker.outputs.images).ov_test.ubuntu_22_04_x64 }}
volumes:
- /usr/local/share/ca-certificates:/usr/local/share/ca-certificates:ro
- /mount:/mount
- ${{ github.workspace }}:${{ github.workspace }}
defaults:
run:
shell: bash
working-directory: ${{ github.workspace }}
env:
OV_WORKSPACE: ${{ github.workspace }}
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_PKG_DIR: ${{ github.workspace }}/install
OPENVINO_JS_DIR: ${{ github.workspace }}/src/bindings/js/node
BUILD_DIR: ${{ github.workspace }}/openvino/openvino_build
BUILD_JS_DIR: ${{ github.workspace }}/openvino/openvino_build_js
COVERAGE_WRITE_STEP_SUMMARY: 'false'
DISPLAY: ':99'
steps:
- name: Checkout sources
uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
timeout-minutes: 15
with:
submodules: 'true'
- name: Download OpenVINO package
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_package
path: ${{ env.INSTALL_DIR }}
- name: Download OpenVINO Node package
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: openvino_node_npm_package
path: ${{ env.OPENVINO_JS_DIR }}
merge-multiple: true
- name: Download coverage build notes
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: coverage_build_notes
path: ${{ github.workspace }}/coverage_notes
- name: Extract OpenVINO package
run: pigz -dc openvino_package.tar.gz | tar -xf - -C "${INSTALL_DIR}"
working-directory: ${{ env.INSTALL_DIR }}
- name: Extract OpenVINO Node package
run: pigz -dc openvino_node_npm_package.tar.gz | tar -xf - -v
working-directory: ${{ env.OPENVINO_JS_DIR }}
- name: Setup Node ${{ env.NODE_VERSION }}
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version: ${{ env.NODE_VERSION }}
cache: npm
cache-dependency-path: ${{ env.OPENVINO_JS_DIR }}/package-lock.json
- name: Configure OpenVINO JS API
working-directory: ${{ env.OPENVINO_JS_DIR }}
run: npm i
- name: Install JS coverage dependencies
working-directory: ${{ env.OPENVINO_JS_DIR }}
run: npm i --no-save c8
- name: Setup Python 3.11
uses: ./.github/actions/setup_python
with:
version: '3.11'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'
- name: Install coverage Python dependencies
run: python3 -m pip install -r .github/scripts/coverage/requirements-ci.txt
- name: Restore coverage notes
run: |
set -euo pipefail
notes_root="${{ github.workspace }}/coverage_notes"
if [[ -d "${notes_root}/coverage_build_notes" ]]; then
notes_root="${notes_root}/coverage_build_notes"
fi
rm -rf "${BUILD_DIR}" "${BUILD_JS_DIR}"
mkdir -p "${BUILD_DIR}" "${BUILD_JS_DIR}"
cp -a "${notes_root}/main-build/." "${BUILD_DIR}/"
if [[ -d "${notes_root}/js-build" ]]; then
cp -a "${notes_root}/js-build/." "${BUILD_DIR}/"
cp -a "${notes_root}/js-build/." "${BUILD_JS_DIR}/"
fi
- name: Install electron deps and run Xvfb
run: Xvfb "$DISPLAY" &
- name: Run CPU JS coverage tests
run: python3 .github/scripts/coverage/coverage.py step run-js-tests
- name: Generate CPU native C/C++ coverage report
run: python3 .github/scripts/coverage/coverage.py step collect-cpp-coverage
- name: Collect JS CPU results
id: cpu_artifact
if: ${{ always() }}
run: |
dir="${RUNNER_TEMP}/coverage-js-cpu"
python3 .github/scripts/coverage/ci_reports.py collect-suite-results \
--workspace "${{ env.OV_WORKSPACE }}" \
--suite js \
--profile cpu \
--lane cpu \
--artifact-name coverage-js-cpu \
--artifact-dir "${dir}"
echo "dir=${dir}" >> "$GITHUB_OUTPUT"
- name: Upload Codecov (JS bindings, CPU)
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ env.OV_WORKSPACE }}/js-lcov.info
disable_search: true
plugins: noop
flags: nodejs-bindings-unit-e2e-cpu
override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
fail_ci_if_error: true
verbose: true
- name: Upload Codecov (C++ runtime, JS suite, CPU)
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ env.OV_WORKSPACE }}/coverage.info
disable_search: true
plugins: noop
flags: cpp-runtime-js-cpu
override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
fail_ci_if_error: true
verbose: true
- name: Upload JS CPU artifact
if: ${{ always() }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: coverage-js-cpu
path: ${{ steps.cpu_artifact.outputs.dir }}
if-no-files-found: warn
retention-days: 14
compression-level: 0
overwrite: true
CoverageSummary:
if: ${{ always() && needs.Build.result == 'success' }}
needs: [Build, CoverageCpp, CoveragePython, CoverageJS]
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
timeout-minutes: 15
with:
sparse-checkout: |
.github/scripts/coverage
submodules: 'false'
- name: Download C++ artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
pattern: coverage-cpp-*
path: ${{ github.workspace }}/artifacts/cpp
- name: Download Python artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
pattern: coverage-python-*
path: ${{ github.workspace }}/artifacts/python
- name: Download JS artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
pattern: coverage-js-*
path: ${{ github.workspace }}/artifacts/js
- name: Generate final summary
run: |
python3 .github/scripts/coverage/ci_reports.py render-summary \
--workspace "${{ github.workspace }}" \
--summary-file "${GITHUB_STEP_SUMMARY}" \
--selection "cpu" \
--selected-lanes "cpu"
- name: Merge duration reports
run: |
python3 .github/scripts/coverage/ci_reports.py merge-durations \
--workspace "${{ github.workspace }}" \
--output "${{ github.workspace }}/coverage-test-durations-all.csv"
- name: Upload merged duration artifact
if: ${{ always() }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: coverage-test-durations
path: coverage-test-durations-all.csv
if-no-files-found: warn
retention-days: 14
compression-level: 0
overwrite: true