diff --git a/.CMake/compiler_opts.cmake b/.CMake/compiler_opts.cmake index b343311033..a56e157244 100644 --- a/.CMake/compiler_opts.cmake +++ b/.CMake/compiler_opts.cmake @@ -113,6 +113,10 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang") if(${OQS_DEBUG_BUILD}) add_compile_options(-g3) add_compile_options(-fno-omit-frame-pointer) + if(${USE_COVERAGE}) + add_compile_options(-coverage) + add_link_options(-coverage) + endif() if(USE_SANITIZER STREQUAL "Address") add_compile_options(-fno-optimize-sibling-calls) add_compile_options(-fsanitize-address-use-after-scope) @@ -175,6 +179,10 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU") if(${OQS_DEBUG_BUILD}) add_compile_options (-Wstrict-overflow) add_compile_options(-ggdb3) + if(${USE_COVERAGE}) + add_compile_options(-coverage) + add_link_options(-coverage) + endif() else() add_compile_options(-O3) add_compile_options(-fomit-frame-pointer) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml new file mode 100644 index 0000000000..8698886224 --- /dev/null +++ b/.github/workflows/code-coverage.yml @@ -0,0 +1,62 @@ +name: Code coverage tests + +permissions: + contents: read + +on: [workflow_call, workflow_dispatch] + +jobs: + coverage: + name: Run code coverage testing + strategy: + matrix: + # The 'id' value for each job should be added to the 'carry-forward' string in the 'finish' job. + include: + - id: x64-generic + runner: ubuntu-latest + CMAKE_ARGS: -DOQS_DIST_BUILD=OFF -DOQS_OPT_TARGET=generic + - id: x64-distbuild + runner: ubuntu-latest + CMAKE_ARGS: -DOQS_DIST_BUILD=ON + - id: arm64-distbuild + runner: ubuntu-24.04-arm + CMAKE_ARGS: -DOQS_DIST_BUILD=ON + runs-on: ${{ matrix.runner }} + container: openquantumsafe/ci-ubuntu-latest:latest + steps: + - name: Install dependencies + run: apt-get update && apt-get install curl + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4 + - name: Configure + run: | + mkdir build && cd build && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_COVERAGE=ON ${{ matrix.CMAKE_ARGS }} .. && \ + cmake -LA -N .. + - name: Build + run: ninja + working-directory: build + - name: Run tests + run: | + python3 -m pytest --verbose --numprocesses=auto \ + tests/test_acvp_vectors.py \ + tests/test_cmdline.py \ + tests/test_kat.py + - name: Run lcov + run: lcov -d . -c -o lcov.info --exclude /usr/lib,/usr/include --ignore-errors unused + - name: Upload to coveralls.io + uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # pin@v2.3.6 + with: + flag-name: ${{ matrix.id }} + parallel: true + + finish: + needs: coverage + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Finish coveralls.io + uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # pin@v2.3.6 + with: + parallel-finished: true + carry-forward: "x64-generic,x64-distbuild,arm64-distbuild" diff --git a/.github/workflows/commit-to-main.yml b/.github/workflows/commit-to-main.yml index 100598154b..e680ea646f 100644 --- a/.github/workflows/commit-to-main.yml +++ b/.github/workflows/commit-to-main.yml @@ -12,6 +12,10 @@ jobs: platform-tests: uses: ./.github/workflows/platforms.yml + code-coverage: + uses: ./.github/workflows/code-coverage.yml + secrets: inherit + scorecard: uses: ./.github/workflows/scorecard.yml secrets: inherit diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 41776b6e6a..aa840cb496 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,6 +18,11 @@ jobs: needs: basic-checks uses: ./.github/workflows/platforms.yml + code-coverage: + needs: basic-checks + uses: ./.github/workflows/code-coverage.yml + secrets: inherit + scorecard: needs: basic-checks uses: ./.github/workflows/scorecard.yml diff --git a/CI.md b/CI.md index 95ad9c2cb6..dc115b2294 100644 --- a/CI.md +++ b/CI.md @@ -26,12 +26,12 @@ For example, "[full tests] [trigger downstream]" will trigger both the platform #### Pull request workflow (`pr.yml`) This workflow runs on pull requests. -It calls [basic checks](#basic.yml), [platform tests](#platforms.yml) and [scorecard analysis](#scorecard.yml). +It calls [basic checks](#basic.yml), [code coverage tests](#code-coverage.yml), [platform tests](#platforms.yml) and [scorecard analysis](#scorecard.yml). #### Commit-to-main workflow (`commit-to-main.yml`) This workflow runs on pushes to the `main` branch (typically done automatically when a pull request is merged). -It calls [platform tests](#platforms.yml), [scorecard analysis](#scorecard.yml), and [basic downstream tests](#downstream-basic.yml). +It calls [platform tests](#platforms.yml), [code coverage tests](#code-coverage.yml), [scorecard analysis](#scorecard.yml), and [basic downstream tests](#downstream-basic.yml). #### Weekly workflow (`weekly.yml`) @@ -53,6 +53,10 @@ Users with "write" permissions can also trigger them manually via the GitHub web This workflow runs a minimal set of tests that should pass before heavier tests are triggered. +#### Code coverage tests (`code-coverage.yml`) + +This workflow runs code coverage tests and uploads the results to [Coveralls.io](https://coveralls.io/github/SWilson4/liboqs). + #### Individual platform tests (`.yml`) These workflows contain tests for the individual [platforms supported by liboqs](PLATFORMS.md). diff --git a/CONFIGURE.md b/CONFIGURE.md index 02a8da2015..70b5cc47c1 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -16,6 +16,7 @@ The following options can be passed to CMake before the build file generation pr - [OQS_USE_CUPQC](#OQS_USE_CUPQC) - [OQS_OPT_TARGET](#OQS_OPT_TARGET) - [OQS_SPEED_USE_ARM_PMU](#OQS_SPEED_USE_ARM_PMU) +- [USE_COVERAGE](#USE_COVERAGE) - [USE_SANITIZER](#USE_SANITIZER) - [OQS_ENABLE_TEST_CONSTANT_TIME](#OQS_ENABLE_TEST_CONSTANT_TIME) - [OQS_STRICT_WARNINGS](#OQS_STRICT_WARNINGS) @@ -36,7 +37,9 @@ This means liboqs is built as a static library by default. Can be set to the following values: -- `Debug`: This turns off all compiler optimizations and produces debugging information. When the compiler is Clang, the [USE_SANITIZER](#USE_SANITIZER) option can also be specified to enable a Clang sanitizer. **This value only has effect when the compiler is GCC or Clang** +- `Debug`: This turns off all compiler optimizations and produces debugging information. **This value only has effect when the compiler is GCC or Clang** + - The [USE_COVERAGE](#USE_COVERAGE) option can also be specified to enable code coverage testing. + - When the compiler is Clang, the [USE_SANITIZER](#USE_SANITIZER) option can also be specified to enable a Clang sanitizer. - `Release`: This compiles code at the `O3` optimization level, and sets other compiler flags that reduce the size of the binary. @@ -173,6 +176,12 @@ Note that this option is not known to work on Apple M1 chips. **Default**: `OFF`. +## USE_COVERAGE + +This has an effect when the compiler is GCC or Clang and when [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) is `Debug`. Can be `ON` or `OFF`. When `ON`, code coverage testing will be enabled. + +**Default**: Unset. + ## USE_SANITIZER This has an effect when the compiler is Clang and when [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) is `Debug`. Then, it can be set to: diff --git a/README.md b/README.md index 51851b1af1..20bf5e3417 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ liboqs [![Main Branch Tests](https://github.com/open-quantum-safe/liboqs/actions/workflows/commit-to-main.yml/badge.svg)](https://github.com/open-quantum-safe/liboqs/actions/workflows/commit-to-main.yml) [![Weekly Tests](https://github.com/open-quantum-safe/liboqs/actions/workflows/weekly.yml/badge.svg)](https://github.com/open-quantum-safe/liboqs/actions/workflows/weekly.yml) ![Travis Build Status](https://img.shields.io/travis/com/open-quantum-safe/liboqs?logo=travis&label=Travis%20CI&labelColor=%23343B42&color=%232EBB4E) +[![Coverage Status](https://coveralls.io/repos/github/SWilson4/liboqs/badge.svg?branch=sw-code-coverage)](https://coveralls.io/github/SWilson4/liboqs?branch=sw-code-coverage) liboqs is an open source C library for quantum-safe cryptographic algorithms. diff --git a/src/oqsconfig.h.cmake b/src/oqsconfig.h.cmake index d21a67f77b..84e49898fc 100644 --- a/src/oqsconfig.h.cmake +++ b/src/oqsconfig.h.cmake @@ -30,6 +30,7 @@ #cmakedefine BUILD_SHARED_LIBS 1 #cmakedefine OQS_BUILD_ONLY_LIB 1 #cmakedefine OQS_OPT_TARGET "@OQS_OPT_TARGET@" +#cmakedefine USE_COVERAGE 1 #cmakedefine USE_SANITIZER "@USE_SANITIZER@" #cmakedefine CMAKE_BUILD_TYPE "@CMAKE_BUILD_TYPE@" diff --git a/tests/system_info.c b/tests/system_info.c index 37689dc20d..7b52ffe7a0 100644 --- a/tests/system_info.c +++ b/tests/system_info.c @@ -216,6 +216,7 @@ static void print_oqs_configuration(void) { * OQS_OPT_TARGET: Visible by looking at compile options (-march or -mcpu): * 'auto' -> "-march|cpu=native" * OQS_SPEED_USE_ARM_PMU: Output with Target platform + * USE_COVERAGE: --coverage option present in compile options * USE_SANITIZER: -fsanitize= option present in compile options * OQS_ENABLE_TEST_CONSTANT_TIME: only shown below */ @@ -273,6 +274,9 @@ static void print_oqs_configuration(void) { #ifdef OQS_BUILD_ONLY_LIB printf("OQS_BUILD_ONLY_LIB "); // pretty much impossible to appear but added for completeness #endif +#ifdef USE_COVERAGE + printf("USE_COVERAGE " ); +#endif #ifdef USE_SANITIZER printf("USE_SANITIZER=%s ", USE_SANITIZER); #endif