diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..c8d62b5 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,66 @@ +name: Coverage + +on: + push: + branches: [main] + pull_request: + +jobs: + coverage: + runs-on: ubuntu-latest + env: + CTEST_OUTPUT_ON_FAILURE: 1 + LLVM_PROFILE_FILE: build/profiles/%p.profraw + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang llvm ninja-build + + - name: Configure + run: | + cmake -S . -B build -G Ninja \ + -DBUILD_TESTING=ON \ + -DTSD_ENABLE_COVERAGE=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ + + - name: Build + run: cmake --build build --config Debug + + - name: Run tests + run: | + mkdir -p build/profiles + ctest --test-dir build --output-on-failure + + - name: Generate coverage report + run: | + shopt -s nullglob + profiles=(build/profiles/*.profraw) + if [ ${#profiles[@]} -eq 0 ]; then + echo "No coverage profiles were generated" >&2 + exit 1 + fi + llvm-profdata merge -sparse "${profiles[@]}" -o build/coverage.profdata + llvm-cov export \ + --format=lcov \ + --instr-profile=build/coverage.profdata \ + build/test_config_parser \ + build/test_statistics \ + build/test_thermal_simd > build/lcov.info + llvm-cov report \ + --instr-profile=build/coverage.profdata \ + build/test_config_parser \ + build/test_statistics \ + build/test_thermal_simd > build/coverage.txt + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + build/coverage.txt + build/lcov.info diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml new file mode 100644 index 0000000..89628a7 --- /dev/null +++ b/.github/workflows/perf-gate.yml @@ -0,0 +1,44 @@ +name: perf-gate + +on: + workflow_call: + inputs: + baseline: + description: Path to the baseline file + required: false + default: tests/perf_smoke.baseline + type: string + threshold: + description: Maximum allowed regression ratio + required: false + default: 0.6 + type: number + +jobs: + perf: + runs-on: ubuntu-latest + env: + CTEST_OUTPUT_ON_FAILURE: 1 + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build + + - name: Configure + run: | + cmake -S . -B build -G Ninja \ + -DBUILD_TESTING=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + + - name: Build perf smoke benchmark + run: cmake --build build --target perf_smoke --config RelWithDebInfo + + - name: Run perf smoke gate + run: | + python3 tools/perf_gate.py \ + --baseline "${{ inputs.baseline }}" \ + --threshold "${{ inputs.threshold }}" \ + -- ./build/perf_smoke diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..7067bc9 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,13 @@ +name: Perf Regression Gate + +on: + push: + branches: [main] + pull_request: + +jobs: + perf-smoke: + uses: ./.github/workflows/perf-gate.yml + with: + baseline: tests/perf_smoke.baseline + threshold: 0.6 diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml new file mode 100644 index 0000000..3076172 --- /dev/null +++ b/.github/workflows/sanitizers.yml @@ -0,0 +1,45 @@ +name: Sanitizer Builds + +on: + push: + branches: [main] + pull_request: + +jobs: + sanitizers: + name: ${{ matrix.compiler }}-${{ matrix.sanitizer }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + compiler: [gcc, clang] + sanitizer: [asan, ubsan, tsan] + env: + CTEST_OUTPUT_ON_FAILURE: 1 + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build + + - name: Configure + run: | + if [ "${{ matrix.compiler }}" = "gcc" ]; then + cxx_compiler=g++ + else + cxx_compiler=clang++ + fi + cmake -S . -B build -G Ninja \ + -DBUILD_TESTING=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DTSD_SANITIZER=${{ matrix.sanitizer }} \ + -DCMAKE_C_COMPILER=${{ matrix.compiler }} \ + -DCMAKE_CXX_COMPILER=${cxx_compiler} + + - name: Build + run: cmake --build build --config RelWithDebInfo + + - name: Run tests + run: ctest --test-dir build --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fa61ee..2f45492 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,47 @@ set(THERMAL_SIMD_DISPATCHER_CPU_FLAGS "-msse4.1" CACHE STRING "CPU-specific comp include(CTest) +option(TSD_ENABLE_COVERAGE "Enable coverage instrumentation" OFF) +set(TSD_SANITIZER "" CACHE STRING "Sanitizer to enable (asan, ubsan, tsan)") + +if(TSD_ENABLE_COVERAGE) + add_compile_options(-fprofile-instr-generate -fcoverage-mapping) + add_link_options(-fprofile-instr-generate -fcoverage-mapping) +endif() + +if(TSD_SANITIZER) + string(TOLOWER "${TSD_SANITIZER}" _tsd_sanitizer_kind) + if(_tsd_sanitizer_kind STREQUAL "asan") + set(_tsd_sanitizer_flag "address") + elseif(_tsd_sanitizer_kind STREQUAL "ubsan") + set(_tsd_sanitizer_flag "undefined") + elseif(_tsd_sanitizer_kind STREQUAL "tsan") + set(_tsd_sanitizer_flag "thread") + elseif(_tsd_sanitizer_kind STREQUAL "address" OR _tsd_sanitizer_kind STREQUAL "undefined" OR _tsd_sanitizer_kind STREQUAL "thread") + set(_tsd_sanitizer_flag "${_tsd_sanitizer_kind}") + else() + message(FATAL_ERROR "Unsupported sanitizer '${TSD_SANITIZER}'. Use asan, ubsan or tsan.") + endif() + + set(_tsd_sanitizer_flags "-fsanitize=${_tsd_sanitizer_flag}") + add_compile_options(${_tsd_sanitizer_flags} -fno-omit-frame-pointer) + add_link_options(${_tsd_sanitizer_flags}) + + if(_tsd_sanitizer_kind STREQUAL "ubsan" OR _tsd_sanitizer_kind STREQUAL "undefined") + add_compile_options(-fno-sanitize-recover=undefined) + add_link_options(-fno-sanitize-recover=undefined) + endif() + + if(_tsd_sanitizer_kind STREQUAL "asan" OR _tsd_sanitizer_kind STREQUAL "address") + add_link_options(-shared-libasan) + endif() + + if(_tsd_sanitizer_kind STREQUAL "tsan" OR _tsd_sanitizer_kind STREQUAL "thread") + add_compile_options(-fPIE) + add_link_options(-pie) + endif() +endif() + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/thermal/simd) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/thermal/simd/version.h.in @@ -66,6 +107,11 @@ if(BUILD_TESTING) target_compile_options(test_thermal_simd PRIVATE -Wall -Wextra -O1 -pthread -fPIC) target_include_directories(test_thermal_simd PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) add_test(NAME thermal_simd COMMAND test_thermal_simd) + + add_executable(perf_smoke tests/perf_smoke.c) + target_link_libraries(perf_smoke PRIVATE thermal_simd_core_tests pthread m) + target_compile_definitions(perf_smoke PRIVATE TSD_ENABLE_TESTS) + target_compile_options(perf_smoke PRIVATE -Wall -Wextra -Wpedantic) endif() install(TARGETS thermal_simd_core diff --git a/tests/perf_smoke.baseline b/tests/perf_smoke.baseline new file mode 100644 index 0000000..67118e0 --- /dev/null +++ b/tests/perf_smoke.baseline @@ -0,0 +1 @@ +0.150 diff --git a/tests/perf_smoke.c b/tests/perf_smoke.c new file mode 100644 index 0000000..44b372f --- /dev/null +++ b/tests/perf_smoke.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#define WORKLOAD_ITERS 1024 +#define WARMUP_ITERATIONS 8 +#define MEASURED_ITERATIONS 64 + +static void smoke_workload(void) { + for (int i = 0; i < WORKLOAD_ITERS; ++i) { + atomic_fetch_add_explicit(&g_tsd_workload_iterations, (uint64_t)1, memory_order_relaxed); + } +} + +int main(void) { + if (setenv("TSD_FAKE_PERF", "1", 1) != 0) { + perror("setenv"); + return 1; + } + + tsd_runtime_config cfg; + tsd_runtime_config_set_defaults(&cfg); + cfg.work_iters = WORKLOAD_ITERS; + tsd_runtime_config_refresh_ticks(&cfg); + + const uint32_t fake_ratios[] = {1000, 950, 925, 900, 875, 850}; + tsd_perf_set_fake_script(fake_ratios, sizeof(fake_ratios) / sizeof(fake_ratios[0]), 1200); + + perf_ctx_t *ctx = tsd_perf_init(smoke_workload); + if (!ctx) { + fprintf(stderr, "failed to initialise perf context\n"); + return 1; + } + + tsd_perf_measure_baseline(ctx, &cfg); + + tsd_thermal_eval_t eval = {0}; + for (int i = 0; i < WARMUP_ITERATIONS; ++i) { + (void)tsd_perf_evaluate(ctx, &eval, &cfg); + } + + struct timespec start = {0}, end = {0}; + clock_gettime(CLOCK_MONOTONIC, &start); + for (int i = 0; i < MEASURED_ITERATIONS; ++i) { + (void)tsd_perf_evaluate(ctx, &eval, &cfg); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + const double start_us = (double)start.tv_sec * 1e6 + (double)start.tv_nsec / 1e3; + const double end_us = (double)end.tv_sec * 1e6 + (double)end.tv_nsec / 1e3; + const double elapsed = fmax(end_us - start_us, 1.0); + const double per_eval = elapsed / (double)MEASURED_ITERATIONS; + + tsd_perf_clear_fake_script(); + tsd_perf_cleanup(ctx); + + printf("PERF_SMOKE_PER_EVAL_US=%.3f\n", per_eval); + printf("PERF_SMOKE_ITERATIONS=%d\n", MEASURED_ITERATIONS); + + return 0; +} diff --git a/tools/perf_gate.py b/tools/perf_gate.py new file mode 100755 index 0000000..4a53909 --- /dev/null +++ b/tools/perf_gate.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Simple performance gate for the perf_smoke benchmark.""" + +from __future__ import annotations + +import argparse +import pathlib +import subprocess +import sys +from typing import List + + +def parse_args(argv: List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--baseline", + required=True, + type=pathlib.Path, + help="Path to the baseline file containing the expected microseconds per evaluation.", + ) + parser.add_argument( + "--threshold", + type=float, + default=0.20, + help="Maximum allowed relative regression (e.g. 0.15 for 15%%).", + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="Command to run (prefix with -- to terminate the argument parser).", + ) + args = parser.parse_args(argv) + if not args.command: + parser.error("a benchmark command must be provided after --") + if args.command[0] == "--": + args.command = args.command[1:] + if not args.command: + parser.error("a benchmark command must be provided after --") + return args + + +def read_baseline(path: pathlib.Path) -> float: + try: + contents = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise SystemExit(f"failed to read baseline '{path}': {exc}") from exc + try: + return float(contents) + except ValueError as exc: + raise SystemExit(f"baseline '{path}' does not contain a valid float") from exc + + +def extract_metric(stdout: str) -> float: + for line in stdout.splitlines(): + if line.startswith("PERF_SMOKE_PER_EVAL_US="): + try: + return float(line.split("=", 1)[1]) + except ValueError as exc: + raise SystemExit("failed to parse benchmark output") from exc + raise SystemExit("benchmark output did not contain PERF_SMOKE_PER_EVAL_US") + + +def main(argv: List[str]) -> int: + args = parse_args(argv) + baseline = read_baseline(args.baseline) + + result = subprocess.run(args.command, capture_output=True, text=True) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + if result.returncode != 0: + return result.returncode + + measured = extract_metric(result.stdout) + if baseline <= 0: + raise SystemExit("baseline value must be positive") + + regression = (measured - baseline) / baseline + print(f"Recorded perf_smoke: {measured:.3f} us (baseline {baseline:.3f} us)") + print(f"Relative change: {regression * 100.0:.2f}% (threshold {args.threshold * 100.0:.2f}%)") + + if regression > args.threshold: + print( + f"Regression of {regression * 100.0:.2f}% exceeds allowed threshold", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))