diff --git a/.github/workflows/testing-macos.yml b/.github/workflows/testing-macos.yml index 2db562535cf0..7fea62c89cf3 100644 --- a/.github/workflows/testing-macos.yml +++ b/.github/workflows/testing-macos.yml @@ -28,6 +28,27 @@ permissions: contents: read jobs: + coverage: + if: "!contains(github.event.pull_request.labels.*.name, 'skip_buildbots')" + name: arm-64 / coverage + runs-on: macos-26 + + steps: + - uses: actions/checkout@v6 + + - name: Install coverage dependencies + run: brew install llvm@21 lld@21 emscripten flatbuffers wabt + + - name: Run coverage suite + run: tools/run-coverage.sh + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: build/macOS-coverage/coverage.info + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + macos: if: "!contains(github.event.pull_request.labels.*.name, 'skip_buildbots')" name: ${{ matrix.arch }} / ${{ matrix.uv_group }} diff --git a/CMakePresets.json b/CMakePresets.json index b0a50f2f72fe..6692d18b300c 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -202,6 +202,33 @@ "CMAKE_SHARED_LINKER_FLAGS": "-fsanitize=fuzzer" } }, + { + "name": "macOS-coverage", + "displayName": "macOS (Coverage)", + "description": "macOS build with source-based LLVM coverage instrumentation", + "inherits": "macOS", + "toolchainFile": "${sourceDir}/cmake/toolchain.macos-homebrew.cmake", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_FLAGS": "-fprofile-instr-generate -fcoverage-mapping", + "CMAKE_CXX_FLAGS": "-fprofile-instr-generate -fcoverage-mapping", + "CMAKE_EXE_LINKER_FLAGS": "-fprofile-instr-generate", + "CMAKE_MODULE_LINKER_FLAGS": "-fprofile-instr-generate", + "CMAKE_SHARED_LINKER_FLAGS": "-fprofile-instr-generate", + "WITH_TUTORIALS": "NO", + "WITH_PYTHON_BINDINGS": "NO", + "WITH_UTILS": "NO", + "WITH_TEST_AUTO_SCHEDULE": "YES", + "WITH_TEST_ERROR": "YES", + "WITH_TEST_WARNING": "NO", + "WITH_TEST_PERFORMANCE": "NO", + "WITH_TEST_GENERATOR": "YES", + "WITH_TEST_RUNTIME": "NO", + "WITH_TEST_FUZZ": "NO", + "WITH_TEST_CORRECTNESS": "YES", + "WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING": "YES" + } + }, { "name": "macOS-vcpkg", "inherits": [ @@ -242,5 +269,30 @@ "BUILD_SHARED_LIBS": "NO" } } + ], + "buildPresets": [ + { + "name": "macOS-coverage", + "configurePreset": "macOS-coverage", + "displayName": "macOS (Coverage)", + "description": "Build with LLVM_PROFILE_FILE set so generator binaries write profiles to the profiles/ dir", + "environment": { + "LLVM_PROFILE_FILE": "${sourceDir}/build/macOS-coverage/profiles/%p-%m.profraw" + } + } + ], + "testPresets": [ + { + "name": "macOS-coverage", + "configurePreset": "macOS-coverage", + "displayName": "macOS (Coverage)", + "description": "Run correctness tests with LLVM source-based coverage profiling", + "environment": { + "LLVM_PROFILE_FILE": "${sourceDir}/build/macOS-coverage/profiles/%p-%m.profraw" + }, + "output": { + "outputOnFailure": true + } + } ] } diff --git a/cmake/HalideTestHelpers.cmake b/cmake/HalideTestHelpers.cmake index c46a4672d3ce..26fa0d377e8f 100644 --- a/cmake/HalideTestHelpers.cmake +++ b/cmake/HalideTestHelpers.cmake @@ -99,8 +99,8 @@ function(add_halide_test TARGET) VISIBILITY_INLINES_HIDDEN TRUE) - if (WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING) - if (WITH_SERIALIZATION) + if (WITH_SERIALIZATION AND WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING) + if (NOT Halide_TARGET MATCHES "wasm") target_compile_definitions(${TARGET} PRIVATE WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING) endif () endif () diff --git a/test/correctness/simd_op_check.h b/test/correctness/simd_op_check.h index 0fe23f1c1ef9..2dfa6e466c25 100644 --- a/test/correctness/simd_op_check.h +++ b/test/correctness/simd_op_check.h @@ -503,7 +503,14 @@ class SimdOpCheckTest { if (!sharder.should_run(t)) continue; const auto &task = tasks.at(t); futures.push_back(pool.async([&]() { - return check_one(task.op, task.name, task.vector_width, task.expr); + // Run check_one on a large-stack thread to avoid overflowing the + // default 512 KB worker stack during deeply recursive LLVM codegen + // (especially under coverage instrumentation). + TestResult result; + Internal::run_with_large_stack([&] { + result = check_one(task.op, task.name, task.vector_width, task.expr); + }); + return result; })); } diff --git a/tools/run-coverage.sh b/tools/run-coverage.sh new file mode 100755 index 000000000000..1fbd20b8aa31 --- /dev/null +++ b/tools/run-coverage.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Collect source-based LLVM coverage for src/ by running the test suite four times: +# 1. host — all correctness, generator, error, and autoscheduler tests +# 2. host-metal — only tests whose name contains "gpu" or "metal" +# 3. host-opencl — only tests whose name contains "gpu" or "opencl" +# 4. wasm-32-wasmrt-wasm_simd128 — all tests (exercises WasmExecutor for JIT) +# +# Prerequisites: Homebrew LLVM 21 (for llvm-profdata and llvm-cov). +# +# Usage: tools/run-coverage.sh [--report-only] +# --report-only Skip build and test steps; re-merge existing profraw files +# and regenerate all reports. Useful after re-running a single +# test manually. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILD_DIR="$REPO_ROOT/build/macOS-coverage" +LLVM_DIR="/opt/homebrew/opt/llvm@21/bin" +NPROC=$(sysctl -n hw.logicalcpu) + +REPORT_ONLY=0 +for arg in "$@"; do + case "$arg" in + --report-only) REPORT_ONLY=1 ;; + *) + echo "Unknown argument: $arg" >&2 + exit 1 + ;; + esac +done + +cd "$REPO_ROOT" + +if [[ $REPORT_ONLY -eq 0 ]]; then + + mkdir -p "$BUILD_DIR/profiles" + + # ── Configure + Build ───────────────────────────────────────────────────────── + cmake --preset macOS-coverage + cmake --build --preset macOS-coverage + + # ── Run 1: host — all tests ────────────────────────────────────────────────── + ctest --preset macOS-coverage -j"$NPROC" || echo "Warning: some tests failed (see above)" + + # ── Reconfigure + rebuild for host-metal ───────────────────────────────────── + cmake --preset macOS-coverage -DHalide_TARGET=host-metal + cmake --build --preset macOS-coverage + + # ── Run 2: host-metal — GPU tests only ─────────────────────────────────────── + ctest --preset macOS-coverage -j"$NPROC" -R "gpu|metal" || echo "Warning: some tests failed (see above)" + + # ── Reconfigure + rebuild for host-opencl ──────────────────────────────────── + cmake --preset macOS-coverage -DHalide_TARGET=host-opencl + cmake --build --preset macOS-coverage + + # ── Run 3: host-opencl — GPU and OpenCL-specific tests ─────────────────────── + ctest --preset macOS-coverage -j"$NPROC" -R "gpu|opencl" || echo "Warning: some tests failed (see above)" + + # ── Reconfigure + rebuild for wasm-32-wasmrt-wasm_simd128 ──────────────────── + cmake --preset macOS-coverage -DHalide_TARGET=wasm-32-wasmrt-wasm_simd128 + cmake --build --preset macOS-coverage + + # ── Run 4: wasm — all tests (exercises WasmExecutor for JIT compilation) ───── + ctest --preset macOS-coverage -j"$NPROC" || echo "Warning: some tests failed (see above)" + +fi # REPORT_ONLY + +# ── Merge profiles ──────────────────────────────────────────────────────────── +"$LLVM_DIR/llvm-profdata" merge \ + "$BUILD_DIR/profiles"/*.profraw \ + -o "$BUILD_DIR/merged.profdata" + +# ── Generate coverage reports ───────────────────────────────────────────────── +HALIDE_LIB=$(find "$BUILD_DIR/src" -maxdepth 1 -name "libHalide.*.*.*.dylib" | head -1) +OBJECTS=(-object "$HALIDE_LIB") + +while IFS= read -r bin; do + OBJECTS+=(-object "$bin") +done < <( + # Exclude generator_aot_* and generator_aotcpp_*: they run pre-compiled + # Halide pipelines at test time and add no unique src/ coverage. + find "$BUILD_DIR/test" -maxdepth 2 -type f -perm +0111 | + grep -v '/generator_aot_\|/generator_aotcpp_' | sort +) + +COV_ARGS=( + --instr-profile="$BUILD_DIR/merged.profdata" + "${OBJECTS[@]}" + --ignore-filename-regex='build/|test/|tutorial/|python_bindings/|apps/|packaging/|include/' + --sources "$REPO_ROOT/src" +) + +"$LLVM_DIR/llvm-cov" report "${COV_ARGS[@]}" 2>/dev/null >"$BUILD_DIR/coverage-report.txt" +"$LLVM_DIR/llvm-cov" show "${COV_ARGS[@]}" --format=html --output-dir="$BUILD_DIR/coverage-html" 2>/dev/null +"$LLVM_DIR/llvm-cov" export "${COV_ARGS[@]}" --format=lcov 2>/dev/null >"$BUILD_DIR/coverage.info" + +echo "Coverage report: $BUILD_DIR/coverage-report.txt" +echo "LCOV data: $BUILD_DIR/coverage.info" +echo "HTML report: $BUILD_DIR/coverage-html/index.html"