ci: add error handling to coverage report script and explicit permiss… #134
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: ["**"] | |
| pull_request: | |
| branches: [main] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| RUSTC_WRAPPER: "" | |
| jobs: | |
| test: | |
| name: Test (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov | |
| - name: Cache cargo registry & build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Run tests with coverage | |
| run: cargo llvm-cov --lib --test unit --test protocol -p cake-core --json --output-path coverage.json | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.os }}-cpu | |
| path: coverage.json | |
| test-android: | |
| name: Test (Android aarch64) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| env: | |
| ANDROID_NDK_VERSION: r27c | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: aarch64-linux-android | |
| components: clippy, llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov | |
| - name: Set up Android NDK ${{ env.ANDROID_NDK_VERSION }} | |
| id: setup-ndk | |
| uses: nttld/setup-ndk@v1 | |
| with: | |
| ndk-version: ${{ env.ANDROID_NDK_VERSION }} | |
| add-to-path: false | |
| - name: Cache cargo registry & build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: android-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: android-cargo- | |
| - name: Cache cargo-ndk binary | |
| id: cache-cargo-ndk | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cargo/bin/cargo-ndk | |
| key: cargo-ndk-bin-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-ndk | |
| if: steps.cache-cargo-ndk.outputs.cache-hit != 'true' | |
| run: cargo install cargo-ndk --locked | |
| - name: Build cake-mobile (Android aarch64) | |
| env: | |
| ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} | |
| run: cargo ndk -t aarch64-linux-android build -p cake-mobile | |
| - name: Clippy (cake-core + cake-mobile, Android) | |
| env: | |
| ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} | |
| run: cargo ndk -t aarch64-linux-android clippy -p cake-core --lib --no-default-features --features "llama,qwen2,qwen3_5" -p cake-mobile -- -D warnings | |
| - name: Run tests with coverage | |
| run: cargo llvm-cov --lib --test unit --test protocol -p cake-core --json --output-path coverage.json | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-android-cpu | |
| path: coverage.json | |
| test-ios: | |
| name: Test (iOS arm64) | |
| runs-on: macos-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: aarch64-apple-ios | |
| components: clippy, llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov | |
| - name: Cache cargo registry & build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ios-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ios-cargo- | |
| - name: Build cake-mobile (iOS arm64) | |
| env: | |
| IPHONEOS_DEPLOYMENT_TARGET: "16.0" | |
| run: cargo build --target=aarch64-apple-ios -p cake-mobile --features metal | |
| - name: Clippy (cake-core + cake-mobile, iOS) | |
| env: | |
| IPHONEOS_DEPLOYMENT_TARGET: "16.0" | |
| run: cargo clippy --target=aarch64-apple-ios -p cake-core --lib --no-default-features --features "llama,qwen2,qwen3_5,metal" -p cake-mobile --features metal -- -D warnings | |
| - name: Run tests with coverage (Metal) | |
| run: cargo llvm-cov --lib --test unit --test protocol -p cake-core --features metal --json --output-path coverage.json | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-ios-metal | |
| path: coverage.json | |
| test-features: | |
| name: Test (${{ matrix.name }}) | |
| runs-on: ${{ matrix.os }} | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: linux-cuda | |
| os: ubuntu-latest | |
| features: cuda | |
| cuda: "12.4.0" | |
| compute_cap: "89" | |
| - name: linux-flash-attn | |
| os: ubuntu-latest | |
| features: flash-attn | |
| cuda: "12.4.0" | |
| compute_cap: "89" | |
| - name: linux-vulkan | |
| os: ubuntu-latest | |
| features: vulkan | |
| cuda: "" | |
| compute_cap: "" | |
| - name: linux-rocm | |
| os: ubuntu-latest | |
| features: rocm | |
| cuda: "" | |
| compute_cap: "" | |
| - name: macos-metal | |
| os: macos-latest | |
| features: metal | |
| cuda: "" | |
| compute_cap: "" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: clippy, llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-llvm-cov | |
| - name: Install system dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y lld | |
| - name: Install CUDA toolkit | |
| if: matrix.cuda != '' | |
| uses: Jimver/cuda-toolkit@v0.2.30 | |
| with: | |
| cuda: ${{ matrix.cuda }} | |
| method: local | |
| linux-local-args: '["--toolkit"]' | |
| log-file-suffix: '${{ matrix.name }}.txt' | |
| - name: Create CUDA runtime stub | |
| if: matrix.cuda != '' | |
| run: | | |
| STUBS_DIR="${CUDA_PATH}/lib64/stubs" | |
| sudo ln -s "${STUBS_DIR}/libcuda.so" "${STUBS_DIR}/libcuda.so.1" | |
| echo "LD_LIBRARY_PATH=${STUBS_DIR}:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" | |
| - name: Cache cargo registry & build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ matrix.name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo-${{ matrix.name }}- | |
| - name: Run tests with coverage | |
| env: | |
| CUDA_COMPUTE_CAP: ${{ matrix.compute_cap }} | |
| run: cargo llvm-cov --lib --test unit --test protocol -p cake-core --features "${{ matrix.features }}" --json --output-path coverage.json | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.name }} | |
| path: coverage.json | |
| clippy: | |
| name: Clippy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: clippy | |
| - name: Cache cargo registry & build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo-clippy- | |
| - name: Clippy (cake-core + cake-cli, all features) | |
| run: cargo clippy -p cake-core --lib --tests -p cake-cli --features "vulkan,rocm" -- -D warnings | |
| coverage-report: | |
| name: Coverage Report | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| needs: [test, test-android, test-ios, test-features] | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Download coverage artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: coverage-* | |
| path: coverage-artifacts | |
| - name: Generate coverage table | |
| run: | | |
| python3 << 'PYEOF' | |
| import json, os, sys | |
| artifact_dir = 'coverage-artifacts' | |
| # NOTE: artifact names must match the upload step names in each test job. | |
| # If matrix.os values or artifact names change, update this map accordingly. | |
| name_map = { | |
| 'coverage-ubuntu-latest-cpu': ('Linux', 'CPU'), | |
| 'coverage-macos-latest-cpu': ('macOS', 'CPU'), | |
| 'coverage-windows-latest-cpu': ('Windows', 'CPU'), | |
| 'coverage-android-cpu': ('Linux', 'CPU (Android host)'), | |
| 'coverage-ios-metal': ('macOS', 'Metal (iOS host)'), | |
| 'coverage-linux-cuda': ('Linux', 'CUDA'), | |
| 'coverage-linux-flash-attn': ('Linux', 'Flash Attention'), | |
| 'coverage-linux-vulkan': ('Linux', 'Vulkan'), | |
| 'coverage-linux-rocm': ('Linux', 'ROCm'), | |
| 'coverage-macos-metal': ('macOS', 'Metal'), | |
| } | |
| rows = [] | |
| if os.path.isdir(artifact_dir): | |
| for dirname in sorted(os.listdir(artifact_dir)): | |
| cov_file = os.path.join(artifact_dir, dirname, 'coverage.json') | |
| if not os.path.isfile(cov_file): | |
| continue | |
| try: | |
| with open(cov_file) as f: | |
| data = json.load(f) | |
| totals = data['data'][0]['totals'] | |
| except (json.JSONDecodeError, KeyError, IndexError) as e: | |
| print(f"WARNING: skipping {dirname}: {e}", file=sys.stderr) | |
| continue | |
| if dirname not in name_map: | |
| print(f"WARNING: unmapped artifact '{dirname}', using raw name", file=sys.stderr) | |
| platform, accel = name_map.get(dirname, (dirname, 'Unknown')) | |
| rows.append((platform, accel, totals['lines'], totals['functions'], totals['regions'])) | |
| summary = os.environ.get('GITHUB_STEP_SUMMARY', '/dev/stdout') | |
| with open(summary, 'a') as out: | |
| out.write('## 📊 Code Coverage by Platform & Acceleration\n\n') | |
| if rows: | |
| out.write('| Platform | Acceleration | Line Coverage | Function Coverage | Region Coverage |\n') | |
| out.write('|----------|-------------|---------------|-------------------|------------------|\n') | |
| for platform, accel, lines, funcs, regions in rows: | |
| lp = f"{lines['percent']:.1f}% ({lines['covered']}/{lines['count']})" | |
| fp = f"{funcs['percent']:.1f}% ({funcs['covered']}/{funcs['count']})" | |
| rp = f"{regions['percent']:.1f}% ({regions['covered']}/{regions['count']})" | |
| out.write(f'| {platform} | {accel} | {lp} | {fp} | {rp} |\n') | |
| else: | |
| out.write('No coverage data available.\n') | |
| PYEOF |