Add kernel conformance CI workflow #2
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kernel Conformance | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| test-kernel: | |
| name: Test ${{ matrix.kernel.name }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kernel: | |
| - name: ipykernel | |
| install: pip install ipykernel && python -m ipykernel install --user | |
| kernel-name: python3 | |
| - name: evcxr | |
| install: cargo install --locked evcxr_jupyter && evcxr_jupyter --install | |
| kernel-name: rust | |
| - name: deno | |
| install: | | |
| curl -fsSL https://deno.land/install.sh | sh | |
| echo "$HOME/.deno/bin" >> $GITHUB_PATH | |
| $HOME/.deno/bin/deno jupyter --install | |
| kernel-name: deno | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up Python | |
| if: matrix.kernel.name == 'ipykernel' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Cache cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/ | |
| ~/.cargo/git/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install kernel | |
| run: ${{ matrix.kernel.install }} | |
| - name: Build test suite | |
| run: cargo build --release | |
| - name: List available kernels | |
| run: ./target/release/jupyter-kernel-test --list-kernels | |
| - name: Run conformance tests | |
| run: | | |
| ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} \ | |
| --format json \ | |
| --output ${{ matrix.kernel.name }}-report.json | |
| continue-on-error: true | |
| - name: Display results | |
| run: ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} || true | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: report-${{ matrix.kernel.name }} | |
| path: ${{ matrix.kernel.name }}-report.json | |
| if-no-files-found: warn | |
| conformance-matrix: | |
| name: Conformance Matrix | |
| needs: test-kernel | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports | |
| pattern: report-* | |
| merge-multiple: true | |
| - name: Generate matrix summary | |
| run: | | |
| echo "## Kernel Conformance Matrix" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Kernel | Tier 1 | Tier 2 | Tier 3 | Tier 4 | Total |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|--------|--------|--------|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| kernel=$(basename "$report" -report.json) | |
| # Extract pass counts per tier using jq | |
| if command -v jq &> /dev/null; then | |
| tier1=$(jq '[.results[] | select(.category == "tier1_basic") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier1_total=$(jq '[.results[] | select(.category == "tier1_basic")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2=$(jq '[.results[] | select(.category == "tier2_interactive") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2_total=$(jq '[.results[] | select(.category == "tier2_interactive")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3=$(jq '[.results[] | select(.category == "tier3_rich_output") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3_total=$(jq '[.results[] | select(.category == "tier3_rich_output")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4=$(jq '[.results[] | select(.category == "tier4_advanced") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4_total=$(jq '[.results[] | select(.category == "tier4_advanced")] | length' "$report" 2>/dev/null || echo "?") | |
| total=$(jq '[.results[] | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| total_tests=$(jq '.results | length' "$report" 2>/dev/null || echo "?") | |
| echo "| $kernel | $tier1/$tier1_total | $tier2/$tier2_total | $tier3/$tier3_total | $tier4/$tier4_total | $total/$total_tests |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| $kernel | - | - | - | - | - |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| fi | |
| done | |
| - name: Upload combined reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-reports | |
| path: reports/ | |
| if-no-files-found: warn |