Add detailed conformance reports with actionable insights #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kernel Conformance | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| test-kernel: | |
| name: Test ${{ matrix.kernel.name }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kernel: | |
| - name: ipykernel | |
| install: pip install ipykernel && python -m ipykernel install --user | |
| kernel-name: python3 | |
| - name: evcxr | |
| install: cargo install --locked evcxr_jupyter && evcxr_jupyter --install | |
| kernel-name: rust | |
| - name: deno | |
| install: | | |
| curl -fsSL https://deno.land/install.sh | sh | |
| echo "$HOME/.deno/bin" >> $GITHUB_PATH | |
| $HOME/.deno/bin/deno jupyter --install | |
| kernel-name: deno | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up Python | |
| if: matrix.kernel.name == 'ipykernel' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Cache cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/ | |
| ~/.cargo/git/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install kernel | |
| run: ${{ matrix.kernel.install }} | |
| - name: Build test suite | |
| run: cargo build --release | |
| - name: List available kernels | |
| run: ./target/release/jupyter-kernel-test --list-kernels | |
| - name: Run conformance tests | |
| run: | | |
| ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} \ | |
| --format json \ | |
| --output ${{ matrix.kernel.name }}-report.json | |
| continue-on-error: true | |
| - name: Display results | |
| run: ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} || true | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: report-${{ matrix.kernel.name }} | |
| path: ${{ matrix.kernel.name }}-report.json | |
| if-no-files-found: warn | |
| conformance-matrix: | |
| name: Conformance Matrix | |
| needs: test-kernel | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports | |
| pattern: report-* | |
| merge-multiple: true | |
| - name: Generate detailed report | |
| run: | | |
| { | |
| echo "## Kernel Conformance Matrix" | |
| echo "" | |
| echo "| Kernel | Tier 1 | Tier 2 | Tier 3 | Tier 4 | Total |" | |
| echo "|--------|--------|--------|--------|--------|-------|" | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| kernel=$(basename "$report" -report.json) | |
| tier1=$(jq '[.results[] | select(.category == "tier1_basic") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier1_total=$(jq '[.results[] | select(.category == "tier1_basic")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2=$(jq '[.results[] | select(.category == "tier2_interactive") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2_total=$(jq '[.results[] | select(.category == "tier2_interactive")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3=$(jq '[.results[] | select(.category == "tier3_rich_output") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3_total=$(jq '[.results[] | select(.category == "tier3_rich_output")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4=$(jq '[.results[] | select(.category == "tier4_advanced") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4_total=$(jq '[.results[] | select(.category == "tier4_advanced")] | length' "$report" 2>/dev/null || echo "?") | |
| total=$(jq '[.results[] | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| total_tests=$(jq '.results | length' "$report" 2>/dev/null || echo "?") | |
| echo "| $kernel | $tier1/$tier1_total | $tier2/$tier2_total | $tier3/$tier3_total | $tier4/$tier4_total | $total/$total_tests |" | |
| fi | |
| done | |
| echo "" | |
| echo "---" | |
| echo "" | |
| # Detailed failure breakdown per kernel | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| kernel=$(basename "$report" -report.json) | |
| impl=$(jq -r '.implementation // "unknown"' "$report" 2>/dev/null) | |
| lang=$(jq -r '.language // "unknown"' "$report" 2>/dev/null) | |
| protocol=$(jq -r '.protocol_version // "unknown"' "$report" 2>/dev/null) | |
| failures=$(jq '[.results[] | select(.result.status != "pass" and .result.status != "unsupported")] | length' "$report" 2>/dev/null || echo "0") | |
| skipped=$(jq '[.results[] | select(.result.status == "unsupported")] | length' "$report" 2>/dev/null || echo "0") | |
| if [ "$failures" != "0" ] || [ "$skipped" != "0" ]; then | |
| echo "<details>" | |
| echo "<summary><strong>$kernel</strong> ($impl) - $failures failures, $skipped skipped</summary>" | |
| echo "" | |
| echo "**Language:** $lang | **Protocol:** $protocol" | |
| echo "" | |
| if [ "$failures" != "0" ]; then | |
| echo "#### Failures" | |
| echo "" | |
| echo "| Test | Message Type | Likely Source | Reason |" | |
| echo "|------|--------------|---------------|--------|" | |
| jq -r '.results[] | select(.result.status == "fail" or .result.status == "timeout") | "| \(.name) | `\(.message_type)` | \(.result.kind // "unknown") | \(.result.reason // "timeout") |"' "$report" 2>/dev/null | |
| echo "" | |
| fi | |
| if [ "$skipped" != "0" ]; then | |
| echo "#### Skipped (Not Implemented)" | |
| echo "" | |
| jq -r '.results[] | select(.result.status == "unsupported") | "- **\(.name)**: \(.description)"' "$report" 2>/dev/null | |
| echo "" | |
| fi | |
| echo "</details>" | |
| echo "" | |
| fi | |
| fi | |
| done | |
| # Cross-kernel comparison for failing tests | |
| echo "---" | |
| echo "" | |
| echo "### Test Results by Message Type" | |
| echo "" | |
| echo "| Test | Message Type |$(for r in reports/*-report.json; do kernel=$(basename "$r" -report.json); echo -n " $kernel |"; done)" | |
| echo "|------|--------------|$(for r in reports/*-report.json; do echo -n "--------|"; done)" | |
| # Get all unique test names | |
| all_tests=$(jq -r '.results[].name' reports/*-report.json 2>/dev/null | sort -u) | |
| for test in $all_tests; do | |
| msg_type=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .message_type' reports/*-report.json 2>/dev/null | head -1) | |
| echo -n "| $test | \`$msg_type\` |" | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| status=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .result.status' "$report" 2>/dev/null) | |
| case "$status" in | |
| pass) echo -n " ✅ |" ;; | |
| fail) echo -n " ❌ |" ;; | |
| unsupported) echo -n " ⏭️ |" ;; | |
| timeout) echo -n " ⏱️ |" ;; | |
| partial_pass) echo -n " ⚠️ |" ;; | |
| *) echo -n " - |" ;; | |
| esac | |
| fi | |
| done | |
| echo "" | |
| done | |
| } | tee -a $GITHUB_STEP_SUMMARY > summary.md | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh pr comment ${{ github.event.pull_request.number }} --body-file summary.md | |
| - name: Upload combined reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-reports | |
| path: reports/ | |
| if-no-files-found: warn |