Use fix branch for KernelInfoReply compatibility (#10) #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kernel Conformance | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| test-kernel: | |
| name: Test ${{ matrix.kernel.name }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kernel: | |
| - name: ipykernel | |
| install: pip install ipykernel && python -m ipykernel install --user | |
| kernel-name: python3 | |
| - name: evcxr | |
| install: cargo install --locked evcxr_jupyter && evcxr_jupyter --install | |
| kernel-name: rust | |
| - name: deno | |
| install: | | |
| curl -fsSL https://deno.land/install.sh | sh | |
| echo "$HOME/.deno/bin" >> $GITHUB_PATH | |
| $HOME/.deno/bin/deno jupyter --install | |
| kernel-name: deno | |
| - name: ark | |
| install: | | |
| # Download Ark binary from GitHub releases | |
| curl -fsSL -o ark.zip https://github.com/posit-dev/ark/releases/download/0.1.232/ark-0.1.232-linux-x64.zip | |
| unzip -o ark.zip | |
| chmod +x ark | |
| sudo mv ark /usr/local/bin/ | |
| # Install kernel | |
| ark --install | |
| kernel-name: ark | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up Python | |
| if: matrix.kernel.name == 'ipykernel' || matrix.kernel.name == 'ark' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up R | |
| if: matrix.kernel.name == 'ark' | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: "4.4" | |
| - name: Install jupyter for Ark | |
| if: matrix.kernel.name == 'ark' | |
| run: python -m pip install jupyter | |
| - name: Cache cargo | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/ | |
| ~/.cargo/git/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install kernel | |
| run: ${{ matrix.kernel.install }} | |
| - name: Build test suite | |
| run: cargo build --release | |
| - name: List available kernels | |
| run: ./target/release/jupyter-kernel-test --list-kernels | |
| - name: Run conformance tests | |
| run: | | |
| ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} \ | |
| --format json \ | |
| --output ${{ matrix.kernel.name }}-report.json | |
| continue-on-error: true | |
| - name: Display results | |
| run: ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} || true | |
| - name: Upload report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: report-${{ matrix.kernel.name }} | |
| path: ${{ matrix.kernel.name }}-report.json | |
| if-no-files-found: warn | |
| conformance-matrix: | |
| name: Conformance Matrix | |
| needs: test-kernel | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports | |
| pattern: report-* | |
| merge-multiple: true | |
| - name: Generate detailed report | |
| run: | | |
| { | |
| echo "## Kernel Conformance Matrix" | |
| echo "" | |
| echo "| Kernel | Tier 1 | Tier 2 | Tier 3 | Tier 4 | Total |" | |
| echo "|--------|--------|--------|--------|--------|-------|" | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| kernel=$(basename "$report" -report.json) | |
| tier1=$(jq '[.results[] | select(.category == "tier1_basic") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier1_total=$(jq '[.results[] | select(.category == "tier1_basic")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2=$(jq '[.results[] | select(.category == "tier2_interactive") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier2_total=$(jq '[.results[] | select(.category == "tier2_interactive")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3=$(jq '[.results[] | select(.category == "tier3_rich_output") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier3_total=$(jq '[.results[] | select(.category == "tier3_rich_output")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4=$(jq '[.results[] | select(.category == "tier4_advanced") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| tier4_total=$(jq '[.results[] | select(.category == "tier4_advanced")] | length' "$report" 2>/dev/null || echo "?") | |
| total=$(jq '[.results[] | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?") | |
| total_tests=$(jq '.results | length' "$report" 2>/dev/null || echo "?") | |
| echo "| $kernel | $tier1/$tier1_total | $tier2/$tier2_total | $tier3/$tier3_total | $tier4/$tier4_total | $total/$total_tests |" | |
| fi | |
| done | |
| echo "" | |
| echo "---" | |
| echo "" | |
| # Detailed failure breakdown per kernel | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| kernel=$(basename "$report" -report.json) | |
| impl=$(jq -r '.implementation // "unknown"' "$report" 2>/dev/null) | |
| lang=$(jq -r '.language // "unknown"' "$report" 2>/dev/null) | |
| protocol=$(jq -r '.protocol_version // "unknown"' "$report" 2>/dev/null) | |
| failures=$(jq '[.results[] | select(.result.status != "pass" and .result.status != "unsupported")] | length' "$report" 2>/dev/null || echo "0") | |
| skipped=$(jq '[.results[] | select(.result.status == "unsupported")] | length' "$report" 2>/dev/null || echo "0") | |
| if [ "$failures" != "0" ] || [ "$skipped" != "0" ]; then | |
| echo "<details>" | |
| echo "<summary><strong>$kernel</strong> ($impl) - $failures failures, $skipped skipped</summary>" | |
| echo "" | |
| echo "**Language:** $lang | **Protocol:** $protocol" | |
| echo "" | |
| if [ "$failures" != "0" ]; then | |
| echo "#### Failures" | |
| echo "" | |
| echo "| Test | Message Type | Likely Source | Reason |" | |
| echo "|------|--------------|---------------|--------|" | |
| jq -r '.results[] | select(.result.status == "fail" or .result.status == "timeout") | "| \(.name) | `\(.message_type)` | \(.result.kind // "unknown") | \(.result.reason // "timeout") |"' "$report" 2>/dev/null | |
| echo "" | |
| fi | |
| if [ "$skipped" != "0" ]; then | |
| echo "#### Skipped (Not Implemented)" | |
| echo "" | |
| jq -r '.results[] | select(.result.status == "unsupported") | "- **\(.name)**: \(.description)"' "$report" 2>/dev/null | |
| echo "" | |
| fi | |
| echo "</details>" | |
| echo "" | |
| fi | |
| fi | |
| done | |
| # Cross-kernel comparison for failing tests | |
| echo "---" | |
| echo "" | |
| echo "### Test Results by Message Type" | |
| echo "" | |
| echo "| Test | Message Type |$(for r in reports/*-report.json; do kernel=$(basename "$r" -report.json); echo -n " $kernel |"; done)" | |
| echo "|------|--------------|$(for r in reports/*-report.json; do echo -n "--------|"; done)" | |
| # Get all unique test names | |
| all_tests=$(jq -r '.results[].name' reports/*-report.json 2>/dev/null | sort -u) | |
| for test in $all_tests; do | |
| msg_type=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .message_type' reports/*-report.json 2>/dev/null | head -1) | |
| echo -n "| $test | \`$msg_type\` |" | |
| for report in reports/*-report.json; do | |
| if [ -f "$report" ]; then | |
| status=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .result.status' "$report" 2>/dev/null) | |
| case "$status" in | |
| pass) echo -n " ✅ |" ;; | |
| fail) echo -n " ❌ |" ;; | |
| unsupported) echo -n " ⏭️ |" ;; | |
| timeout) echo -n " ⏱️ |" ;; | |
| partial_pass) echo -n " ⚠️ |" ;; | |
| *) echo -n " - |" ;; | |
| esac | |
| fi | |
| done | |
| echo "" | |
| done | |
| } | tee -a $GITHUB_STEP_SUMMARY > summary.md | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh pr comment ${{ github.event.pull_request.number }} --body-file summary.md | |
| - name: Upload combined reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-reports | |
| path: reports/ | |
| if-no-files-found: warn |