conformance.yml

name: Kernel Conformance

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

permissions:
  contents: read
  pull-requests: write

jobs:
  test-kernel:
    name: Test ${{ matrix.kernel.name }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        kernel:
          - name: ipykernel
            install: pip install ipykernel && python -m ipykernel install --user
            kernel-name: python3
          - name: evcxr
            install: cargo install --locked evcxr_jupyter && evcxr_jupyter --install
            kernel-name: rust
          - name: deno
            install: |
              curl -fsSL https://deno.land/install.sh | sh
              echo "$HOME/.deno/bin" >> $GITHUB_PATH
              $HOME/.deno/bin/deno jupyter --install
            kernel-name: deno

    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Set up Python
        if: matrix.kernel.name == 'ipykernel'
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Cache cargo
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/
            ~/.cargo/git/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Install kernel
        run: ${{ matrix.kernel.install }}

      - name: Build test suite
        run: cargo build --release

      - name: List available kernels
        run: ./target/release/jupyter-kernel-test --list-kernels

      - name: Run conformance tests
        run: |
          ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} \
            --format json \
            --output ${{ matrix.kernel.name }}-report.json
        continue-on-error: true

      - name: Display results
        run: ./target/release/jupyter-kernel-test ${{ matrix.kernel.kernel-name }} || true

      - name: Upload report
        uses: actions/upload-artifact@v4
        with:
          name: report-${{ matrix.kernel.name }}
          path: ${{ matrix.kernel.name }}-report.json
          if-no-files-found: warn

  conformance-matrix:
    name: Conformance Matrix
    needs: test-kernel
    runs-on: ubuntu-latest
    if: always()

    steps:
      - uses: actions/checkout@v4

      - name: Download all reports
        uses: actions/download-artifact@v4
        with:
          path: reports
          pattern: report-*
          merge-multiple: true

      - name: Generate detailed report
        run: |
          {
            echo "## Kernel Conformance Matrix"
            echo ""
            echo "| Kernel | Tier 1 | Tier 2 | Tier 3 | Tier 4 | Total |"
            echo "|--------|--------|--------|--------|--------|-------|"

            for report in reports/*-report.json; do
              if [ -f "$report" ]; then
                kernel=$(basename "$report" -report.json)
                tier1=$(jq '[.results[] | select(.category == "tier1_basic") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?")
                tier1_total=$(jq '[.results[] | select(.category == "tier1_basic")] | length' "$report" 2>/dev/null || echo "?")
                tier2=$(jq '[.results[] | select(.category == "tier2_interactive") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?")
                tier2_total=$(jq '[.results[] | select(.category == "tier2_interactive")] | length' "$report" 2>/dev/null || echo "?")
                tier3=$(jq '[.results[] | select(.category == "tier3_rich_output") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?")
                tier3_total=$(jq '[.results[] | select(.category == "tier3_rich_output")] | length' "$report" 2>/dev/null || echo "?")
                tier4=$(jq '[.results[] | select(.category == "tier4_advanced") | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?")
                tier4_total=$(jq '[.results[] | select(.category == "tier4_advanced")] | length' "$report" 2>/dev/null || echo "?")
                total=$(jq '[.results[] | select(.result.status == "pass")] | length' "$report" 2>/dev/null || echo "?")
                total_tests=$(jq '.results | length' "$report" 2>/dev/null || echo "?")
                echo "| $kernel | $tier1/$tier1_total | $tier2/$tier2_total | $tier3/$tier3_total | $tier4/$tier4_total | $total/$total_tests |"
              fi
            done

            echo ""
            echo "---"
            echo ""

            # Detailed failure breakdown per kernel
            for report in reports/*-report.json; do
              if [ -f "$report" ]; then
                kernel=$(basename "$report" -report.json)
                impl=$(jq -r '.implementation // "unknown"' "$report" 2>/dev/null)
                lang=$(jq -r '.language // "unknown"' "$report" 2>/dev/null)
                protocol=$(jq -r '.protocol_version // "unknown"' "$report" 2>/dev/null)

                failures=$(jq '[.results[] | select(.result.status != "pass" and .result.status != "unsupported")] | length' "$report" 2>/dev/null || echo "0")
                skipped=$(jq '[.results[] | select(.result.status == "unsupported")] | length' "$report" 2>/dev/null || echo "0")

                if [ "$failures" != "0" ] || [ "$skipped" != "0" ]; then
                  echo "<details>"
                  echo "<summary><strong>$kernel</strong> ($impl) - $failures failures, $skipped skipped</summary>"
                  echo ""
                  echo "**Language:** $lang | **Protocol:** $protocol"
                  echo ""

                  if [ "$failures" != "0" ]; then
                    echo "#### Failures"
                    echo ""
                    echo "| Test | Message Type | Description | Reason |"
                    echo "|------|--------------|-------------|--------|"
                    jq -r '.results[] | select(.result.status == "fail" or .result.status == "timeout") | "| \(.name) | `\(.message_type)` | \(.description) | \(.result.reason // "timeout") |"' "$report" 2>/dev/null
                    echo ""
                  fi

                  if [ "$skipped" != "0" ]; then
                    echo "#### Skipped (Not Implemented)"
                    echo ""
                    jq -r '.results[] | select(.result.status == "unsupported") | "- **\(.name)**: \(.description)"' "$report" 2>/dev/null
                    echo ""
                  fi

                  echo "</details>"
                  echo ""
                fi
              fi
            done

            # Cross-kernel comparison for failing tests
            echo "---"
            echo ""
            echo "### Test Results by Message Type"
            echo ""
            echo "| Test | Message Type |$(for r in reports/*-report.json; do kernel=$(basename "$r" -report.json); echo -n " $kernel |"; done)"
            echo "|------|--------------|$(for r in reports/*-report.json; do echo -n "--------|"; done)"

            # Get all unique test names
            all_tests=$(jq -r '.results[].name' reports/*-report.json 2>/dev/null | sort -u)
            for test in $all_tests; do
              msg_type=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .message_type' reports/*-report.json 2>/dev/null | head -1)
              echo -n "| $test | \`$msg_type\` |"
              for report in reports/*-report.json; do
                if [ -f "$report" ]; then
                  status=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .result.status' "$report" 2>/dev/null)
                  case "$status" in
                    pass) echo -n " ✅ |" ;;
                    fail) echo -n " ❌ |" ;;
                    unsupported) echo -n " ⏭️ |" ;;
                    timeout) echo -n " ⏱️ |" ;;
                    partial_pass) echo -n " ⚠️ |" ;;
                    *) echo -n " - |" ;;
                  esac
                fi
              done
              echo ""
            done
          } | tee -a $GITHUB_STEP_SUMMARY > summary.md

      - name: Comment on PR
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh pr comment ${{ github.event.pull_request.number }} --body-file summary.md

      - name: Upload combined reports
        uses: actions/upload-artifact@v4
        with:
          name: conformance-reports
          path: reports/
          if-no-files-found: warn