Merge pull request #3 from runtimed/detailed-conformance-reports

Add detailed conformance reports with actionable insights

Author: rgbkrk

.github/workflows/conformance.yml

@@ -95,7 +95,7 @@ jobs:
           pattern: report-*
           merge-multiple: true
 
-      - name: Generate matrix summary
+      - name: Generate detailed report
         run: |
           {
             echo "## Kernel Conformance Matrix"
@@ -119,6 +119,79 @@ jobs:
                 echo "| $kernel | $tier1/$tier1_total | $tier2/$tier2_total | $tier3/$tier3_total | $tier4/$tier4_total | $total/$total_tests |"
               fi
             done
+
+            echo ""
+            echo "---"
+            echo ""
+
+            # Detailed failure breakdown per kernel
+            for report in reports/*-report.json; do
+              if [ -f "$report" ]; then
+                kernel=$(basename "$report" -report.json)
+                impl=$(jq -r '.implementation // "unknown"' "$report" 2>/dev/null)
+                lang=$(jq -r '.language // "unknown"' "$report" 2>/dev/null)
+                protocol=$(jq -r '.protocol_version // "unknown"' "$report" 2>/dev/null)
+
+                failures=$(jq '[.results[] | select(.result.status != "pass" and .result.status != "unsupported")] | length' "$report" 2>/dev/null || echo "0")
+                skipped=$(jq '[.results[] | select(.result.status == "unsupported")] | length' "$report" 2>/dev/null || echo "0")
+
+                if [ "$failures" != "0" ] || [ "$skipped" != "0" ]; then
+                  echo "<details>"
+                  echo "<summary><strong>$kernel</strong> ($impl) - $failures failures, $skipped skipped</summary>"
+                  echo ""
+                  echo "**Language:** $lang | **Protocol:** $protocol"
+                  echo ""
+
+                  if [ "$failures" != "0" ]; then
+                    echo "#### Failures"
+                    echo ""
+                    echo "| Test | Message Type | Likely Source | Reason |"
+                    echo "|------|--------------|---------------|--------|"
+                    jq -r '.results[] | select(.result.status == "fail" or .result.status == "timeout") | "| \(.name) | `\(.message_type)` | \(.result.kind // "unknown") | \(.result.reason // "timeout") |"' "$report" 2>/dev/null
+                    echo ""
+                  fi
+
+                  if [ "$skipped" != "0" ]; then
+                    echo "#### Skipped (Not Implemented)"
+                    echo ""
+                    jq -r '.results[] | select(.result.status == "unsupported") | "- **\(.name)**: \(.description)"' "$report" 2>/dev/null
+                    echo ""
+                  fi
+
+                  echo "</details>"
+                  echo ""
+                fi
+              fi
+            done
+
+            # Cross-kernel comparison for failing tests
+            echo "---"
+            echo ""
+            echo "### Test Results by Message Type"
+            echo ""
+            echo "| Test | Message Type |$(for r in reports/*-report.json; do kernel=$(basename "$r" -report.json); echo -n " $kernel |"; done)"
+            echo "|------|--------------|$(for r in reports/*-report.json; do echo -n "--------|"; done)"
+
+            # Get all unique test names
+            all_tests=$(jq -r '.results[].name' reports/*-report.json 2>/dev/null | sort -u)
+            for test in $all_tests; do
+              msg_type=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .message_type' reports/*-report.json 2>/dev/null | head -1)
+              echo -n "| $test | \`$msg_type\` |"
+              for report in reports/*-report.json; do
+                if [ -f "$report" ]; then
+                  status=$(jq -r --arg t "$test" '.results[] | select(.name == $t) | .result.status' "$report" 2>/dev/null)
+                  case "$status" in
+                    pass) echo -n " ✅ |" ;;
+                    fail) echo -n " ❌ |" ;;
+                    unsupported) echo -n " ⏭️ |" ;;
+                    timeout) echo -n " ⏱️ |" ;;
+                    partial_pass) echo -n " ⚠️ |" ;;
+                    *) echo -n " - |" ;;
+                  esac
+                fi
+              done
+              echo ""
+            done
           } | tee -a $GITHUB_STEP_SUMMARY > summary.md
 
       - name: Comment on PR

Cargo.lock

@@ -332,6 +332,10 @@ impl KernelUnderTest {
 pub struct ConformanceTest {
     pub name: &'static str,
     pub category: TestCategory,
+    /// Human-readable description of what this test validates
+    pub description: &'static str,
+    /// The primary protocol message type being tested (e.g., "kernel_info_request")
+    pub message_type: &'static str,
     pub run: fn(&mut KernelUnderTest) -> std::pin::Pin<Box<dyn std::future::Future<Output = TestResult> + Send + '_>>,
 }
 
@@ -369,6 +373,8 @@ pub async fn run_conformance_suite(
         results.push(TestRecord {
             name: test.name.to_string(),
             category: test.category,
+            description: test.description.to_string(),
+            message_type: test.message_type.to_string(),
             result,
             duration: test_start.elapsed(),
         });

src/harness.rs

@@ -29,4 +29,4 @@ pub use harness::{run_conformance_suite, ConformanceTest, KernelUnderTest};
 pub use report::{render_json, render_markdown, render_matrix_json, render_matrix_markdown, render_terminal};
 pub use snippets::LanguageSnippets;
 pub use tests::all_tests;
-pub use types::{ConformanceMatrix, KernelReport, TestCategory, TestRecord, TestResult};
+pub use types::{ConformanceMatrix, FailureKind, KernelReport, TestCategory, TestRecord, TestResult};

src/lib.rs

@@ -49,9 +49,12 @@ pub fn render_terminal(report: &KernelReport) -> String {
                 emoji, symbol, record.name, record.duration
             ));
 
-            // Show failure reason
-            if let TestResult::Fail { reason } = &record.result {
+            // Show failure reason and hint
+            if let TestResult::Fail { reason, kind } = &record.result {
                 output.push_str(&format!("      Reason: {}\n", reason));
+                if let Some(k) = kind {
+                    output.push_str(&format!("      Likely source: {} | {}\n", k.likely_source(), k.actionable_hint()));
+                }
             }
             if let TestResult::PartialPass { score, notes } = &record.result {
                 output.push_str(&format!("      Score: {:.0}% - {}\n", score * 100.0, notes));
@@ -113,7 +116,7 @@ pub fn render_markdown(report: &KernelReport) -> String {
     for record in &report.results {
         let result_str = match &record.result {
             TestResult::Pass => "PASS".to_string(),
-            TestResult::Fail { reason } => format!("FAIL: {}", truncate(reason, 30)),
+            TestResult::Fail { reason, .. } => format!("FAIL: {}", truncate(reason, 30)),
             TestResult::Unsupported => "SKIP".to_string(),
             TestResult::Timeout => "TIMEOUT".to_string(),
             TestResult::PartialPass { score, .. } => format!("PARTIAL ({:.0}%)", score * 100.0),