feat: add analysis scripts for benchmark evaluations and improve table detection logic

Author: raphaelmansuy

analyze_benchmark.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+
+# Load evaluations
+hybrid_eval = json.loads(Path('benchmark/prediction/edgeparse_hybrid/evaluation.json').read_text())
+baseline_eval = json.loads(Path('benchmark/prediction/edgeparse/evaluation.json').read_text())
+
+# Compare documents with tables
+docs_with_tables = []
+for doc_data in hybrid_eval['documents']:
+    doc_id = doc_data['document_id']
+    hybrid_teds = doc_data['scores'].get('teds')
+    baseline_teds = None
+    for baseline_doc in baseline_eval['documents']:
+        if baseline_doc['document_id'] == doc_id:
+            baseline_teds = baseline_doc['scores'].get('teds')
+            break
+    
+    if hybrid_teds is not None:
+        improvement = hybrid_teds - baseline_teds if baseline_teds is not None else 0
+        docs_with_tables.append({
+            'doc_id': doc_id,
+            'hybrid_teds': hybrid_teds,
+            'baseline_teds': baseline_teds,
+            'improvement': improvement,
+            'overall': doc_data['scores'].get('overall'),
+        })
+
+# Sort by improvement descending
+docs_with_tables.sort(key=lambda x: x['improvement'], reverse=True)
+
+print('Top 10 documents where hybrid helped:')
+for i, d in enumerate(docs_with_tables[:10], 1):
+    print(f"{i}. {d['doc_id']}: baseline={d['baseline_teds']:.4f} -> hybrid={d['hybrid_teds']:.4f} ({d['improvement']:+.4f})")
+
+print('\nBottom 10 documents where hybrid hurt:')
+for i, d in enumerate(docs_with_tables[-10:], 1):
+    print(f"{i}. {d['doc_id']}: baseline={d['baseline_teds']:.4f} -> hybrid={d['hybrid_teds']:.4f} ({d['improvement']:+.4f})")
+
+print(f'\nTotal docs with tables: {len(docs_with_tables)}')
+print(f'Docs where hybrid improved: {sum(1 for d in docs_with_tables if d["improvement"] > 0.01)}')
+print(f'Docs where hybrid hurt: {sum(1 for d in docs_with_tables if d["improvement"] < -0.01)}')
+print(f'\nAverage improvement: {sum(d["improvement"] for d in docs_with_tables) / len(docs_with_tables):.4f}')

analyze_worst.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+import json
+all_docs = json.loads(open('benchmark/prediction/edgeparse_hybrid/evaluation.json').read())['documents']
+worst = sorted(all_docs, key=lambda x: x['scores'].get('overall', 0))[:10]
+print('Worst performing documents (hybrid mode):')
+for d in worst:
+    print(f"{d['document_id']}: overall={d['scores']['overall']:.4f}, teds={d['scores'].get('teds', 'N/A')}, mhs={d['scores'].get('mhs', 'N/A')}")
+
+print('\n\nDocuments with low TEDS scores (table issues):')
+teds_docs = [(d['document_id'], d['scores'].get('teds', None)) for d in all_docs if d['scores'].get('teds') is not None and d['scores'].get('teds', 1) < 0.3]
+teds_docs.sort(key=lambda x: x[1])
+for doc_id, teds in teds_docs[:10]:
+    print(f"{doc_id}: TEDS={teds:.4f}")

crates/edgeparse-core/src/output/markdown.rs

@@ -1445,6 +1445,9 @@ fn render_layout_single_caption_chart_document_cached(
     if doc.number_of_pages != 1 {
         return None;
     }
+    if document_has_populated_table(doc) {
+        return None;
+    }
 
     let caption_indices = doc
         .kids
@@ -1555,6 +1558,22 @@ fn render_layout_single_caption_chart_document_cached(
     Some(output.trim_end().to_string() + "\n")
 }
 
+fn document_has_populated_table(doc: &PdfDocument) -> bool {
+    doc.kids.iter().any(|element| {
+        table_border_from_element(element).is_some_and(|table| {
+            table.num_rows >= 2
+                && table.num_columns >= 2
+                && table.rows.iter().any(|row| {
+                    row.cells
+                        .iter()
+                        .filter(|cell| !cell_text_content(cell).trim().is_empty())
+                        .count()
+                        >= 2
+                })
+        })
+    })
+}
+
 fn looks_like_chart_noise_element(_element: &ContentElement, text: &str) -> bool {
     if text.is_empty() {
         return false;
@@ -13174,6 +13193,32 @@ mod tests {
         assert!(md.contains("| kaolinite | 10 |"));
     }
 
+    #[test]
+    fn test_single_caption_chart_renderer_skips_documents_with_populated_tables() {
+        let mut doc = PdfDocument::new("table-with-caption.pdf".to_string());
+        doc.number_of_pages = 1;
+        for idx in 0..10 {
+            let bottom = 720.0 - idx as f64 * 18.0;
+            doc.kids.push(make_paragraph(
+                "Explanatory body text that should remain outside the chart-only renderer.",
+                bottom,
+                bottom + 10.0,
+            ));
+        }
+        doc.kids.push(make_paragraph(
+            "Figure 7.2: Kinematic Viscosity of Water at Atmospheric Pressure.",
+            150.0,
+            162.0,
+        ));
+        doc.kids.push(make_two_column_table(&[
+            ("Temperature", "Viscosity"),
+            ("20", "1.004"),
+            ("25", "0.893"),
+        ]));
+
+        assert!(render_layout_single_caption_chart_document(&doc).is_none());
+    }
+
     #[test]
     fn test_blank_right_column_table_is_not_misrendered_as_toc() {
         let mut doc = PdfDocument::new("flocculation-table.pdf".to_string());