Add generic markdown layout export helpers

README.md

@@ -141,6 +141,8 @@ from warp_ingest.ingestor import pdf_ingestor
 
 export = pdf_ingestor.parse_to_opencontracts("document.pdf")   # OpenContracts export
 markdown = pdf_ingestor.parse_to_markdown("document.pdf")       # Markdown export
+payload = pdf_ingestor.parse_to_markdown_payload("document.pdf") # Blocks + tables + geometry
+layout = pdf_ingestor.parse_to_layout_predictions("document.pdf") # Generic layout predictions
 ingestor = pdf_ingestor.PDFIngestor("document.pdf", {"render_format": "all"})
 blocks = ingestor.blocks
 ```

benchmarks/parsebench/warp_layout_adapter.py

@@ -18,8 +18,6 @@
 
 from __future__ import annotations
 
-from typing import Any
-
 from parse_bench.evaluation.layout_adapters.base import LayoutAdapter
 from parse_bench.evaluation.layout_adapters.registry import register_layout_adapter
 from parse_bench.evaluation.layout_label_mappers.base import (
@@ -39,7 +37,7 @@
 from parse_bench.schemas.layout_ontology import CanonicalLabel
 from parse_bench.schemas.pipeline_io import InferenceResult
 
-from benchmarks.parsebench.warp_markdown import _table_html
+from warp_ingest.ingestor.markdown_exporter import render_layout_predictions
 
 # Warp emits already-canonical labels; a non-LlamaParse carrier model + a
 # warp-keyed passthrough mapper keep label resolution off any provider-specific
@@ -58,27 +56,6 @@ def to_canonical(
         return CanonicalLabel(label)
 
 
-# Warp block_type -> ParseBench Canonical17 label.
-_LABEL_MAP = {
-    "header": CanonicalLabel.SECTION_HEADER.value,
-    "para": CanonicalLabel.TEXT.value,
-    "list_item": CanonicalLabel.LIST_ITEM.value,
-    "table_row": CanonicalLabel.TABLE.value,
-}
-
-
-def _union_box(boxes: list[list[float]]) -> list[float] | None:
-    """Union of ``[left, top, w, h]`` boxes -> ``[x1, y1, x2, y2]``."""
-    rects = [b for b in boxes if b]
-    if not rects:
-        return None
-    x1 = min(b[0] for b in rects)
-    y1 = min(b[1] for b in rects)
-    x2 = max(b[0] + b[2] for b in rects)
-    y2 = max(b[1] + b[3] for b in rects)
-    return [x1, y1, x2, y2]
-
-
 @register_layout_adapter("warp_ingest", priority=100)
 class WarpIngestLayoutAdapter(LayoutAdapter):
     """Build a ``LayoutOutput`` from Warp-Ingest's geometric block stream."""
@@ -94,90 +71,28 @@ def to_layout_output(
             if isinstance(inference_result.raw_output, dict)
             else {}
         )
-        page_dim = raw.get("page_dim") or [612.0, 792.0]
-        img_w = max(1, int(round(float(page_dim[0]))))
-        img_h = max(1, int(round(float(page_dim[1]))))
-        blocks = raw.get("blocks", []) or []
-
+        rendered = render_layout_predictions(raw, page_filter=page_filter)
+        img_w = max(1, int(rendered.get("image_width", 612)))
+        img_h = max(1, int(rendered.get("image_height", 792)))
         predictions: list[LayoutPrediction] = []
-        order = 0
-
-        # Table accumulator (merge consecutive table_row blocks per table_idx).
-        tbl_idx: Any = None
-        tbl_boxes: list[list[float]] = []
-        tbl_header: list[str] | None = None
-        tbl_rows: list[list[str]] = []
-        tbl_page = 1
-
-        def flush_table() -> None:
-            nonlocal order, tbl_idx, tbl_boxes, tbl_header, tbl_rows
-            if not (tbl_rows or tbl_header):
-                tbl_idx = None
-                return
-            box = _union_box(tbl_boxes)
-            if box is not None:
-                header = tbl_header
-                body = [r for r in tbl_rows if not (header and r == header)]
-                predictions.append(
-                    LayoutPrediction(
-                        bbox=box,
-                        score=1.0,
-                        label=CanonicalLabel.TABLE.value,
-                        page=tbl_page,
-                        content=LayoutTableContent(html=_table_html(header, body)),
-                        provider_metadata={"order_index": order},
-                    )
-                )
-                order += 1
-            tbl_idx = None
-            tbl_boxes = []
-            tbl_header = None
-            tbl_rows = []
-
-        for b in blocks:
-            page = int(b.get("page_idx", 0) or 0) + 1
-            btype = b.get("block_type")
-            box = b.get("box")
-            text = (b.get("block_text") or "").strip()
-
-            if btype == "table_row":
-                tidx = b.get("table_idx")
-                if (tbl_rows or tbl_header) and tidx != tbl_idx:
-                    flush_table()
-                tbl_idx = tidx
-                tbl_page = page
-                if box:
-                    tbl_boxes.append(box)
-                hdr = b.get("header_cell_values")
-                if tbl_header is None and hdr:
-                    tbl_header = [str(c) for c in hdr]
-                cells = b.get("cell_values") or ([text] if text else [])
-                tbl_rows.append([str(c) for c in cells])
-                continue
-
-            if tbl_rows or tbl_header:
-                flush_table()
-
-            if not text or box is None:
-                continue
-            xyxy = [box[0], box[1], box[0] + box[2], box[1] + box[3]]
+        for prediction in rendered["predictions"]:
+            content_payload = prediction.get("content") or {}
+            if content_payload.get("type") == "table":
+                content = LayoutTableContent(html=content_payload.get("html", ""))
+            else:
+                content = LayoutTextContent(text=content_payload.get("text", ""))
             predictions.append(
                 LayoutPrediction(
-                    bbox=xyxy,
-                    score=1.0,
-                    label=_LABEL_MAP.get(btype, CanonicalLabel.TEXT.value),
-                    page=page,
-                    content=LayoutTextContent(text=text),
-                    provider_metadata={"order_index": order},
+                    bbox=prediction["bbox"],
+                    score=float(prediction.get("score", 1.0)),
+                    label=str(prediction.get("label") or CanonicalLabel.TEXT.value),
+                    page=int(prediction.get("page", 1)),
+                    content=content,
+                    provider_metadata={
+                        "order_index": int(prediction.get("order_index", 0))
+                    },
                 )
             )
-            order += 1
-
-        if tbl_rows or tbl_header:
-            flush_table()
-
-        if page_filter is not None:
-            predictions = [p for p in predictions if p.page == page_filter]
 
         return LayoutOutput(
             example_id=inference_result.request.example_id,

benchmarks/parsebench/warp_markdown.py

@@ -38,6 +38,7 @@
 from typing import Any
 
 from benchmarks.parsebench.table_providers import get_table_provider
+from warp_ingest.ingestor.markdown_exporter import box_xywh as _engine_box_xywh
 
 # Tunables for the table keep-open behavior (swept via the fast eval). Defaults
 # match the shipped rule; env overrides are for measurement only.
@@ -89,16 +90,7 @@ def _box_xywh(box_style: Any) -> list[float] | None:
     BoxStyle is indexable as ``[top, left, right, width, height]`` and also
     exposes ``.top/.left/.width/.height``; we read defensively.
     """
-    if box_style is None:
-        return None
-    try:
-        left = float(getattr(box_style, "left", box_style[1]))
-        top = float(getattr(box_style, "top", box_style[0]))
-        width = float(getattr(box_style, "width", box_style[3]))
-        height = float(getattr(box_style, "height", box_style[4]))
-    except (TypeError, IndexError, ValueError):
-        return None
-    return [left, top, width, height]
+    return _engine_box_xywh(box_style)
 
 
 def _renderer_manifest() -> dict[str, Any]:
@@ -775,14 +767,14 @@ def _reorder_page_columns(
     splits = [(xa + xb) / 2.0 for xa, xb in gutters]
 
     def col_of(b: dict[str, Any]) -> int:
-        l, _t, w, _h = b["box"]
-        cx = l + w / 2.0
+        left, _top, width, _height = b["box"]
+        cx = left + width / 2.0
         return sum(1 for s in splits if cx >= s)
 
     def spans_gutter(b: dict[str, Any]) -> bool:
-        l, _t, w, _h = b["box"]
-        r = l + w
-        return any(l < xa and r > xb for xa, xb in gutters)
+        left, _top, width, _height = b["box"]
+        right = left + width
+        return any(left < xa and right > xb for xa, xb in gutters)
 
     ordered = sorted(page_blocks, key=lambda b: b["box"][1])  # top-to-bottom sweep
     out: list[dict[str, Any]] = []

tests/test_markdown_exporter.py

@@ -6,6 +6,7 @@
 from warp_ingest.ingestor.markdown_exporter import (
     box_xywh,
     parse_to_markdown,
+    render_layout_predictions,
     render_pages,
 )
 
@@ -23,6 +24,19 @@ def __getitem__(self, index):
     assert box_xywh(None) is None
 
 
+def test_box_xywh_from_attrs_only_and_dict():
+    class _AttrOnlyBox:
+        top, left, width, height = 10.0, 20.0, 100.0, 12.0
+
+    assert box_xywh(_AttrOnlyBox()) == [20.0, 10.0, 100.0, 12.0]
+    assert box_xywh({"top": 1, "left": 2, "width": 3, "height": 4}) == [
+        2.0,
+        1.0,
+        3.0,
+        4.0,
+    ]
+
+
 def test_render_pages_groups_tables_and_emphasis():
     payload = {
         "num_pages": 1,
@@ -141,6 +155,69 @@ def test_render_pages_appends_unplaced_table_html():
     assert "<td>Loose</td>" in markdown
 
 
+def test_render_pages_accepts_ext_tables_alias():
+    payload = {
+        "num_pages": 1,
+        "blocks": [
+            {
+                "page_idx": 0,
+                "block_type": "para",
+                "block_text": "Keep this prose.",
+                "box": [50, 10, 200, 10],
+            }
+        ],
+        "ext_tables": {0: [[None, "<table><tr><td>Alias</td></tr></table>"]]},
+    }
+
+    markdown = render_pages(payload)[0][1]
+
+    assert "Keep this prose." in markdown
+    assert "<td>Alias</td>" in markdown
+
+
+def test_render_layout_predictions_merges_tables_per_page_and_table_idx():
+    payload = {
+        "num_pages": 2,
+        "page_dim": [612.0, 792.0],
+        "blocks": [
+            {
+                "page_idx": 0,
+                "block_type": "para",
+                "block_text": "Intro",
+                "box": [10, 20, 100, 10],
+            },
+            {
+                "page_idx": 0,
+                "block_type": "table_row",
+                "cell_values": ["a"],
+                "table_idx": 0,
+                "box": [50, 100, 200, 12],
+            },
+            {
+                "page_idx": 1,
+                "block_type": "table_row",
+                "cell_values": ["b"],
+                "table_idx": 0,
+                "box": [60, 110, 180, 12],
+            },
+        ],
+    }
+
+    rendered = render_layout_predictions(payload)
+    predictions = rendered["predictions"]
+
+    assert rendered["image_width"] == 612
+    assert [prediction["page"] for prediction in predictions] == [1, 1, 2]
+    tables = [
+        prediction
+        for prediction in predictions
+        if prediction["content"]["type"] == "table"
+    ]
+    assert len(tables) == 2
+    assert tables[0]["bbox"] == [50.0, 100.0, 250.0, 112.0]
+    assert tables[1]["bbox"] == [60.0, 110.0, 240.0, 122.0]
+
+
 def test_parse_to_markdown_public_api():
     path = os.path.join(FIX_DIR, "sample.pdf")
 
@@ -149,6 +226,16 @@ def test_parse_to_markdown_public_api():
         path,
         include_native_tables=False,
     )
+    payload = pdf_ingestor.parse_to_markdown_payload(
+        path,
+        include_native_tables=False,
+    )
+    layout = pdf_ingestor.parse_to_layout_predictions(
+        path,
+        include_native_tables=False,
+    )
 
     assert len(markdown) > 100
     assert markdown == wrapper_markdown
+    assert payload["blocks"]
+    assert layout["predictions"]