Derive package version from git tags via hatch-vcs

.github/workflows/python-publish.yml

@@ -21,6 +21,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          # hatch-vcs derives the package version from `git describe`; a
+          # shallow, tagless checkout would build a 0.x dev version.
+          fetch-depth: 0
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

benchmarks/olmocr_bench/RESULTS.md

@@ -35,7 +35,8 @@ lift **33.7 → 40.7** came from a portfolio of layout-boundary levers, none of
 which touch the engine's `json`/`html`/`opencontracts` renders or the in-domain
 regression suites (all green):
 - **Tables** 38.0 → **63.2** — the pluggable `pymupdf4llm` table provider
-  (`[tables]` extra) activated on the render path (`benchmarks/parsebench/warp_markdown.py`).
+  (benchmark-only ablation; `pip install pymupdf4llm markdown2`) activated on
+  the render path (`benchmarks/parsebench/warp_markdown.py`).
 - **Headers/footers** 48.4 → **79.7** — a within-page running-chrome stripper
   (geometry-only: extreme-margin + isolated + compact) — Warp previously stripped
   only *cross-page* repeats, which never fire on single-page bench PDFs.

benchmarks/parsebench/README.md

@@ -27,7 +27,7 @@ zero network** and is fully reproducible — no API keys needed.
 |---|---|
 | `warp_markdown.py` | Pure renderer: Warp blocks → per-page Markdown (HTML `<table>`s, ranked ATX headings, bullets) + per-block geometry. No `parse_bench` dependency, so it is unit-tested standalone. Table cells come from warp's own native table engine by default (region-aware replacement); content-stripping transforms are off by default. |
 | `warp_ingest_provider.py` | ParseBench `@register_provider("warp_ingest")` PARSE provider, thin wrapper over `warp_markdown`. |
-| `table_providers.py` | Pluggable table-cell providers. Default `WARP_TABLE_PROVIDER=native` = warp's own license-clean engine (`warp_ingest.ingestor.table_engine`, pure MIT stack — pdfplumber ruled grids + whitespace-channel grid inference). `pymupdf4llm` is kept as an opt-in ablation only (AGPL, and with `pymupdf-layout` installed its tables come from a Polyform-Noncommercial ONNX model); `none` disables. |
+| `table_providers.py` | Pluggable table-cell providers. Default `WARP_TABLE_PROVIDER=native` = warp's own license-clean engine (`warp_ingest.ingestor.table_engine`, pure MIT stack — pdfplumber ruled grids + whitespace-channel grid inference); `none` disables. |
 | `warp_layout_adapter.py` | Layout/visual-grounding adapter: turns Warp's per-block geometry into a `LayoutOutput` so warp is scored on the **Visual Grounding** dimension too (text-only baselines can't be). Registers a warp-keyed passthrough label mapper. |
 | `run.py` | Drives the official ParseBench pipeline in-process for Warp-Ingest + baselines; prints a leaderboard-style comparison. |
 | `setup_parsebench.sh` | One-time: clone pinned ParseBench + `pip install` the framework and baseline parsers. |

benchmarks/parsebench/run.py

@@ -21,11 +21,21 @@
 
 import argparse
 import json
+import multiprocessing
 import os
 import shutil
 import sys
 from pathlib import Path
 
+# The warp provider/pipeline/layout-adapter registrations below live in *this*
+# process, and ParseBench's evaluation ProcessPoolExecutor workers must inherit
+# them or every layout example fails with "no provider adapter matched"
+# (Visual Grounding silently collapses to ~10 while the run still exits 0).
+# Python 3.14 changed the Linux default start method from fork to forkserver,
+# whose fresh worker processes lose the registrations — pin fork explicitly.
+if sys.platform.startswith("linux"):
+    multiprocessing.set_start_method("fork", force=True)
+
 # Default comparison set: faithful Warp-Ingest + the four local-library
 # baselines that also appear on the official leaderboard.
 DEFAULT_PIPELINES = [

benchmarks/parsebench/table_providers.py

@@ -17,8 +17,6 @@
   (``warp_ingest.ingestor.table_engine``: pdfplumber ruled grids + region-local
   whitespace-channel grid inference over warp's table regions). No external
   parser involved.
-* ``pymupdf4llm`` — legacy delegation to an external parser, kept only as a
-  benchmark ablation baseline.
 * ``none`` — no provider; warp's raw table rendering.
 """
 
@@ -30,53 +28,6 @@
 TableProvider = Callable[..., dict[int, list[Any]]]
 
 
-def _md_tables_to_html(content: str) -> list[str]:
-    """Pull each markdown pipe-table block out of *content* as an HTML ``<table>``."""
-    import markdown2
-
-    lines = content.split("\n")
-    tables: list[str] = []
-    buf: list[str] = []
-    in_table = False
-
-    def flush() -> None:
-        nonlocal buf
-        if len(buf) >= 2:
-            html = markdown2.markdown("\n".join(buf), extras=["tables"]).strip()
-            if "<table>" in html.lower():
-                tables.append(html)
-        buf = []
-
-    for line in lines:
-        if "|" in line and line.strip().startswith("|"):
-            in_table = True
-            buf.append(line)
-        else:
-            if in_table:
-                flush()
-                in_table = False
-    if in_table:
-        flush()
-    return tables
-
-
-def _pymupdf4llm_provider(
-    pdf_path: str, regions_by_page: Optional[dict] = None
-) -> dict[int, list[str]]:
-    import pymupdf4llm
-
-    chunks = pymupdf4llm.to_markdown(
-        pdf_path, page_chunks=True, show_progress=False, use_ocr=False
-    )
-    out: dict[int, list[str]] = {}
-    for i, ch in enumerate(chunks):
-        text = ch.get("text", "") if isinstance(ch, dict) else str(ch)
-        tables = _md_tables_to_html(text)
-        if tables:
-            out[i] = tables
-    return out
-
-
 def _native_provider(
     pdf_path: str, regions_by_page: Optional[dict] = None
 ) -> dict[int, list[tuple[tuple, str]]]:
@@ -92,11 +43,4 @@ def get_table_provider() -> Optional[TableProvider]:
         return None
     if choice in ("native", "auto", "warp_native", "table_engine"):
         return _native_provider
-    if choice in ("pymupdf4llm", "pymupdf"):
-        try:
-            import markdown2  # noqa: F401
-            import pymupdf4llm  # noqa: F401
-        except ImportError:
-            return None
-        return _pymupdf4llm_provider
     return None

pyproject.toml

@@ -1,6 +1,9 @@
 [project]
 name = "warp-ingest"
-version = "1.0.2"
+# The version is derived from the latest git tag (v2.0.1 -> 2.0.1) by
+# hatch-vcs, so cutting a release only requires pushing a tag -- there is no
+# version string in the repo to bump (see [tool.hatch.version]).
+dynamic = ["version"]
 description = "Pure-Python, RAG-friendly document parsers (PDF/HTML/text/XML/Markdown). Java- and Tika-free."
 readme = "README.md"
 requires-python = ">=3.10,<3.15"
@@ -56,9 +59,7 @@ ocr = [
     "shapely>=2.1.2",
     "six>=1.17.0",
 ]
-# Full runtime install for the hosted ingestion service. The benchmark-only
-# `tables` extra remains separate because it pulls an ablation provider, not a
-# normal production dependency.
+# Full runtime install for the hosted ingestion service.
 all = [
     "fastapi>=0.115,<1.0",
     "onnxruntime>=1.17.0,<1.23; python_version == '3.10'",
@@ -72,15 +73,6 @@ all = [
     "six>=1.17.0",
     "uvicorn>=0.30,<1.0",
 ]
-# Benchmark-ablation baseline for the pluggable table provider. Warp's own
-# native table engine (warp_ingest/ingestor/table_engine.py) is the default and
-# needs nothing beyond the core install; this extra exists only to reproduce
-# the ablation comparison in benchmarks/parsebench/RESULTS.md.
-tables = [
-    "pymupdf4llm>=0.0.17",
-    "markdown2>=2.4.0",
-]
-
 [project.urls]
 Repository = "https://github.com/Open-Source-Legal/Warp-Ingest"
 Issues = "https://github.com/Open-Source-Legal/Warp-Ingest/issues"
@@ -98,18 +90,25 @@ dev = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.7,<0.12.0"]
-build-backend = "uv_build"
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.targets.wheel]
+packages = ["warp_ingest"]
+
+# Keep the sdist to the package itself (plus auto-included metadata files);
+# tests/fixtures alone is ~90MB and has no place on PyPI.
+[tool.hatch.build.targets.sdist]
+include = ["warp_ingest"]
 
 [tool.uv]
 # Strip rapidocr-onnxruntime's `opencv-python` pin (never-true marker): the
 # ocr extra supplies `opencv-python-headless` instead — same cv2 API, no libGL.
 override-dependencies = ["opencv-python; sys_platform == 'never'"]
 
-[tool.uv.build-backend]
-module-name = "warp_ingest"
-module-root = ""
-
 [tool.black]
 target-version = ['py310']
 include = '\.pyi?$'

scripts/tbl_engine_spike.py

@@ -96,44 +96,6 @@ def _pymupdf_pages(pdf_path, strategy=None):
     return pages
 
 
-def _md_tables_to_html(content):
-    import markdown2
-
-    lines = content.split("\n")
-    out, tbl, in_t = [], [], False
-
-    def flush():
-        nonlocal tbl
-        if len(tbl) >= 2:
-            html = markdown2.markdown("\n".join(tbl), extras=["tables"]).strip()
-            out.append(html if "<table>" in html.lower() else "\n".join(tbl))
-        else:
-            out.extend(tbl)
-        tbl = []
-
-    for line in lines:
-        if "|" in line and line.strip().startswith("|"):
-            in_t = True
-            tbl.append(line)
-        else:
-            if in_t:
-                flush()
-                in_t = False
-            out.append(line)
-    if in_t:
-        flush()
-    return "\n".join(out)
-
-
-def _pymupdf4llm_pages(pdf_path):
-    import pymupdf4llm
-
-    chunks = pymupdf4llm.to_markdown(
-        pdf_path, page_chunks=True, show_progress=False, use_ocr=False
-    )
-    return [_md_tables_to_html(ch.get("text", "")) for ch in chunks]
-
-
 def _warp_regions(pdf_path):
     """{page_idx: [(x0, top, x1, bottom), ...]} from warp's table spans."""
     from benchmarks.parsebench.warp_markdown import extract_warp_blocks
@@ -206,8 +168,6 @@ def _score_one(args):
                 pages = _pdfplumber_pages(pdf_path, settings=_PLUMBER_TEXT)
             elif engine == "pymupdf_text":
                 pages = _pymupdf_pages(pdf_path, strategy="text")
-            elif engine == "pymupdf4llm":
-                pages = _pymupdf4llm_pages(pdf_path)
             elif engine == "native":
                 pages = _native_pages(pdf_path)
             elif engine == "native_ruled":
@@ -266,7 +226,6 @@ def main() -> int:
             "pdfplumber_text",
             "pymupdf",
             "pymupdf_text",
-            "pymupdf4llm",
             "native",
             "native_ruled",
         ],