From daf16aa7beaaf4e4601113a331a917688b7bf17c Mon Sep 17 00:00:00 2001 From: JSv4 Date: Tue, 7 Jul 2026 06:46:48 -0500 Subject: [PATCH 1/4] Derive package version from git tags via hatch-vcs The version in pyproject.toml was never bumped past 1.0.2, so the v2.0.0 and v2.0.1 release publishes both rebuilt 1.0.2 and were rejected by PyPI. Switch the build backend to hatchling + hatch-vcs so the version is derived from the release tag (v2.0.1 -> 2.0.1) and a release only requires pushing a tag; there is no version string left in the repo. - uv remains the frontend (sync/lock/build); uv_build rejects dynamic = ["version"], which is why the backend changes - publish workflow checks out full history so git describe sees tags - sdist is restricted to the package (tests/fixtures is ~90MB) Co-Authored-By: Claude Fable 5 --- .github/workflows/python-publish.yml | 4 ++++ pyproject.toml | 24 +++++++++++++++++------- uv.lock | 3 +-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0a9e78b..c064948 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -21,6 +21,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # hatch-vcs derives the package version from `git describe`; a + # shallow, tagless checkout would build a 0.x dev version. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/pyproject.toml b/pyproject.toml index 4ef272c..1d6d634 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,9 @@ [project] name = "warp-ingest" -version = "1.0.2" +# The version is derived from the latest git tag (v2.0.1 -> 2.0.1) by +# hatch-vcs, so cutting a release only requires pushing a tag -- there is no +# version string in the repo to bump (see [tool.hatch.version]). +dynamic = ["version"] description = "Pure-Python, RAG-friendly document parsers (PDF/HTML/text/XML/Markdown). Java- and Tika-free." readme = "README.md" requires-python = ">=3.10,<3.15" @@ -98,18 +101,25 @@ dev = [ ] [build-system] -requires = ["uv_build>=0.11.7,<0.12.0"] -build-backend = "uv_build" +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.targets.wheel] +packages = ["warp_ingest"] + +# Keep the sdist to the package itself (plus auto-included metadata files); +# tests/fixtures alone is ~90MB and has no place on PyPI. +[tool.hatch.build.targets.sdist] +include = ["warp_ingest"] [tool.uv] # Strip rapidocr-onnxruntime's `opencv-python` pin (never-true marker): the # ocr extra supplies `opencv-python-headless` instead — same cv2 API, no libGL. override-dependencies = ["opencv-python; sys_platform == 'never'"] -[tool.uv.build-backend] -module-name = "warp_ingest" -module-root = "" - [tool.black] target-version = ['py310'] include = '\.pyi?$' diff --git a/uv.lock b/uv.lock index 0ed03f5..851f2b8 100644 --- a/uv.lock +++ b/uv.lock @@ -1881,7 +1881,6 @@ wheels = [ [[package]] name = "warp-ingest" -version = "1.0.2" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, @@ -1971,7 +1970,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'all'", specifier = ">=0.30,<1.0" }, { name = "uvicorn", marker = "extra == 'service'", specifier = ">=0.30,<1.0" }, ] -provides-extras = ["parser", "service", "ocr", "all", "tables"] +provides-extras = ["all", "ocr", "parser", "service", "tables"] [package.metadata.requires-dev] dev = [ From 16a061b8084c96d362921fe04192154a60cbf6c1 Mon Sep 17 00:00:00 2001 From: JSv4 Date: Tue, 7 Jul 2026 06:53:40 -0500 Subject: [PATCH 2/4] Drop the benchmark-only tables extra from published metadata The [tables] extra existed solely to reproduce the pymupdf4llm table ablation in benchmarks/parsebench/RESULTS.md; the shipped package never imports those dependencies (the native table engine is the default and self-contained), and publishing an AGPL-licensed ablation dependency as a package extra invites accidental installs. The benchmark provider already degrades gracefully when the packages are absent; reproducing the ablation is a plain `pip install pymupdf4llm markdown2`. Co-Authored-By: Claude Fable 5 --- benchmarks/olmocr_bench/RESULTS.md | 3 +- pyproject.toml | 13 +--- uv.lock | 112 +---------------------------- 3 files changed, 4 insertions(+), 124 deletions(-) diff --git a/benchmarks/olmocr_bench/RESULTS.md b/benchmarks/olmocr_bench/RESULTS.md index 15c823d..7f2d17e 100644 --- a/benchmarks/olmocr_bench/RESULTS.md +++ b/benchmarks/olmocr_bench/RESULTS.md @@ -35,7 +35,8 @@ lift **33.7 → 40.7** came from a portfolio of layout-boundary levers, none of which touch the engine's `json`/`html`/`opencontracts` renders or the in-domain regression suites (all green): - **Tables** 38.0 → **63.2** — the pluggable `pymupdf4llm` table provider - (`[tables]` extra) activated on the render path (`benchmarks/parsebench/warp_markdown.py`). + (benchmark-only ablation; `pip install pymupdf4llm markdown2`) activated on + the render path (`benchmarks/parsebench/warp_markdown.py`). - **Headers/footers** 48.4 → **79.7** — a within-page running-chrome stripper (geometry-only: extreme-margin + isolated + compact) — Warp previously stripped only *cross-page* repeats, which never fire on single-page bench PDFs. diff --git a/pyproject.toml b/pyproject.toml index 1d6d634..63d9992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,9 +59,7 @@ ocr = [ "shapely>=2.1.2", "six>=1.17.0", ] -# Full runtime install for the hosted ingestion service. The benchmark-only -# `tables` extra remains separate because it pulls an ablation provider, not a -# normal production dependency. +# Full runtime install for the hosted ingestion service. all = [ "fastapi>=0.115,<1.0", "onnxruntime>=1.17.0,<1.23; python_version == '3.10'", @@ -75,15 +73,6 @@ all = [ "six>=1.17.0", "uvicorn>=0.30,<1.0", ] -# Benchmark-ablation baseline for the pluggable table provider. Warp's own -# native table engine (warp_ingest/ingestor/table_engine.py) is the default and -# needs nothing beyond the core install; this extra exists only to reproduce -# the ablation comparison in benchmarks/parsebench/RESULTS.md. -tables = [ - "pymupdf4llm>=0.0.17", - "markdown2>=2.4.0", -] - [project.urls] Repository = "https://github.com/Open-Source-Legal/Warp-Ingest" Issues = "https://github.com/Open-Source-Legal/Warp-Ingest/issues" diff --git a/uv.lock b/uv.lock index 851f2b8..c9750a4 100644 --- a/uv.lock +++ b/uv.lock @@ -513,15 +513,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, ] -[[package]] -name = "markdown2" -version = "2.5.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/ae/07d4a5fcaa5509221287d289323d75ac8eda5a5a4ac9de2accf7bbcc2b88/markdown2-2.5.5.tar.gz", hash = "sha256:001547e68f6e7fcf0f1cb83f7e82f48aa7d48b2c6a321f0cd20a853a8a2d1664", size = 157249, upload-time = "2026-03-02T20:46:53.411Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/af/4b3891eb0a49d6cfd5cbf3e9bf514c943afc2b0f13e2c57cc57cd88ecc21/markdown2-2.5.5-py3-none-any.whl", hash = "sha256:be798587e09d1f52d2e4d96a649c4b82a778c75f9929aad52a2c95747fa26941", size = 56250, upload-time = "2026-03-02T20:46:52.032Z" }, -] - [[package]] name = "mpmath" version = "1.3.0" @@ -540,38 +531,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] -[[package]] -name = "networkx" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - [[package]] name = "nltk" version = "3.9.4" @@ -1279,60 +1238,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] -[[package]] -name = "pymupdf" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/6d6c5d6c0a3551bffd47681a6240caf941727f195b45593cf20ab36f018f/pymupdf-1.28.0.tar.gz", hash = "sha256:e53f3567403a92da15caa9e7ae0164327fff48817e9f40175367fb9de524258d", size = 87637751, upload-time = "2026-06-29T09:08:47.547Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/b7/88043e38cc7529de070f0c9bd267fa258035cca0b4ad5260536b994594a7/pymupdf-1.28.0-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:892b89ba88e8f98b53133b62877a9dc9b5e7dc6a4aeb837b612db56a8d2e03ac", size = 24597385, upload-time = "2026-06-29T09:03:30.608Z" }, - { url = "https://files.pythonhosted.org/packages/33/f4/23775bbda0781b61fc398cc75079a2b0e64696d8fcf93271748883e9627e/pymupdf-1.28.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:4d692dcf44d3566ae96bc6f6346c6ad432274a29ba617bf7a9fe18009e24adb4", size = 23828292, upload-time = "2026-06-29T09:03:46.129Z" }, - { url = "https://files.pythonhosted.org/packages/1c/f5/bf75fc7a415722f8b33662054f82d88520c0cbfd4c36d0e08aeaec605e49/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:47a5c29ed4eb0744de9c4e37bb49b1259b18d4d75fcc8a7c130f7c9fa15956f6", size = 25045507, upload-time = "2026-06-29T09:04:03.86Z" }, - { url = "https://files.pythonhosted.org/packages/58/69/5d12c9f1f2d76f28383d6110a069c79fbfced5a4f97bb1ee6e8354f52bb7/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44f0973f5e5edbaec95bc34b64e71d1959d4ee90b1328de1b4f4f5b4fa78673f", size = 25716599, upload-time = "2026-06-29T09:04:19.367Z" }, - { url = "https://files.pythonhosted.org/packages/4d/b4/ec0e017bc42857cc86bd651441dbc41cc18be48d4698ecd27aac491e0c9a/pymupdf-1.28.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4d61ec323a706e153a12e262e51febfb43eeaa20977785ace135d18d48bcdc83", size = 25940489, upload-time = "2026-06-29T09:04:36.624Z" }, - { url = "https://files.pythonhosted.org/packages/06/86/f831fef09013f33b3c9c09fb3923f2ff53e1e437f6ace14b8ae46392f558/pymupdf-1.28.0-cp310-abi3-win32.whl", hash = "sha256:caea2b3b67347fd79e5d15ed7929b0e886aac594ea228073b6d39de0078189da", size = 18489703, upload-time = "2026-06-29T20:50:30.599Z" }, - { url = "https://files.pythonhosted.org/packages/2e/5d/1a03f53eb0449900469335fcfc742ca28e3ba159b7d650e0921d50b8b308/pymupdf-1.28.0-cp310-abi3-win_amd64.whl", hash = "sha256:e01e90fd86abfeb37ceb921eddb951f988a11d45ff6ce6b7664f2039849068ec", size = 19773102, upload-time = "2026-06-29T09:04:49.773Z" }, - { url = "https://files.pythonhosted.org/packages/72/f6/1e52ce243ca792254f6223b4017c5667194c146ce9b88baf37bc5eb3d1c9/pymupdf-1.28.0-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:74c6d00ba2a9aad3a635db73b07c15db462b480741d831a34a75a56535ebc22b", size = 18357011, upload-time = "2026-06-29T20:50:50.353Z" }, - { url = "https://files.pythonhosted.org/packages/62/b1/46b5b3d8ef3cc71114667cf10c4d8b33f39af97253af32e9a0986775b638/pymupdf-1.28.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b3e1399c7a64c6914239116a369efcdaac4cfb9e838bde2656d7accc4a85c72d", size = 25753599, upload-time = "2026-06-29T09:05:09.398Z" }, -] - -[[package]] -name = "pymupdf-layout" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "onnxruntime", version = "1.22.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pymupdf" }, - { name = "pyyaml" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/fe/eb3c960cbc6e5be5acffbd6811f281ba6e176a841b592858e25e28ac0a97/pymupdf_layout-1.28.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:dc4e2f4b48951633607020d80de0bed00b2450eafd56ecda224f611c19e5f9e8", size = 41567727, upload-time = "2026-06-29T09:05:35.532Z" }, - { url = "https://files.pythonhosted.org/packages/60/14/9a330a7d77cbe05fa9e97d9f2aa11d627a556eec795eee08bcbeedc16ac6/pymupdf_layout-1.28.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:911aa80b3ddcca2ddbc1df5fc59f652935b392052251397e75963d3de9559dbc", size = 41566722, upload-time = "2026-06-29T09:06:01.73Z" }, - { url = "https://files.pythonhosted.org/packages/0b/58/2607c539540ce261d05d2677c0d839b78a4191d328accc1d0e9385a06799/pymupdf_layout-1.28.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35b98e1ce9382709622e03c34b57d1b51a155ceb3d9122f11f8e9b2aa4f1ee55", size = 41576718, upload-time = "2026-06-29T09:06:29.258Z" }, - { url = "https://files.pythonhosted.org/packages/56/3a/dc5ab8573300b0f1b7fb996aa33ce4683c27b190a8e8be6f18d80714183d/pymupdf_layout-1.28.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a6bd191301570a0863d6e04418324e823d0fd7f18b4fe1db2b9e53e715d2f8ff", size = 41578107, upload-time = "2026-06-29T09:06:54.224Z" }, - { url = "https://files.pythonhosted.org/packages/b8/73/79af357b1cacb02ea9e054b5b368dee5a0ccd7168c6229e54e1a70e74c53/pymupdf_layout-1.28.0-cp310-abi3-win_amd64.whl", hash = "sha256:07195ec4ad6317dd70bf1d4eca44d9dd93231d8db3648ab9f4b771557a59ea48", size = 41577085, upload-time = "2026-06-29T09:07:20.443Z" }, -] - -[[package]] -name = "pymupdf4llm" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pymupdf" }, - { name = "pymupdf-layout" }, - { name = "tabulate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/15/36ff769451f5e62cbca89c3b59739e625565df7140778bd6dd11db32caea/pymupdf4llm-1.28.0.tar.gz", hash = "sha256:713595be867f7cb52893e57aa1b058d5721d017b2ba7b6a3d185a05e15978852", size = 2072657, upload-time = "2026-06-29T09:08:52.23Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/1a/8b997bbeeaf63286d7179fe4cdc29dcc328105f2558f01394370dc0fcaa9/pymupdf4llm-1.28.0-py3-none-any.whl", hash = "sha256:6e74e8666806cc6040d9159053130f0761d21ef17e2a9c938df8108f15160cb3", size = 164656, upload-time = "2026-06-29T09:05:11.914Z" }, -] - [[package]] name = "pypdfium2" version = "5.10.1" @@ -1769,15 +1674,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - [[package]] name = "tomli" version = "2.4.1" @@ -1921,10 +1817,6 @@ service = [ { name = "python-multipart" }, { name = "uvicorn" }, ] -tables = [ - { name = "markdown2" }, - { name = "pymupdf4llm" }, -] [package.dev-dependencies] dev = [ @@ -1943,7 +1835,6 @@ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.0,<5.0.0" }, { name = "fastapi", marker = "extra == 'all'", specifier = ">=0.115,<1.0" }, { name = "fastapi", marker = "extra == 'service'", specifier = ">=0.115,<1.0" }, - { name = "markdown2", marker = "extra == 'tables'", specifier = ">=2.4.0" }, { name = "nltk", specifier = ">=3.9.1" }, { name = "numpy", specifier = ">=1.24,<3" }, { name = "onnxruntime", marker = "python_full_version == '3.10.*' and extra == 'all'", specifier = ">=1.17.0,<1.23" }, @@ -1955,7 +1846,6 @@ requires-dist = [ { name = "pdfplumber", specifier = ">=0.11.4,<0.12.0" }, { name = "pyclipper", marker = "extra == 'all'", specifier = ">=1.4.0" }, { name = "pyclipper", marker = "extra == 'ocr'", specifier = ">=1.4.0" }, - { name = "pymupdf4llm", marker = "extra == 'tables'", specifier = ">=0.0.17" }, { name = "pypdfium2", specifier = ">=5" }, { name = "python-multipart", marker = "extra == 'all'", specifier = ">=0.0.9" }, { name = "python-multipart", marker = "extra == 'service'", specifier = ">=0.0.9" }, @@ -1970,7 +1860,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'all'", specifier = ">=0.30,<1.0" }, { name = "uvicorn", marker = "extra == 'service'", specifier = ">=0.30,<1.0" }, ] -provides-extras = ["all", "ocr", "parser", "service", "tables"] +provides-extras = ["all", "ocr", "parser", "service"] [package.metadata.requires-dev] dev = [ From 554c1c6734fd2b602f7547a9162a2a0ca911e5f3 Mon Sep 17 00:00:00 2001 From: JSv4 Date: Tue, 7 Jul 2026 06:56:52 -0500 Subject: [PATCH 3/4] Remove the pymupdf4llm table-provider ablation entirely Follow-through on dropping the [tables] extra: delete the ablation code itself, not just its published dependency. The pymupdf4llm delegation path in table_providers.py, the pymupdf4llm engine mode in tbl_engine_spike.py, and the Fast-vs-Quality timing script existed only to compare against the external parser the native table engine already ties; an unknown WARP_TABLE_PROVIDER value now simply disables the provider. The pymupdf4llm *baseline pipeline* (a rival parser in the leaderboard field run) is unaffected. Co-Authored-By: Claude Fable 5 --- benchmarks/parsebench/README.md | 2 +- benchmarks/parsebench/table_providers.py | 56 ------------- scripts/tbl_engine_spike.py | 41 --------- scripts/tbl_mode_timing.py | 101 ----------------------- 4 files changed, 1 insertion(+), 199 deletions(-) delete mode 100644 scripts/tbl_mode_timing.py diff --git a/benchmarks/parsebench/README.md b/benchmarks/parsebench/README.md index 4219558..a827318 100644 --- a/benchmarks/parsebench/README.md +++ b/benchmarks/parsebench/README.md @@ -27,7 +27,7 @@ zero network** and is fully reproducible — no API keys needed. |---|---| | `warp_markdown.py` | Pure renderer: Warp blocks → per-page Markdown (HTML ``s, ranked ATX headings, bullets) + per-block geometry. No `parse_bench` dependency, so it is unit-tested standalone. Table cells come from warp's own native table engine by default (region-aware replacement); content-stripping transforms are off by default. | | `warp_ingest_provider.py` | ParseBench `@register_provider("warp_ingest")` PARSE provider, thin wrapper over `warp_markdown`. | -| `table_providers.py` | Pluggable table-cell providers. Default `WARP_TABLE_PROVIDER=native` = warp's own license-clean engine (`warp_ingest.ingestor.table_engine`, pure MIT stack — pdfplumber ruled grids + whitespace-channel grid inference). `pymupdf4llm` is kept as an opt-in ablation only (AGPL, and with `pymupdf-layout` installed its tables come from a Polyform-Noncommercial ONNX model); `none` disables. | +| `table_providers.py` | Pluggable table-cell providers. Default `WARP_TABLE_PROVIDER=native` = warp's own license-clean engine (`warp_ingest.ingestor.table_engine`, pure MIT stack — pdfplumber ruled grids + whitespace-channel grid inference); `none` disables. | | `warp_layout_adapter.py` | Layout/visual-grounding adapter: turns Warp's per-block geometry into a `LayoutOutput` so warp is scored on the **Visual Grounding** dimension too (text-only baselines can't be). Registers a warp-keyed passthrough label mapper. | | `run.py` | Drives the official ParseBench pipeline in-process for Warp-Ingest + baselines; prints a leaderboard-style comparison. | | `setup_parsebench.sh` | One-time: clone pinned ParseBench + `pip install` the framework and baseline parsers. | diff --git a/benchmarks/parsebench/table_providers.py b/benchmarks/parsebench/table_providers.py index bbe1a32..943d73a 100644 --- a/benchmarks/parsebench/table_providers.py +++ b/benchmarks/parsebench/table_providers.py @@ -17,8 +17,6 @@ (``warp_ingest.ingestor.table_engine``: pdfplumber ruled grids + region-local whitespace-channel grid inference over warp's table regions). No external parser involved. -* ``pymupdf4llm`` — legacy delegation to an external parser, kept only as a - benchmark ablation baseline. * ``none`` — no provider; warp's raw table rendering. """ @@ -30,53 +28,6 @@ TableProvider = Callable[..., dict[int, list[Any]]] -def _md_tables_to_html(content: str) -> list[str]: - """Pull each markdown pipe-table block out of *content* as an HTML ``
``.""" - import markdown2 - - lines = content.split("\n") - tables: list[str] = [] - buf: list[str] = [] - in_table = False - - def flush() -> None: - nonlocal buf - if len(buf) >= 2: - html = markdown2.markdown("\n".join(buf), extras=["tables"]).strip() - if "
" in html.lower(): - tables.append(html) - buf = [] - - for line in lines: - if "|" in line and line.strip().startswith("|"): - in_table = True - buf.append(line) - else: - if in_table: - flush() - in_table = False - if in_table: - flush() - return tables - - -def _pymupdf4llm_provider( - pdf_path: str, regions_by_page: Optional[dict] = None -) -> dict[int, list[str]]: - import pymupdf4llm - - chunks = pymupdf4llm.to_markdown( - pdf_path, page_chunks=True, show_progress=False, use_ocr=False - ) - out: dict[int, list[str]] = {} - for i, ch in enumerate(chunks): - text = ch.get("text", "") if isinstance(ch, dict) else str(ch) - tables = _md_tables_to_html(text) - if tables: - out[i] = tables - return out - - def _native_provider( pdf_path: str, regions_by_page: Optional[dict] = None ) -> dict[int, list[tuple[tuple, str]]]: @@ -92,11 +43,4 @@ def get_table_provider() -> Optional[TableProvider]: return None if choice in ("native", "auto", "warp_native", "table_engine"): return _native_provider - if choice in ("pymupdf4llm", "pymupdf"): - try: - import markdown2 # noqa: F401 - import pymupdf4llm # noqa: F401 - except ImportError: - return None - return _pymupdf4llm_provider return None diff --git a/scripts/tbl_engine_spike.py b/scripts/tbl_engine_spike.py index ac24b0e..84ff0cd 100644 --- a/scripts/tbl_engine_spike.py +++ b/scripts/tbl_engine_spike.py @@ -96,44 +96,6 @@ def _pymupdf_pages(pdf_path, strategy=None): return pages -def _md_tables_to_html(content): - import markdown2 - - lines = content.split("\n") - out, tbl, in_t = [], [], False - - def flush(): - nonlocal tbl - if len(tbl) >= 2: - html = markdown2.markdown("\n".join(tbl), extras=["tables"]).strip() - out.append(html if "
" in html.lower() else "\n".join(tbl)) - else: - out.extend(tbl) - tbl = [] - - for line in lines: - if "|" in line and line.strip().startswith("|"): - in_t = True - tbl.append(line) - else: - if in_t: - flush() - in_t = False - out.append(line) - if in_t: - flush() - return "\n".join(out) - - -def _pymupdf4llm_pages(pdf_path): - import pymupdf4llm - - chunks = pymupdf4llm.to_markdown( - pdf_path, page_chunks=True, show_progress=False, use_ocr=False - ) - return [_md_tables_to_html(ch.get("text", "")) for ch in chunks] - - def _warp_regions(pdf_path): """{page_idx: [(x0, top, x1, bottom), ...]} from warp's table spans.""" from benchmarks.parsebench.warp_markdown import extract_warp_blocks @@ -206,8 +168,6 @@ def _score_one(args): pages = _pdfplumber_pages(pdf_path, settings=_PLUMBER_TEXT) elif engine == "pymupdf_text": pages = _pymupdf_pages(pdf_path, strategy="text") - elif engine == "pymupdf4llm": - pages = _pymupdf4llm_pages(pdf_path) elif engine == "native": pages = _native_pages(pdf_path) elif engine == "native_ruled": @@ -266,7 +226,6 @@ def main() -> int: "pdfplumber_text", "pymupdf", "pymupdf_text", - "pymupdf4llm", "native", "native_ruled", ], diff --git a/scripts/tbl_mode_timing.py b/scripts/tbl_mode_timing.py deleted file mode 100644 index a0b26fb..0000000 --- a/scripts/tbl_mode_timing.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -"""Latency comparison: Warp-Fast (provider off) vs Warp-Quality (pymupdf4llm -table provider on), single-threaded per-document parse time — the service-latency -view for positioning the two modes. - -For each sampled PDF we time extract_warp_blocks twice: once with the table -provider disabled (WARP_TABLE_PROVIDER=none) and once enabled (=auto). The -provider re-parses the whole document with pymupdf4llm, so the delta is its cost. -""" - -from __future__ import annotations - -import argparse -import contextlib -import importlib -import io -import os -import sys -import time -from pathlib import Path - -REPO = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO)) -DATA = REPO / "parsebench_work" / "data" - - -def _load(group): - from parse_bench.test_cases.loader import _load_jsonl_dataset - - cs = [c for c in _load_jsonl_dataset(DATA) if getattr(c, "group", None) == group] - cs.sort(key=lambda c: c.test_id) - return cs - - -def _time_parse(pdf_path, provider_mode): - """Time a single extract_warp_blocks under the given provider mode (seconds), - returning (seconds, num_pages).""" - os.environ["WARP_TABLE_PROVIDER"] = provider_mode - # warp_markdown caches get_table_provider's import, but reads the env each - # call, so toggling the env between calls switches modes within one process. - from benchmarks.parsebench import warp_markdown - - sink = io.StringIO() - t0 = time.perf_counter() - with contextlib.redirect_stdout(sink), contextlib.redirect_stderr(sink): - raw = warp_markdown.extract_warp_blocks(pdf_path) - dt = time.perf_counter() - t0 - return dt, int(raw.get("num_pages", 0) or 0) - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--n-table", type=int, default=20) - ap.add_argument("--n-text", type=int, default=10) - args = ap.parse_args() - - table = _load("table")[: args.n_table] - text = _load("text_content")[: args.n_text] - sample = [("table", c) for c in table] + [("text", c) for c in text] - - # Warm up imports + file cache (don't count the first parse). - _time_parse(str(sample[0][1].file_path), "none") - _time_parse(str(sample[0][1].file_path), "auto") - - rows = [] - for grp, c in sample: - p = str(c.file_path) - # quality first then fast, or vice versa; run fast then quality, both - # after a warm file cache from the warmup of doc0 (other docs cold on - # first read but both modes pay it). - t_fast, npg = _time_parse(p, "none") - t_qual, _ = _time_parse(p, "auto") - rows.append((grp, c.test_id, npg, t_fast, t_qual)) - - def _summ(label, idx): - vals = sorted(r[idx] for r in rows) - n = len(vals) - total = sum(vals) - mean = total / n - med = vals[n // 2] - return f"{label:16s} total={total:6.1f}s mean={mean:5.2f}s median={med:5.2f}s" - - print(f"\nsampled {len(rows)} docs ({len(table)} table + {len(text)} text)") - print(_summ("Warp-Fast", 3)) - print(_summ("Warp-Quality", 4)) - fast_tot = sum(r[3] for r in rows) - qual_tot = sum(r[4] for r in rows) - print( - f"\nWarp-Quality / Warp-Fast = {qual_tot/fast_tot:.2f}x (provider overhead)" - ) - print(f"added latency per doc (mean) = {(qual_tot-fast_tot)/len(rows):.2f}s") - - rows.sort(key=lambda r: r[4] - r[3], reverse=True) - print("\nslowest provider deltas (doc, pages, fast -> quality):") - for grp, tid, npg, tf, tq in rows[:8]: - print(f" [{grp:5s}] {tid[:42]:42s} pg={npg:>3} {tf:6.2f}s -> {tq:6.2f}s") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 58fb623faff8640adede037caa03922ccfed0092 Mon Sep 17 00:00:00 2001 From: JSv4 Date: Tue, 7 Jul 2026 06:59:17 -0500 Subject: [PATCH 4/4] Pin fork start method in the ParseBench harness for py3.14 Python 3.14 changed the Linux multiprocessing default from fork to forkserver; ParseBench's evaluation workers then no longer inherit the in-process warp pipeline/adapter registrations, every layout example fails with "no provider adapter matched", and Visual Grounding silently collapses to ~10 while the run exits 0. Co-Authored-By: Claude Fable 5 --- benchmarks/parsebench/run.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/benchmarks/parsebench/run.py b/benchmarks/parsebench/run.py index 393139d..7af5e74 100644 --- a/benchmarks/parsebench/run.py +++ b/benchmarks/parsebench/run.py @@ -21,11 +21,21 @@ import argparse import json +import multiprocessing import os import shutil import sys from pathlib import Path +# The warp provider/pipeline/layout-adapter registrations below live in *this* +# process, and ParseBench's evaluation ProcessPoolExecutor workers must inherit +# them or every layout example fails with "no provider adapter matched" +# (Visual Grounding silently collapses to ~10 while the run still exits 0). +# Python 3.14 changed the Linux default start method from fork to forkserver, +# whose fresh worker processes lose the registrations — pin fork explicitly. +if sys.platform.startswith("linux"): + multiprocessing.set_start_method("fork", force=True) + # Default comparison set: faithful Warp-Ingest + the four local-library # baselines that also appear on the official leaderboard. DEFAULT_PIPELINES = [