Skip to content

Commit 15707b0

Browse files
authored
Merge pull request #52 from artiz/claude/issue-35-python-dropin
feat(docling-py): finalize strangler-fig drop-in over real DoclingDoument
2 parents 2a5e686 + 80b1b21 commit 15707b0

11 files changed

Lines changed: 412 additions & 118 deletions

File tree

.github/workflows/pypi-publish.yml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# Publish the `docling-rs` PyPI package (the PyO3 Python bindings in
2+
# crates/docling-py) for a chosen release. Like the npm package, the bindings are
3+
# a compiled native extension, so it can't be a one-line `twine upload`: wheels
4+
# are built on a matrix of runners (one abi3 wheel per OS/arch — `abi3-py39`, so
5+
# a single wheel covers every Python ≥ 3.9), an sdist is built once, and a
6+
# publish job uploads them all to PyPI.
7+
#
8+
# Trigger: manual only (workflow_dispatch), mirroring npm-publish.yml. By default
9+
# it builds the checked-out ref at the version already declared in
10+
# crates/docling-py/pyproject.toml; pass `version` (or a release `tag`) to
11+
# override. Run it from the Actions tab (or `gh workflow run pypi-publish.yml`,
12+
# optionally `-f version=0.16.0`).
13+
#
14+
# Requires one repository secret: PYPI_TOKEN — a PyPI API token with publish
15+
# rights to the `docling-rs` project. (To switch to OIDC trusted publishing
16+
# instead, drop the `password:` line and add `id-token: write` + an
17+
# `environment:` to the publish job, as docling-core's pypi.yml does.)
18+
#
19+
# The ONNX Runtime is statically bundled at build time (`ort` `download-binaries`)
20+
# and pdfium is loaded at runtime from the model cache, so the wheels are
21+
# self-contained for the declarative path and need no extra native setup here —
22+
# same story as the npm build.
23+
24+
name: pypi publish
25+
26+
on:
27+
workflow_dispatch:
28+
inputs:
29+
tag:
30+
description: "Release tag to build (e.g. v0.16.0). Blank = current ref."
31+
required: false
32+
default: ""
33+
version:
34+
description: "Override the PyPI version (e.g. 0.16.0). Blank = tag, or the version in pyproject.toml."
35+
required: false
36+
default: ""
37+
38+
concurrency:
39+
group: pypi-publish-${{ inputs.tag || github.ref }}
40+
cancel-in-progress: false
41+
42+
defaults:
43+
run:
44+
working-directory: crates/docling-py
45+
46+
jobs:
47+
wheels:
48+
name: wheel ${{ matrix.target }}
49+
runs-on: ${{ matrix.host }}
50+
strategy:
51+
fail-fast: false
52+
matrix:
53+
include:
54+
- target: x86_64-unknown-linux-gnu
55+
host: ubuntu-22.04
56+
manylinux: "2_28"
57+
# Native ARM64 Linux runner — avoids cross-compiling the ONNX/ort build.
58+
- target: aarch64-unknown-linux-gnu
59+
host: ubuntu-24.04-arm
60+
manylinux: "2_28"
61+
- target: x86_64-pc-windows-msvc
62+
host: windows-latest
63+
manylinux: "auto"
64+
# macOS wheels are omitted: GitHub-hosted macOS runners are blocked in
65+
# this environment and darwin can't be cross-compiled on Linux (needs
66+
# the Apple SDK). macOS users build from source with `maturin develop`.
67+
# Re-add when runners are available:
68+
# - { target: aarch64-apple-darwin, host: macos-14, manylinux: "auto" }
69+
# - { target: x86_64-apple-darwin, host: macos-14, manylinux: "auto" }
70+
steps:
71+
- uses: actions/checkout@v4
72+
with:
73+
ref: ${{ inputs.tag || github.ref }}
74+
75+
# Version override: patch [project].version in pyproject.toml before the
76+
# build so it lands in the wheel metadata/filename. No override → keep the
77+
# version already declared in the file.
78+
- name: Resolve version
79+
shell: bash
80+
run: |
81+
v="${{ inputs.version }}"
82+
if [ -z "$v" ] && [ -n "${{ inputs.tag }}" ]; then v="${{ inputs.tag }}"; v="${v#v}"; fi
83+
if [ -n "$v" ]; then
84+
sed -i.bak -E "0,/^version = \".*\"/s//version = \"$v\"/" pyproject.toml && rm -f pyproject.toml.bak
85+
echo "Building docling-rs $v"
86+
else
87+
echo "Building docling-rs $(grep -m1 '^version = ' pyproject.toml | sed -E 's/.*"([^"]+)".*/\1/') (from pyproject.toml)"
88+
fi
89+
90+
- name: Build wheel
91+
uses: PyO3/maturin-action@v1
92+
with:
93+
target: ${{ matrix.target }}
94+
manylinux: ${{ matrix.manylinux }}
95+
args: --release --out dist
96+
sccache: "true"
97+
working-directory: crates/docling-py
98+
99+
- name: Upload wheel
100+
uses: actions/upload-artifact@v4
101+
with:
102+
name: wheel-${{ matrix.target }}
103+
path: crates/docling-py/dist/*.whl
104+
if-no-files-found: error
105+
106+
sdist:
107+
name: sdist
108+
runs-on: ubuntu-latest
109+
steps:
110+
- uses: actions/checkout@v4
111+
with:
112+
ref: ${{ inputs.tag || github.ref }}
113+
- name: Resolve version
114+
shell: bash
115+
run: |
116+
v="${{ inputs.version }}"
117+
if [ -z "$v" ] && [ -n "${{ inputs.tag }}" ]; then v="${{ inputs.tag }}"; v="${v#v}"; fi
118+
if [ -n "$v" ]; then
119+
sed -i.bak -E "0,/^version = \".*\"/s//version = \"$v\"/" pyproject.toml && rm -f pyproject.toml.bak
120+
fi
121+
- name: Build sdist
122+
uses: PyO3/maturin-action@v1
123+
with:
124+
command: sdist
125+
args: --out dist
126+
working-directory: crates/docling-py
127+
- name: Upload sdist
128+
uses: actions/upload-artifact@v4
129+
with:
130+
name: sdist
131+
path: crates/docling-py/dist/*.tar.gz
132+
if-no-files-found: error
133+
134+
publish:
135+
name: publish to PyPI
136+
needs: [wheels, sdist]
137+
runs-on: ubuntu-latest
138+
steps:
139+
# All wheels + the sdist → a single dist/ for the upload.
140+
- name: Download artifacts
141+
uses: actions/download-artifact@v4
142+
with:
143+
path: dist
144+
merge-multiple: true
145+
146+
- name: Publish to PyPI
147+
uses: pypa/gh-action-pypi-publish@release/v1
148+
with:
149+
packages-dir: dist
150+
password: ${{ secrets.PYPI_TOKEN }}
151+
# Idempotent re-runs: don't fail if a version is already uploaded.
152+
skip-existing: true

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
/target
22
Cargo.lock
33

4+
# Stray CLI conversion outputs (`docling-rs --to dclx <file>` writes next to CWD)
5+
/*.dclx
6+
47
# Secrets — never commit (holds CARGO_REGISTRY_TOKEN)
58
.env
69
.env.*

MIGRATION.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ crates/
5353
├── docling-asr/ # audio decode (symphonia), mel.rs, whisper.rs (ONNX), tokenizer.rs
5454
├── docling-cli/ # `--strict`, `--to md|json`, `--images placeholder|embedded|referenced`
5555
├── docling-node/ # Node.js/Bun N-API bindings (napi-rs), published to npm as `docling.rs`
56+
├── docling-py/ # PyO3 bindings (maturin), published to PyPI as `docling-rs` (strangler-fig over docling-core)
5657
└── docling-rag/ # RAG layer on top of the converter (chunking, embeddings, vector search, REST API)
5758
```
5859

@@ -280,6 +281,11 @@ deliberate scope boundary or a cosmetic, single-fixture polish gap.
280281
(SQLite+sqlite-vec / PostgreSQL+pgvector), LLM, sources and queues, plus an
281282
eval harness and a REST API. See the crate README.
282283
- **`docling-node`** — Node.js/Bun N-API bindings (npm package).
284+
- **`docling-py`** — PyO3 bindings (PyPI package `docling-rs`): a strangler-fig
285+
drop-in for docling's Python API where the Rust engine is the document
286+
processor and `result.document` is a genuine `docling_core` `DoclingDocument`,
287+
so its `export_to_markdown()` / `export_to_dict()` / chunkers are docling's
288+
own code.
283289
- **MHTML backend** — no docling analogue.
284290

285291
## 7. Testing
@@ -368,8 +374,8 @@ The port followed roughly: **Phase 0** skeleton & API → **Phase 2** text/marku
368374
PPTX, XLSX, EPUB, ODF) → **Phase 4** long tail (XML families, LaTeX, Email,
369375
WebVTT, JSON) → **Phase 5–6** the PDF/image ML pipeline (pdfium + ONNX layout/OCR
370376
+ geometric tables) → output formats (strict Markdown, JSON, image extraction) →
371-
**Phase 7** audio/ASR (symphonia + ONNX Whisper). PyO3 interop bindings remain
372-
the one unbuilt piece.
377+
**Phase 7** audio/ASR (symphonia + ONNX Whisper). The Node.js/Bun (`docling-node`)
378+
and Python (`docling-py`, PyO3) interop bindings followed.
373379

374380
## The meat-grinder mascot 🦀
375381

README.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,31 @@ Runnable Node + Bun examples are in
306306
[`crates/docling-node/README.md`](./crates/docling-node/README.md) for
307307
the full API.
308308

309+
## Python bindings
310+
311+
docling.rs also ships as a PyPI package, **`docling-rs`** — PyO3 bindings (built
312+
with [maturin](https://www.maturin.rs)) in
313+
[`crates/docling-py`](./crates/docling-py). It's a *strangler-fig* drop-in for
314+
docling's Python API: only the document processor is Rust, and
315+
`result.document` is a genuine `docling_core` `DoclingDocument`, so
316+
`export_to_markdown()`, `export_to_dict()`, `export_to_doctags()` and the
317+
chunkers are docling's own Python code.
318+
319+
```python
320+
# was: from docling.document_converter import DocumentConverter
321+
from docling_rs import DocumentConverter
322+
323+
result = DocumentConverter().convert("report.docx")
324+
print(result.document.export_to_markdown())
325+
data = result.document.export_to_dict() # docling JSON wire format (schema 1.10.0)
326+
```
327+
328+
Declarative formats (Markdown, HTML, DOCX, XLSX, …) work with no models; the
329+
PDF/image pipeline downloads pdfium + the ONNX models on first use via
330+
`docling_rs.download_models()`. See
331+
[`crates/docling-py/README.md`](./crates/docling-py/README.md) for the full API
332+
and local build steps.
333+
309334
## Getting the ML models
310335

311336
The PDF/image pipeline needs native assets that aren't bundled in the crate or

crates/docling-py/README.md

Lines changed: 48 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,21 @@ print(result.document.export_to_markdown())
1414
data = result.document.export_to_dict() # docling-core JSON wire format (schema 1.10.0)
1515
```
1616

17-
> **Status: experimental, not published.** The PyPI distribution name
18-
> (`docling.rs`) is tentative and may change; nothing is uploaded anywhere.
19-
> Build and install locally as below. The crate is intentionally outside the
20-
> repo's Cargo workspace and its crates.io publish flow.
17+
**Only the document processor is Rust.** The engine parses the input and returns
18+
docling-core's JSON wire format; this package validates it into a genuine
19+
[`docling_core.types.doc.DoclingDocument`](https://github.com/docling-project/docling-core).
20+
So `result.document` **is** the docling object — `export_to_markdown()`,
21+
`export_to_dict()`, `export_to_doctags()`, the serializers, and the
22+
[chunkers](https://github.com/docling-project/docling-core) are docling's own
23+
Python code, unchanged. `docling-core` is a runtime dependency; nothing else from
24+
docling is required for the declarative path.
25+
26+
> **Status: experimental.** The PyPI distribution name is `docling-rs`.
27+
> Releases are cut manually (like the npm package) via the
28+
> [`pypi-publish`](../../.github/workflows/pypi-publish.yml) workflow — see
29+
> [Publishing](#publishing) below. The crate is intentionally outside the repo's
30+
> Cargo workspace and its crates.io publish flow. For development, build and
31+
> install locally as shown next.
2132
2233
## Try it locally
2334

@@ -51,13 +62,13 @@ PY
5162

5263
| docling.rs | docling counterpart | notes |
5364
|---|---|---|
54-
| `DocumentConverter(strict=False, fetch_images=False, artifacts_path=None)` | `DocumentConverter(...)` | `strict` = docling.rs-only cleaner Markdown; default output is docling-legacy byte parity. `artifacts_path` overrides the model cache dir. |
65+
| `DocumentConverter(fetch_images=False, artifacts_path=None)` | `DocumentConverter(...)` | `fetch_images` resolves remote/local `<img src>` (HTML/EPUB); `artifacts_path` overrides the model cache dir. |
5566
| `.convert(path) -> ConversionResult` | `.convert(source)` | str / `pathlib.Path`. Releases the GIL during conversion. |
5667
| `.convert_bytes(name, data)` | `DocumentStream` | extension of `name` drives format detection |
57-
| `result.status` / `result.document` | same | status: `"success" / "partial_success" / "failure"` |
58-
| `document.export_to_markdown()` | same | plus `export_to_markdown(strict=True/False)` per-call override |
59-
| `document.export_to_dict()` / `export_to_json()` | `export_to_dict()` | dict / JSON string of the docling wire format |
60-
| `document.save_as_markdown(p)` / `save_as_json(p)` | same | |
68+
| `result.status` / `result.document` / `result.input.file` | same | `.status` is a `ConversionStatus` str-enum (`"success" / "partial_success" / "failure"`); `.document` is a genuine `docling_core` `DoclingDocument` |
69+
| `document.export_to_markdown(...)` | same | docling-core's own method — all of docling's params (`image_placeholder`, `page_break_placeholder`, …) apply |
70+
| `document.export_to_dict()` / `export_to_json()` / `export_to_doctags()` | same | docling-core's own serializers over the wire format |
71+
| `document.save_as_markdown(p)` / `save_as_json(p)` / chunkers | same | anything `docling_core` offers on a `DoclingDocument` works, since it *is* one |
6172
| `docling_rs.download_models()` | `docling-tools models download` | idempotent; `~/.cache/docling.rs` or `$DOCLING_RS_CACHE_DIR`; INT8 models fetched when hosted and preferred automatically (`DOCLING_RS_FP32=1` opts out) |
6273

6374
Model/env resolution order: explicit `DOCLING_*` env vars → the cache dir set
@@ -67,7 +78,31 @@ platforms set `PDFIUM_DYNAMIC_LIB_PATH` to a local build.
6778

6879
## Not covered (yet)
6980

70-
Chunkers, VLM/enrichment pipelines, and docling's full options model
71-
(`PdfPipelineOptions`, per-format backends selection). The document object
72-
carries rendered text for inline formatting rather than structured
73-
`formatting` fields — see `MIGRATION.md` §4 for the documented divergences.
81+
VLM/enrichment pipelines and docling's full options model
82+
(`PdfPipelineOptions`, per-format backend selection). Chunkers **are** available
83+
now — the returned object is a real `docling_core` `DoclingDocument`, so
84+
`docling_core.transforms.chunker`'s `HierarchicalChunker` / `HybridChunker`
85+
operate on it directly (install docling-core's own extras for those:
86+
`pip install "docling-core[chunking]"`). The document carries rendered text for
87+
inline formatting rather than structured `formatting` fields — see
88+
`MIGRATION.md` §4 for the documented divergences.
89+
90+
## Publishing
91+
92+
Releases are **manual**, mirroring the npm package: the
93+
[`pypi-publish`](../../.github/workflows/pypi-publish.yml) GitHub Actions workflow
94+
(`workflow_dispatch`) builds an `abi3` wheel per platform (Linux x86-64/arm64 as
95+
`manylinux_2_28`, Windows x86-64 — one wheel covers every Python ≥ 3.9) plus an
96+
sdist, and uploads them to PyPI.
97+
98+
```bash
99+
# From the Actions tab, or:
100+
gh workflow run pypi-publish.yml # version from pyproject.toml
101+
gh workflow run pypi-publish.yml -f version=0.16.0
102+
```
103+
104+
It needs one repository secret, `PYPI_TOKEN` (a PyPI API token with publish
105+
rights to `docling-rs`); re-runs are idempotent (`skip-existing`). macOS wheels
106+
are omitted (no hosted runners here); macOS users install the sdist, which
107+
compiles from source. The ONNX runtime is bundled in the wheel; pdfium is fetched
108+
at runtime by `download_models()`.

crates/docling-py/pyproject.toml

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
# NOT published to PyPI yet — the distribution name below is tentative and the
2-
# package is built/installed locally with maturin (see README.md).
1+
# PyPI distribution name: `docling-rs`. Released manually via the
2+
# `.github/workflows/pypi-publish.yml` workflow (workflow_dispatch), which builds
3+
# the abi3 wheels + sdist and uploads them; local builds use maturin (see
4+
# README.md).
35
[build-system]
46
requires = ["maturin>=1.5,<2"]
57
build-backend = "maturin"
@@ -17,6 +19,12 @@ classifiers = [
1719
"Programming Language :: Python :: 3",
1820
"License :: OSI Approved :: MIT License",
1921
]
22+
# The document *model* and its serializers/chunkers are docling's own Python
23+
# code: the Rust engine only produces the JSON wire format, which this package
24+
# validates into a genuine `docling_core.types.doc.DoclingDocument`.
25+
dependencies = [
26+
"docling-core>=2.0",
27+
]
2028

2129
[project.urls]
2230
Repository = "https://github.com/docling-project/docling.rs"

0 commit comments

Comments
 (0)