diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..7f4581f0 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,99 @@ +# Publish the in-browser demo (crates/docling-wasm/www) to GitHub Pages, so +# docling.rs can be tried from any device without a checkout or a toolchain. +# +# The site is fully static: the wasm module is built here and copied in next to +# the page. No models are published — they are large, and the page resolves them +# at runtime (device file → same-origin ./models/ → Hugging Face), so the +# declarative converters and text-layer PDFs work immediately and OCR works as +# soon as the visitor supplies models. +# +# GitHub Pages cannot send COOP/COEP headers, which ONNX Runtime Web needs for +# threads; www/coi.js installs a service worker that adds them, so the deployed +# page still gets multi-threaded wasm. +# +# One-time setup in the repository: Settings → Pages → Build and deployment → +# Source: "GitHub Actions". +name: pages + +on: + push: + branches: [master] + paths: + - "crates/docling-wasm/**" + - "crates/docling/**" + - "crates/docling-core/**" + - "crates/docling-pdf/**" + - ".github/workflows/pages.yml" + # Deploy on demand too (e.g. the first run, before any matching push). + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +# One deployment at a time; let a running one finish so the site is never +# left half-published. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: build the demo + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Install Rust (stable + wasm32 target) + run: | + rustup toolchain install stable --profile minimal + rustup default stable + rustup target add wasm32-unknown-unknown + + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + target + key: ${{ runner.os }}-cargo-v2-wasm-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-v2-wasm- + + - name: Build docling-wasm (release) + run: cargo build -p docling-wasm --target wasm32-unknown-unknown --release --locked + + - name: Generate the JS bindings + run: | + # Pin wasm-bindgen-cli to the wasm-bindgen version in Cargo.lock: a + # mismatch produces glue the module rejects at load time. + version=$(cargo metadata --format-version 1 --locked \ + | python3 -c 'import json,sys; print(next(p["version"] for p in json.load(sys.stdin)["packages"] if p["name"]=="wasm-bindgen"))') + echo "wasm-bindgen $version" + cargo install wasm-bindgen-cli --version "$version" --locked + wasm-bindgen --target web --out-dir crates/docling-wasm/www/pkg \ + target/wasm32-unknown-unknown/release/docling_wasm.wasm + + - name: Assemble the site + run: | + mkdir -p site + cp -r crates/docling-wasm/www/. site/ + # Jekyll would swallow paths that start with an underscore. + touch site/.nojekyll + ls -la site site/pkg + + - uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + name: deploy + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index 120efc42..f79ee34f 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ pipeline is supported, validated byte-for-byte against live docling. See [`docs/MIGRATION.md`](./docs/MIGRATION.md) for the full architecture, the Python → Rust mapping, and per-format conformance. +**▶ [Try it in your browser](https://docling-project.github.io/docling.rs/)** — +the whole converter compiled to wasm: drop a DOCX, PDF, XLSX, EPUB … and get +Markdown, docling JSON or DocLang XML back. Nothing is uploaded; the page runs +entirely on your device, phone included. Scanned pages can be OCR'd there too +(layout + PP-OCR + TableFormer via ONNX Runtime Web) once you point it at the +models. See [`crates/docling-wasm`](./crates/docling-wasm/README.md). + Developed with **Claude Code** and _[TENET](https://github.com/artiz/tenet/tree/master)_ (minimalistic spec driven design framework). ## Status diff --git a/crates/docling-pdf/Cargo.toml b/crates/docling-pdf/Cargo.toml index 8cde9c3f..63ba04ca 100644 --- a/crates/docling-pdf/Cargo.toml +++ b/crates/docling-pdf/Cargo.toml @@ -69,5 +69,10 @@ tokenizers = { version = "0.20", default-features = false, features = ["onig"], [target.'cfg(target_arch = "wasm32")'.dependencies] lopdf = { version = "0.44", features = ["wasm_js"] } +# The text-layer diagnostic example reaches for lopdf directly to tell a load +# failure apart from "loaded, but no text came out". +[dev-dependencies] +lopdf = "0.44" + [lib] name = "docling_pdf" diff --git a/crates/docling-pdf/examples/text_layer.rs b/crates/docling-pdf/examples/text_layer.rs new file mode 100644 index 00000000..b286bbd9 --- /dev/null +++ b/crates/docling-pdf/examples/text_layer.rs @@ -0,0 +1,160 @@ +//! Run **exactly** what a wasm build does with a PDF: `convert_text_layer` — +//! the pure-Rust content-stream parser, no pdfium, no ONNX. Use it to tell a +//! browser "this PDF needs OCR" apart from a genuine scan: the browser falls +//! back to OCR precisely when this produces no nodes, so an empty result here +//! reproduces that decision offline. +//! +//! Note this is *not* what the CLI's `--no-ocr` runs — that goes through +//! pdfium's text extraction, which can read layers this parser cannot. When the +//! two disagree, the breakdown below says which stage lost the text. +//! +//! ```bash +//! cargo run -p docling-pdf --no-default-features --example text_layer -- +//! ``` +//! (`--no-default-features` keeps pdfium/onnxruntime out of the build.) + +fn main() { + let path = match std::env::args().nth(1) { + Some(p) => p, + None => { + eprintln!("usage: text_layer "); + std::process::exit(2); + } + }; + let bytes = std::fs::read(&path).expect("read the pdf"); + let name = std::path::Path::new(&path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.clone()); + + // Stage 1 — does lopdf load the document at all? `textparse` swallows this + // failure (an unloadable PDF and a scan both come out empty), so ask + // directly: encryption, a damaged xref or an unsupported filter all land + // here, and they mean something very different from "no text layer". + match lopdf::Document::load_mem(&bytes) { + Err(e) => { + eprintln!("1. lopdf load: FAILED — {e}"); + eprintln!(" (the browser reports this as \"no embedded text layer\")"); + eprintln!( + " xref repair: {}", + docling_pdf::textparse::xref_repair_status(&bytes) + ); + probe_tail(&bytes); + } + Ok(doc) => { + eprintln!( + "1. lopdf load: ok — {} page(s), version {}, encrypted: {}", + doc.get_pages().len(), + doc.version, + doc.is_encrypted() + ); + } + } + + // Stage 2 — raw line cells out of the content streams. Zero here with a + // successful load means the text is there but we cannot decode it (font + // encoding, an unhandled operator, a filter), not that the page is a scan. + let pages = docling_pdf::textparse::pdf_textlines(&bytes); + let cells: usize = pages.iter().map(|(_, _, c)| c.len()).sum(); + eprintln!("2. text lines: {cells} across {} page(s)", pages.len()); + if cells == 0 { + eprintln!( + " where it is lost:{}", + docling_pdf::textparse::content_diagnosis(&bytes) + ); + } + for (i, (_, _, c)) in pages.iter().enumerate().take(3) { + if let Some(first) = c.iter().find(|c| !c.text.trim().is_empty()) { + eprintln!(" page {}: first line {:?}", i + 1, first.text); + } + } + + // Stage 3 — the assembled document, i.e. what the browser actually gets. + match docling_pdf::convert_text_layer(&bytes, &name) { + Ok(doc) => { + let md = doc.export_to_markdown(); + eprintln!( + "3. convert_text_layer: {} node(s), {} chars — {}", + doc.nodes.len(), + md.len(), + if doc.nodes.is_empty() { + "EMPTY: a browser build would fall back to OCR here" + } else { + "the browser would use this directly (no OCR)" + } + ); + println!("{md}"); + } + Err(e) => { + eprintln!("3. convert_text_layer: FAILED — {e}"); + std::process::exit(1); + } + } +} + +/// When the document will not load, show the cross-reference machinery the +/// parser choked on: the last `startxref`, where it points, and what actually +/// sits there. A classic `xref` table, an xref *stream* (`/Type /XRef`), bytes +/// past the end of the file or plain garbage each point at a different repair, +/// and this is enough to tell them apart without handing the PDF around. +fn probe_tail(bytes: &[u8]) { + let find_last = |needle: &[u8]| { + bytes + .windows(needle.len()) + .rposition(|w| w == needle) + .map(|i| (i, needle.len())) + }; + let printable = |b: &[u8]| -> String { + b.iter() + .map(|&c| { + if (0x20..0x7f).contains(&c) { + c as char + } else if c == b'\n' || c == b'\r' { + '\u{23ce}' + } else { + '.' + } + }) + .collect() + }; + + eprintln!(" file: {} bytes", bytes.len()); + eprintln!( + " markers: trailer={} startxref={} xref={} /XRef={} %%EOF={}", + find_last(b"trailer").is_some(), + find_last(b"startxref").is_some(), + find_last(b"xref").is_some(), + find_last(b"/XRef").is_some(), + find_last(b"%%EOF").is_some(), + ); + + // Trailing bytes after the final %%EOF confuse a strict tail scan. + if let Some((i, len)) = find_last(b"%%EOF") { + let after = bytes.len() - (i + len); + eprintln!(" bytes after the last %%EOF: {after}"); + } + + let Some((sx, _)) = find_last(b"startxref") else { + eprintln!(" no startxref at all — the xref would have to be rebuilt by scanning objects"); + return; + }; + let tail = &bytes[sx..bytes.len().min(sx + 60)]; + eprintln!(" last startxref block: {:?}", printable(tail)); + + // The number after `startxref` is the byte offset of the xref section. + let digits: String = tail + .iter() + .skip(b"startxref".len()) + .skip_while(|c| c.is_ascii_whitespace()) + .take_while(|c| c.is_ascii_digit()) + .map(|&c| c as char) + .collect(); + match digits.parse::() { + Ok(off) if off < bytes.len() => { + let end = bytes.len().min(off + 120); + eprintln!(" at offset {off} -> {:?}", printable(&bytes[off..end])); + } + Ok(off) => eprintln!(" startxref points to {off}, past the end of the file"), + Err(_) => eprintln!(" startxref carries no parseable offset"), + } +} diff --git a/crates/docling-pdf/src/assemble.rs b/crates/docling-pdf/src/assemble.rs index 916c14cc..14d18495 100644 --- a/crates/docling-pdf/src/assemble.rs +++ b/crates/docling-pdf/src/assemble.rs @@ -895,7 +895,7 @@ fn code_region_text(region: &Region, cells: &[TextCell]) -> String { /// left edges), then place each cell. A model-free stand-in for TableFormer that /// recovers grid-aligned tables from the precise PDF text layer (it does not /// resolve row/column spans). -fn reconstruct_table(region: &Region, cells: &[TextCell]) -> Vec> { +pub fn reconstruct_table(region: &Region, cells: &[TextCell]) -> Vec> { let mut inside: Vec<&TextCell> = cells .iter() .filter(|c| { @@ -965,6 +965,59 @@ fn reconstruct_table(region: &Region, cells: &[TextCell]) -> Vec> { grid } +/// Does the geometric reconstruction of a table look trustworthy enough to use +/// as-is, instead of paying for TableFormer? +/// +/// [`reconstruct_table`] derives columns by clustering cell **left edges**. On a +/// clean grid that is exact, but when a column's entries are not left-aligned +/// (or the OCR boxes wobble) the clustering splits one real column into several, +/// and the result is a wide, mostly-empty grid — the "spurious empty columns" +/// failure TableFormer exists to fix. +/// +/// Two symptoms separate the two cases, and both are properties of the grid +/// alone (no model needed): +/// * **density** — a real table is mostly full; a split-up one is mostly holes; +/// * **thin columns** — a column carrying at most one entry across several rows +/// is almost always a split artefact rather than a real column. +/// +/// Deliberately conservative: it answers `true` only for grids that are plainly +/// well-formed, so the expensive path stays the default whenever there is doubt. +/// A caller that skips TableFormer on `true` trades no quality for the time. +pub fn geometric_table_is_reliable(rows: &[Vec]) -> bool { + let ncols = rows.iter().map(Vec::len).max().unwrap_or(0); + // Fewer than two columns is not a grid this heuristic can vouch for: it is + // exactly the shape a collapsed table takes, and TableFormer may recover + // real structure from it. + if rows.len() < 2 || ncols < 2 { + return false; + } + let filled = |c: &String| !c.trim().is_empty(); + let total = rows.len() * ncols; + let full = rows.iter().flatten().filter(|c| filled(c)).count(); + if (full as f32) < MIN_TABLE_FILL * total as f32 { + return false; + } + // A column used by at most one row, when there are rows enough to tell. + if rows.len() >= 3 { + for ci in 0..ncols { + let used = rows + .iter() + .filter(|r| r.get(ci).is_some_and(filled)) + .count(); + if used <= 1 { + return false; + } + } + } + true +} + +/// Share of a geometric grid's cells that must carry text for it to be trusted +/// without TableFormer. Chosen well above the density a left-edge split +/// produces (those land nearer a third) and below what a genuine table with a +/// few blank cells reaches. +const MIN_TABLE_FILL: f32 = 0.6; + /// The union bbox of the text cells assigned to a region (same >50%-overlap /// rule as [`region_text`]), or `None` when no cell lands in it. docling's /// LayoutPostprocessor shrinks a regular cluster's bbox to its cells, and the @@ -1043,7 +1096,7 @@ pub fn crop_region_scaled(page: &PdfPage, bbox: [f32; 4], target_scale: f32) -> /// Crop a layout region from the rendered page image and encode it as PNG (the /// figure bytes docling stores on a `PictureItem`). Region coordinates are page /// points; the image is rendered at `page.scale`. -#[cfg(feature = "ml")] +#[cfg(feature = "ocr-prep")] fn crop_region(page: &PdfPage, region: &Region) -> Option { let s = page.scale; let (iw, ih) = (page.image.width(), page.image.height()); @@ -1305,9 +1358,9 @@ pub fn assemble_page( }; // Without the page render (text-layer-only build) a picture keeps // its caption/classification but carries no cropped pixels. - #[cfg(feature = "ml")] + #[cfg(feature = "ocr-prep")] let image = crate::timing::timed("crop_region", || crop_region(page, region)); - #[cfg(not(feature = "ml"))] + #[cfg(not(feature = "ocr-prep"))] let image: Option = None; nodes.push(located( loc, @@ -1657,6 +1710,89 @@ mod tests { use crate::pdfium_backend::{LinkAnnot, PdfPage, TextCell}; use docling_core::Node; + /// The geometric-reliability gate, on the two shapes it has to tell apart. + #[test] + fn geometric_reliability_rejects_split_column_grids() { + let g = |rows: &[&[&str]]| -> Vec> { + rows.iter() + .map(|r| r.iter().map(|c| c.to_string()).collect()) + .collect() + }; + // A genuine grid: dense, every column carrying entries. Nothing for + // TableFormer to improve, so geometry is used as-is. + assert!(super::geometric_table_is_reliable(&g(&[ + &["Datum", "Leistung", "Anzahl", "Kosten"], + &["04.07", "Internet", "1", "40.30"], + &["04.07", "Telefon", "2", "8.06"], + ]))); + // The left-edge split artefact (the shape a scanned invoice produced): + // one real label column plus values scattered across three sparse ones. + assert!(!super::geometric_table_is_reliable(&g(&[ + &["www.magenta.at/faq", "", "", ""], + &["Serviceteam", "", "", ""], + &["Telefon", "0676/2000", "", ""], + &["Kundennummer", "", "", "1.21699482"], + &["Rechnungsnummer", "", "922769430725", ""], + &["Rechnungsdatum", "", "", "04.07.2025"], + ]))); + // A column only one row ever uses is a split artefact even when the + // grid is otherwise dense. + assert!(!super::geometric_table_is_reliable(&g(&[ + &["a", "b", ""], + &["c", "d", ""], + &["e", "f", "g"], + ]))); + // Degenerate shapes are never vouched for — TableFormer may recover + // structure a collapsed reconstruction lost. + assert!(!super::geometric_table_is_reliable(&g(&[&[ + "only one column" + ]]))); + assert!(!super::geometric_table_is_reliable(&[])); + } + + /// A `picture` region is cropped out of the rendered page, whatever built + /// that page. The browser pipeline (#157) has no pdfium but does hand over + /// the rasterized bitmap through `from_cells_with_image`, so it must get + /// the same figure bytes the native path does — that is what makes + /// `images = "embedded"` inline real pixels instead of a placeholder. + #[cfg(feature = "ocr-prep")] + #[test] + fn picture_regions_are_cropped_from_a_host_supplied_page_image() { + let mut img = image::RgbImage::new(200, 200); + // Paint the figure area so the crop is distinguishable from the page. + for y in 100..160 { + for x in 20..120 { + img.put_pixel(x, y, image::Rgb([255, 0, 0])); + } + } + // scale 2.0: the region is in page points, the bitmap in pixels. + let page = PdfPage::from_cells_with_image(100.0, 100.0, 2.0, Vec::new(), img); + let region = Region { + label: "picture", + score: 0.9, + l: 10.0, + t: 50.0, + r: 60.0, + b: 80.0, + }; + let (nodes, _) = super::assemble_page(&page, vec![region], &[None], &[None]); + // Layout-derived nodes carry provenance, so the picture arrives wrapped. + let image = nodes + .iter() + .find_map(|n| match n { + Node::Located { inner, .. } => match &**inner { + Node::Picture { image, .. } => image.as_ref(), + _ => None, + }, + Node::Picture { image, .. } => image.as_ref(), + _ => None, + }) + .expect("a picture node with cropped pixels"); + assert_eq!(image.mimetype, "image/png"); + assert_eq!((image.width, image.height), (100, 60), "region × scale"); + assert!(!image.data.is_empty(), "PNG bytes were encoded"); + } + #[test] fn link_anchors_split_a_shared_word_cell_between_adjacent_links() { // A common header layout: one text run holds several pipe-separated diff --git a/crates/docling-pdf/src/lib.rs b/crates/docling-pdf/src/lib.rs index ddc4397b..85b71d53 100644 --- a/crates/docling-pdf/src/lib.rs +++ b/crates/docling-pdf/src/lib.rs @@ -14,6 +14,11 @@ // build still flags genuinely dead code). #![cfg_attr(not(feature = "ml"), allow(dead_code))] +// Reading-order assembly. Public under `ocr-prep` so the browser pipeline can +// reuse the geometric table reconstruction and its reliability gate (#157). +#[cfg(feature = "ocr-prep")] +pub mod assemble; +#[cfg(not(feature = "ocr-prep"))] mod assemble; mod dp_lines; #[cfg(feature = "ml")] diff --git a/crates/docling-pdf/src/ocr_prep.rs b/crates/docling-pdf/src/ocr_prep.rs index 1b1da9b1..0c0be1b5 100644 --- a/crates/docling-pdf/src/ocr_prep.rs +++ b/crates/docling-pdf/src/ocr_prep.rs @@ -435,3 +435,48 @@ mod tests { assert_eq!(decode_row(&chars, &probs, 4), "ab"); } } + +#[cfg(test)] +mod word_segmentation { + use image::{Rgb, RgbImage}; + + /// A white line carrying two ink blocks separated by `gap` pixels. + fn line_with_gap(h: u32, gap: u32) -> RgbImage { + let w = 30 + gap + 30 + 10; + let mut img = RgbImage::from_pixel(w, h, Rgb([255, 255, 255])); + for (x0, x1) in [(5u32, 35u32), (35 + gap, 65 + gap)] { + for x in x0..x1.min(w) { + for y in h / 4..(3 * h / 4) { + img.put_pixel(x, y, Rgb([0, 0, 0])); + } + } + } + img + } + + /// [`segment_words`] splits on a gap of `0.6 x line height`, which is the + /// property the rest of the browser table path inherits: an inter-word + /// space never splits, but a column gap wider than that does. Anything + /// narrower stays a single box — and a single box can only ever land in one + /// TableFormer cell, so this threshold is the floor on how tight a table's + /// columns may be before its cells merge. + #[test] + fn words_split_only_on_gaps_above_six_tenths_of_the_line_height() { + for h in [16u32, 24, 32, 40] { + let split_at = (1..=40u32) + .find(|&gap| super::segment_words(&line_with_gap(h, gap)).len() >= 2) + .expect("some gap splits"); + let ratio = split_at as f32 / h as f32; + assert!( + (0.5..=0.65).contains(&ratio), + "h={h}: split at {split_at}px ({ratio:.2} x height)" + ); + // Just below the threshold the two blocks are one word. + assert_eq!( + super::segment_words(&line_with_gap(h, split_at - 1)).len(), + 1, + "h={h}: a narrower gap must not split" + ); + } + } +} diff --git a/crates/docling-pdf/src/pdfium_backend.rs b/crates/docling-pdf/src/pdfium_backend.rs index f2d8a932..612eb9ad 100644 --- a/crates/docling-pdf/src/pdfium_backend.rs +++ b/crates/docling-pdf/src/pdfium_backend.rs @@ -9,7 +9,7 @@ //! loop is driven through the raw `PdfiumLibraryBindings` FFI on a second handle //! to the same bytes (no fork; stays publishable). -#[cfg(feature = "ml")] +#[cfg(feature = "ocr-prep")] use image::RgbImage; #[cfg(feature = "ml")] use pdfium_render::prelude::*; @@ -46,7 +46,11 @@ pub struct PdfPage { /// Per-word cells (one per word, not joined into lines) for TableFormer cell /// matching. pub word_cells: Vec, - #[cfg(feature = "ml")] + /// The rendered page bitmap. Present whenever pixels are available at all + /// (`ocr-prep` ⊂ `ml`): the native pipeline renders it with pdfium, the + /// browser pipeline receives it from the host canvas. Picture regions are + /// cropped out of it. + #[cfg(feature = "ocr-prep")] pub image: RgbImage, /// Hyperlink annotations on the page (rect in top-left page coords + target /// URI), restricted to web/mail/tel schemes. Used only by strict Markdown. @@ -68,11 +72,28 @@ impl PdfPage { cells, code_cells: Vec::new(), word_cells: Vec::new(), - #[cfg(feature = "ml")] + #[cfg(feature = "ocr-prep")] image: RgbImage::new(0, 0), links: Vec::new(), } } + + /// Same as [`from_cells`](Self::from_cells) but carrying the rendered page + /// bitmap, so picture regions can be cropped out of it (#157: the browser + /// pipeline gets the same figure bytes the native one does). + #[cfg(feature = "ocr-prep")] + pub fn from_cells_with_image( + width: f32, + height: f32, + scale: f32, + cells: Vec, + image: RgbImage, + ) -> Self { + Self { + image, + ..Self::from_cells(width, height, scale, cells) + } + } } /// A PDF link annotation: its rectangle (top-left page coordinates, matching diff --git a/crates/docling-pdf/src/scanned.rs b/crates/docling-pdf/src/scanned.rs index fbfb1cc8..95c56645 100644 --- a/crates/docling-pdf/src/scanned.rs +++ b/crates/docling-pdf/src/scanned.rs @@ -33,6 +33,69 @@ pub fn refine_regions( regions } +/// Drop text regions that would read the same cells twice. +/// +/// The layout model can emit overlapping detections — typically one region +/// covering a whole block *and* smaller ones covering its parts, when the +/// raster is noisier than the model likes (the browser's pdf.js render, where +/// this was observed as a paragraph printed twice; pdfium's render of the same +/// page did not trigger it). Assembly assigns each region the cells >50% +/// inside it without consuming them, so every such overlap duplicates text. +/// +/// The rule is deliberately narrow: a *text-like* region is dropped only when +/// every cell it claims is also claimed by one *single* other region — i.e. it +/// is a strict re-reader of part of a larger region. The larger region +/// survives, which is also what the native raster produces on the observed +/// page (one merged paragraph). Ties (two regions claiming identical sets) +/// keep the higher-scoring one. +pub fn drop_duplicate_text_claims(regions: Vec, cells: &[TextCell]) -> Vec { + let claimed: Vec> = regions + .iter() + .map(|r| { + cells + .iter() + .enumerate() + .filter(|(_, c)| { + let ca = ((c.r - c.l) * (c.b - c.t)).max(1.0); + let iw = (c.r.min(r.r) - c.l.max(r.l)).max(0.0); + let ih = (c.b.min(r.b) - c.t.max(r.t)).max(0.0); + iw * ih / ca > 0.5 + }) + .map(|(i, _)| i) + .collect() + }) + .collect(); + let text_like = |label: &str| matches!(label, "text" | "list_item" | "section_header"); + let mut drop = vec![false; regions.len()]; + for a in 0..regions.len() { + if !text_like(regions[a].label) || claimed[a].is_empty() { + continue; + } + for b in 0..regions.len() { + if a == b || drop[b] || !text_like(regions[b].label) { + continue; + } + let subset = claimed[a].iter().all(|i| claimed[b].contains(i)); + if !subset { + continue; + } + // Identical sets: keep the higher score (stable on ties by index). + let identical = claimed[a].len() == claimed[b].len(); + if identical && (regions[a].score, b) > (regions[b].score, a) { + continue; + } + drop[a] = true; + break; + } + } + regions + .into_iter() + .zip(drop) + .filter(|(_, d)| !d) + .map(|(r, _)| r) + .collect() +} + /// Assemble one refined page — geometric tables (no TableFormer), no /// enrichments — into its nodes and hyperlink pairs. pub fn assemble_page(page: &PdfPage, regions: Vec) -> AssembledPage { @@ -65,3 +128,73 @@ pub fn finish_document(name: &str, pages: Vec) -> DoclingDocument crate::assemble::merge_continuations(&mut doc.nodes); doc } + +#[cfg(test)] +mod duplicate_claims { + use crate::layout::Region; + use crate::pdfium_backend::TextCell; + + fn cell(l: f32, t: f32, r: f32, b: f32) -> TextCell { + TextCell { + text: "x".into(), + l, + t, + r, + b, + } + } + fn region(label: &'static str, score: f32, l: f32, t: f32, r: f32, b: f32) -> Region { + Region { + label, + score, + l, + t, + r, + b, + } + } + + /// The observed browser failure: the model detects a block *and* its two + /// halves, so both halves re-read cells the block already claims and a + /// paragraph prints twice. The parts must go, the block must stay — that + /// matches what the native raster produced (one merged paragraph). + #[test] + fn part_regions_of_a_block_are_dropped() { + let cells = vec![cell(10.0, 10.0, 90.0, 20.0), cell(10.0, 30.0, 90.0, 40.0)]; + let regions = vec![ + region("text", 0.9, 5.0, 5.0, 95.0, 45.0), // the block + region("text", 0.8, 5.0, 5.0, 95.0, 25.0), // its top half + region("text", 0.8, 5.0, 25.0, 95.0, 45.0), // its bottom half + ]; + let kept = super::drop_duplicate_text_claims(regions, &cells); + assert_eq!(kept.len(), 1, "kept: {kept:?}"); + assert_eq!((kept[0].t, kept[0].b), (5.0, 45.0), "the block survives"); + } + + /// Disjoint regions — the normal page shape — are untouched, and so are + /// non-text labels even when they overlap text (a table region legitimately + /// contains the same cells its caption/text neighbours touch). + #[test] + fn disjoint_and_non_text_regions_are_kept() { + let cells = vec![cell(10.0, 10.0, 90.0, 20.0), cell(10.0, 50.0, 90.0, 60.0)]; + let regions = vec![ + region("text", 0.9, 5.0, 5.0, 95.0, 25.0), + region("text", 0.9, 5.0, 45.0, 95.0, 65.0), + region("table", 0.9, 5.0, 5.0, 95.0, 65.0), // overlaps both, not text + ]; + assert_eq!(super::drop_duplicate_text_claims(regions, &cells).len(), 3); + } + + /// Two identical claims keep exactly one — the higher score. + #[test] + fn identical_claims_keep_the_higher_score() { + let cells = vec![cell(10.0, 10.0, 90.0, 20.0)]; + let regions = vec![ + region("text", 0.7, 5.0, 5.0, 95.0, 25.0), + region("text", 0.9, 6.0, 6.0, 94.0, 24.0), + ]; + let kept = super::drop_duplicate_text_claims(regions, &cells); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].score, 0.9); + } +} diff --git a/crates/docling-pdf/src/textparse.rs b/crates/docling-pdf/src/textparse.rs index 040ee09f..4b282159 100644 --- a/crates/docling-pdf/src/textparse.rs +++ b/crates/docling-pdf/src/textparse.rs @@ -634,10 +634,263 @@ fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) { (612.0, 792.0) } +/// Localize where a page's text is lost, for the `text_layer` diagnostic. +/// Extraction can come up empty at three different points — no content stream +/// reached the parser, the stream did not decode into operators, or it ran but +/// produced no glyphs (fonts/encodings) — and from the outside all three look +/// the same. Report them per page. +pub fn content_diagnosis(bytes: &[u8]) -> String { + let Some(doc) = load_document(bytes) else { + return "document does not load".into(); + }; + let mut pages: Vec<_> = doc.get_pages().into_iter().collect(); + pages.sort_by_key(|(n, _)| *n); + let mut out = String::new(); + let mut caches = DocCaches::default(); + for (n, pid) in pages.into_iter().take(4) { + let content_bytes = doc.get_page_content(pid); + let ops = lopdf::content::Content::decode(&content_bytes) + .map(|c| c.operations.len()) + .ok(); + let res = page_res(&doc, pid); + let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len()); + let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len(); + out.push_str(&format!( + "\n page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}", + content_bytes.len(), + ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()), + if res.is_some() { "ok" } else { "MISSING" }, + fonts.map_or("-".to_string(), |n| n.to_string()), + glyphs, + )); + } + out +} + +/// Why the cross-reference repair did or did not fire, for the `text_layer` +/// diagnostic. A PDF that will not load is indistinguishable from a scan in +/// production (both convert to nothing), so the reason has to be askable. +pub fn xref_repair_status(bytes: &[u8]) -> String { + if Document::load_mem(bytes).is_ok() { + return "loads unaided; no repair needed".into(); + } + match pad_short_xref_entries(bytes) { + Ok(fixed) => match Document::load_mem(&fixed) { + Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(), + Err(e) => format!("padded the entries, but it still will not load: {e}"), + }, + Err(why) => format!("repair declined — {why}"), + } +} + +/// Load a PDF, repairing the one malformation that otherwise costs us the whole +/// document: **19-byte cross-reference entries**. +/// +/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a +/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one) +/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file +/// outright (`invalid file trailer`) where pdfium reads it happily, so a +/// perfectly good text layer looked to the browser exactly like a scan and cost +/// ten seconds of OCR. +/// +/// Padding is only attempted when it cannot move anything the xref points at: +/// a single `xref` section that begins after the last object. The repair then +/// has to prove itself — the padded bytes are used only if they load — so a +/// mis-repair degrades to today's behaviour rather than to silent garbage. +fn load_document(bytes: &[u8]) -> Option { + // Try progressively more repair, and accept a candidate only once the pages + // actually carry content — a document whose streams were dropped still + // "loads", so loading alone is not evidence the repair helped. A + // well-formed file returns on the first attempt and pays for nothing. + let mut fallback = None; + if let Some(doc) = best_effort_load(bytes, &mut fallback) { + return Some(doc); + } + let xref_fixed = pad_short_xref_entries(bytes).ok(); + if let Some(fixed) = &xref_fixed { + if let Some(doc) = best_effort_load(fixed, &mut fallback) { + return Some(doc); + } + } + // Both defects can coexist, and the second only becomes visible once the + // first is repaired, so build on whatever the previous step produced. + let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes)); + if let Some(doc) = best_effort_load(&lengths_fixed, &mut fallback) { + return Some(doc); + } + fallback +} + +/// Load `data`, returning it only when its pages carry content; a document that +/// merely parses is remembered as the fallback for when nothing does better. +fn best_effort_load(data: &[u8], fallback: &mut Option) -> Option { + match Document::load_mem(data) { + Ok(doc) if has_page_content(&doc) => Some(doc), + Ok(doc) => { + fallback.get_or_insert(doc); + None + } + Err(_) => None, + } +} + +/// Does any page actually hand us a content stream? A document whose streams +/// were dropped still parses — it simply has nothing to read — so this is what +/// tells a successful repair from a pointless one. +fn has_page_content(doc: &Document) -> bool { + doc.get_pages() + .into_values() + .take(4) + .any(|pid| !doc.get_page_content(pid).is_empty()) +} + +/// Correct `/Length` values that disagree with where `endstream` actually is. +/// +/// The same generator that writes short xref entries also overstates its +/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past +/// the data, fails to find `endstream` there and drops the stream — the object +/// comes back as a bare dictionary, so the page has no content at all and the +/// document looks like a scan. pdfium instead trusts `endstream`, which is what +/// this does. +/// +/// The rewrite is length-preserving: the corrected number is written over the +/// old digits and padded with spaces, so every byte offset in the file — and +/// therefore the whole cross-reference table — stays valid. +fn fix_stream_lengths(bytes: &[u8]) -> Vec { + let mut out = bytes.to_vec(); + let mut i = 0; + while let Some(rel) = find(&out[i..], b"stream") { + let kw = i + rel; + i = kw + 6; + // Skip `endstream` (the keyword we are measuring *to*). + if kw >= 3 && &out[kw - 3..kw] == b"end" { + continue; + } + // The stream data starts after the EOL that follows the keyword. + let mut data = kw + 6; + if out.get(data..data + 2) == Some(b"\r\n".as_slice()) { + data += 2; + } else if matches!(out.get(data), Some(b'\n' | b'\r')) { + data += 1; + } + let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else { + continue; + }; + // `/Length ` in the dictionary just before the keyword. + let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0); + let Some(lrel) = find(&out[dict_start..kw], b"/Length") else { + continue; + }; + let mut d = dict_start + lrel + 7; + while matches!(out.get(d), Some(b' ')) { + d += 1; + } + let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count(); + if digits == 0 { + continue; + } + let declared: usize = match std::str::from_utf8(&out[d..d + digits]) + .ok() + .and_then(|s| s.parse().ok()) + { + Some(v) => v, + None => continue, + }; + let actual = end - data; + // Only shrink, and only when the new value fits the space the old one + // occupied — growing the number would move every following byte. + let replacement = actual.to_string(); + if actual == declared || replacement.len() > digits { + continue; + } + out[d..d + digits].fill(b' '); + out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes()); + } + out +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes, +/// or `None` when the file's shape makes that unsafe (see [`load_document`]). +fn pad_short_xref_entries(bytes: &[u8]) -> Result, &'static str> { + // Exactly one xref section, and it must start after every object, so that + // growing it shifts nothing the table's offsets refer to. + let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r'); + let mut starts = (0..bytes.len().saturating_sub(4)) + .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i)); + let xref_at = starts + .next() + .ok_or("no classic `xref` section (an xref stream?)")?; + if starts.next().is_some() { + return Err("more than one xref section (incremental update)"); + } + let last_obj = bytes + .windows(3) + .rposition(|w| w == b"obj") + .ok_or("no objects found")?; + if last_obj > xref_at { + return Err("an object follows the xref — padding would move it"); + } + + let mut out = bytes[..xref_at].to_vec(); + out.extend_from_slice(b"xref\n"); + let mut i = xref_at + 4; + let skip_ws = |i: &mut usize| { + while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) { + *i += 1; + } + }; + loop { + skip_ws(&mut i); + // Either the next subsection header ("first count") or the trailer. + if bytes[i..].starts_with(b"trailer") { + out.extend_from_slice(&bytes[i..]); + return Ok(out); + } + let header_end = i + bytes[i..] + .iter() + .position(|c| matches!(c, b'\n' | b'\r')) + .ok_or("subsection header runs off the end")?; + let header = std::str::from_utf8(&bytes[i..header_end]) + .map_err(|_| "subsection header is not text")? + .trim(); + let mut parts = header.split_whitespace(); + let count: usize = parts + .nth(1) + .and_then(|c| c.parse().ok()) + .ok_or("unparseable subsection header")?; + if parts.next().is_some() || count == 0 { + return Err("unexpected subsection header shape"); + } + out.extend_from_slice(header.as_bytes()); + out.push(b'\n'); + i = header_end; + for _ in 0..count { + skip_ws(&mut i); + // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows. + let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?; + let well_formed = entry[..10].iter().all(u8::is_ascii_digit) + && entry[10] == b' ' + && entry[11..16].iter().all(u8::is_ascii_digit) + && entry[16] == b' ' + && matches!(entry[17], b'n' | b'f'); + if !well_formed { + return Err("xref entry is not `nnnnnnnnnn ggggg n`"); + } + out.extend_from_slice(entry); + out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes + i += 18; + } + } +} + /// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page /// `index`, before the sanitizer. For comparing char cells to docling-parse. pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> { - let Ok(doc) = Document::load_mem(bytes) else { + let Some(doc) = load_document(bytes) else { return Vec::new(); }; let mut pages: Vec<_> = doc.get_pages().into_iter().collect(); @@ -655,7 +908,7 @@ pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32 /// text parser + the docling-parse line sanitizer. Used by the pipeline and the /// `textparse_dump` example. pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec)> { - let Ok(doc) = Document::load_mem(bytes) else { + let Some(doc) = load_document(bytes) else { return Vec::new(); }; let mut caches = DocCaches::default(); @@ -677,7 +930,7 @@ pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec Vec<(f32, f32, Vec)> { - let Ok(doc) = Document::load_mem(bytes) else { + let Some(doc) = load_document(bytes) else { return Vec::new(); }; let mut caches = DocCaches::default(); @@ -709,7 +962,7 @@ pub struct PageParserCells { /// `code` splits only at the parser's own space glyphs (monospace keeps its /// source spacing). Used by the pipeline to retire pdfium's text path. pub fn pdf_all_cells(bytes: &[u8]) -> Vec { - let Ok(doc) = Document::load_mem(bytes) else { + let Some(doc) = load_document(bytes) else { return Vec::new(); }; let mut caches = DocCaches::default(); @@ -737,7 +990,7 @@ pub fn pdf_all_cells(bytes: &[u8]) -> Vec { /// wasm32. A page the parser can't read (no text layer) comes back with empty /// cells; there is no pdfium fallback on this path. pub fn pdf_text_pages(bytes: &[u8]) -> Vec { - let Ok(doc) = Document::load_mem(bytes) else { + let Some(doc) = load_document(bytes) else { return Vec::new(); }; let mut caches = DocCaches::default(); @@ -748,7 +1001,9 @@ pub fn pdf_text_pages(bytes: &[u8]) -> Vec { .map(|(_, pid)| { let (w, h) = page_size(&doc, pid); let glyphs = page_glyphs_cached(&doc, pid, &mut caches); - let (prose, words) = crate::dp_lines::line_and_word_cells(&glyphs, h, true); + let (mut prose, mut words) = crate::dp_lines::line_and_word_cells(&glyphs, h, true); + drop_overpainted_cells(&mut prose); + drop_overpainted_cells(&mut words); crate::pdfium_backend::PdfPage { width: w, height: h, @@ -757,7 +1012,7 @@ pub fn pdf_text_pages(bytes: &[u8]) -> Vec { cells: prose, code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h), word_cells: words, - #[cfg(feature = "ml")] + #[cfg(feature = "ocr-prep")] image: image::RgbImage::new(1, 1), links: Vec::new(), } @@ -765,6 +1020,49 @@ pub fn pdf_text_pages(bytes: &[u8]) -> Vec { .collect() } +/// Drop line cells that are *painted over each other* — glyphs used as artwork. +/// +/// Some generators draw their logo with a symbol font: on the reporting +/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs +/// encoded as `"` and `==` on top of one another, and the flat text-layer +/// output opened with that garbage. Nothing in the font metadata gives it away +/// (the *text* fonts in the same file are also flagged symbolic, and the logo +/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with +/// different text where one lies inside the other on the same line is +/// physically impossible for prose — ink from two words never occupies the +/// same box. Both cells of such a pair are paint, not text. +/// +/// Containment (not mere overlap) keeps this narrow: adjacent words touch but +/// never contain each other, and a same-text near-duplicate (double-draw faux +/// bold) is left alone for the sanitizer's usual handling. Applied on the +/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the +/// PDF corpus, and there the layout model already sinks logo marks into +/// `picture` regions. +fn drop_overpainted_cells(cells: &mut Vec) { + let mut paint = vec![false; cells.len()]; + for i in 0..cells.len() { + for j in 0..cells.len() { + if i == j || cells[i].text == cells[j].text { + continue; + } + let (a, b) = (&cells[i], &cells[j]); + // Same line band: the vertical overlap covers most of the shorter. + let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0); + if vo < 0.6 * (a.b - a.t).min(b.b - b.t) { + continue; + } + // `a` horizontally inside `b` (with a small tolerance). + let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0); + if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) { + paint[i] = true; + paint[j] = true; + } + } + } + let mut keep = paint.iter().map(|p| !p); + cells.retain(|_| keep.next().unwrap()); +} + /// The text-state scalars inherited by a Form XObject when it is invoked via /// `Do` (the PDF graphics state includes the text parameters, but not the text /// matrices, which a form re-establishes inside its own `BT`/`ET`). @@ -1489,3 +1787,192 @@ fn macroman_table() -> HashMap { } m } + +#[cfg(test)] +mod xref_repair { + /// Build a tiny one-page PDF whose cross-reference entries are either the + /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators + /// emit — everything else about the two files is identical. + fn pdf_with_xref(two_byte_eol: bool) -> Vec { + let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n"; + let stream = format!("<>stream\n", content.len()).into_bytes(); + let objs: Vec> = vec![ + b"<>".to_vec(), + b"<>".to_vec(), + b"<>>>>>" + .to_vec(), + [stream.as_slice(), content.as_slice(), b"endstream"].concat(), + b"<>".to_vec(), + ]; + + let mut out = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (i, body) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"endobj\n"); + } + let xref_at = out.len(); + let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" }; + out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes()); + out.extend_from_slice(b"0000000000 65535 f"); + out.extend_from_slice(eol); + for off in &offsets { + out.extend_from_slice(format!("{off:010} 00000 n").as_bytes()); + out.extend_from_slice(eol); + } + out.extend_from_slice( + format!("trailer<>\n", objs.len() + 1).as_bytes(), + ); + out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes()); + out + } + + /// A 19-byte cross-reference entry (a bare LF where the spec wants a + /// two-byte EOL) makes lopdf reject the whole file, so a readable text + /// layer used to look exactly like a scan — in the browser that meant ten + /// seconds of OCR for nothing. The repair must recover *the same* parse the + /// well-formed file gives. + #[test] + fn short_xref_entries_still_parse() { + let good = pdf_with_xref(true); + let broken = pdf_with_xref(false); + assert!( + broken.len() < good.len(), + "the broken file is the shorter one" + ); + assert!( + lopdf::Document::load_mem(&good).is_ok(), + "the control file must load unaided" + ); + assert!( + lopdf::Document::load_mem(&broken).is_err(), + "lopdf rejects 19-byte entries — if this ever passes, drop the repair" + ); + + let cells = |b: &[u8]| -> Vec { + super::pdf_textlines(b) + .into_iter() + .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text)) + .collect() + }; + let from_good = cells(&good); + assert!( + from_good.iter().any(|t| t.contains("922769430725")), + "control text: {from_good:?}" + ); + assert_eq!( + cells(&broken), + from_good, + "repair must match the good parse" + ); + } + + /// The same generator overstates `/Length`, so lopdf reads past the data, + /// misses `endstream` and drops the stream — the object comes back as a + /// bare dictionary and the page has no content at all. Trust `endstream` + /// instead, and do it without moving a single byte. + #[test] + fn overstated_stream_length_still_yields_content() { + let good = pdf_with_xref(true); + // Inflate the content stream's /Length by one, exactly as the invoice + // that prompted this does. + let broken = { + let at = good + .windows(8) + .position(|w| w == b"/Length ") + .expect("a /Length") + + 8; + let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count(); + let n: usize = std::str::from_utf8(&good[at..at + digits]) + .unwrap() + .parse() + .unwrap(); + let inflated = (n + 1).to_string(); + assert_eq!(inflated.len(), digits, "keep the digit count"); + let mut b = good.clone(); + b[at..at + digits].copy_from_slice(inflated.as_bytes()); + b + }; + assert_eq!(broken.len(), good.len(), "the defect must not move bytes"); + // lopdf alone loses the stream: the page parses but carries no content. + let raw = lopdf::Document::load_mem(&broken).expect("still loads"); + assert!( + raw.get_pages() + .into_values() + .all(|p| raw.get_page_content(p).is_empty()), + "lopdf should drop the stream — if it stops, drop this repair" + ); + // Ours recovers the same text the well-formed file gives. + let text = |b: &[u8]| -> Vec { + super::pdf_textlines(b) + .into_iter() + .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text)) + .collect() + }; + let expected = text(&good); + assert!(!expected.is_empty(), "control must produce text"); + assert_eq!(text(&broken), expected); + } + + /// The repair only fires where padding cannot move an object: it declines a + /// file whose xref precedes an object (an incremental update), rather than + /// shifting every offset the table records. + #[test] + fn repair_declines_when_padding_would_move_objects() { + let mut incremental = pdf_with_xref(false); + incremental.extend_from_slice(b"6 0 obj<>endobj\n"); + let declined = super::pad_short_xref_entries(&incremental).unwrap_err(); + assert!( + declined.contains("object follows the xref"), + "reason: {declined}" + ); + } +} + +#[cfg(test)] +mod overpainted { + use crate::pdfium_backend::TextCell; + + fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell { + TextCell { + text: text.into(), + l, + t, + r, + b, + } + } + + /// The reporting invoice's logo: a `"` painted inside a `==` on one band — + /// artwork drawn with glyphs. Both cells go; the real text on the next + /// band stays. + #[test] + fn stacked_logo_glyphs_are_dropped() { + let mut cells = vec![ + cell("\"", 72.7, 21.5, 86.4, 31.5), + cell("==", 59.4, 21.5, 99.6, 31.5), + cell("Herr", 65.2, 151.3, 81.7, 161.3), + ]; + super::drop_overpainted_cells(&mut cells); + assert_eq!(cells.len(), 1, "cells: {cells:?}"); + assert_eq!(cells[0].text, "Herr"); + } + + /// Adjacent words on a line touch but never contain each other — prose is + /// untouched, and so is a same-text near-duplicate (double-drawn faux + /// bold), which is not evidence of artwork. + #[test] + fn prose_and_double_draw_are_kept() { + let mut cells = vec![ + cell("Telefon", 354.3, 133.2, 381.5, 143.2), + cell("0676/2000", 387.3, 133.2, 428.7, 143.2), + cell("Bold", 100.0, 50.0, 130.0, 60.0), + cell("Bold", 100.3, 50.0, 130.3, 60.0), + ]; + super::drop_overpainted_cells(&mut cells); + assert_eq!(cells.len(), 4); + } +} diff --git a/crates/docling-wasm/README.md b/crates/docling-wasm/README.md index 6a5011fe..a04be7fc 100644 --- a/crates/docling-wasm/README.md +++ b/crates/docling-wasm/README.md @@ -26,12 +26,84 @@ converters and the PDF text parser are pure Rust. ## API ```ts -convert(bytes: Uint8Array, filename: string, to?: "md" | "json" | "doclang"): string +convert( + bytes: Uint8Array, + filename: string, + to?: "md" | "json" | "doclang", // default "md" + images?: "placeholder" | "embedded", // default "placeholder", Markdown only +): string supported_extensions(): string // JSON array, e.g. for version(): string ``` -The filename's extension drives format detection, same as the CLI. +The filename's extension drives format detection, same as the CLI. `images` +mirrors docling-serve's option: `placeholder` emits docling's ``, +`embedded` inlines the picture as a `data:` URI so the Markdown is +self-contained (a page has no filesystem, so there is no `referenced` mode). + +### Convert a file the user picked + +```js +import init, { convert } from "./pkg/docling_wasm.js"; +await init(); + +const file = input.files[0]; +const bytes = new Uint8Array(await file.arrayBuffer()); +const markdown = convert(bytes, file.name, "md"); +const json = convert(bytes, file.name, "json"); +const withPics = convert(bytes, file.name, "md", "embedded"); +``` + +### Digital PDFs with structure (no OCR) + +A PDF that carries a text layer needs no recognition at all — but headings, +tables and pictures are things the *layout* model finds, so the pure text-layer +path (`convert`) can only emit flat paragraphs. `DigitalConverter` closes that +gap: the text comes out of the file (exact, no recognition errors) and only the +layout model runs over the rendered pages. It is the same thing the native +pipeline does, which OCRs a page only when it has no text cells. + +```js +const conv = new DigitalConverter(pdfBytes); // throws when there is no text layer +for (let i = 0; i < conv.page_count(); i++) { + // rasterize page i with pdf.js at 2 px/point, then: + await conv.add_page(i, rgba, w, h, 2.0, layout); +} +const markdown = conv.finish("bill.pdf", "md", "embedded"); +``` + +`addPageTf(..., tf)` adds TableFormer, still only for the tables whose geometric +reconstruction looks unreliable. No recognition model is fetched on this path. + +### Scanned pages (OCR) + +Scanned PDFs and images need the ML models; everything else runs with no +network at all. The full wiring — model resolution, Web Worker, pdf.js +rasterization — is [`www/index.html`](./www/index.html); the short version: + +```js +import { createOcr } from "./pipeline.js"; + +const ocr = createOcr({ onStatus: (msg) => console.log(msg) }); +await ocr.boot(); // wasm + layout model + +// A standalone image is its own page. +const md = await ocr.convertImage(await file.arrayBuffer(), file.name, "en", "md"); + +// A scanned PDF: feed rasterized pages in order (2 px/point = the native +// pipeline's RENDER_SCALE), then finish. +await ocr.startDoc("en", /* TableFormer */ false); +for (const page of pages) await ocr.addPage(page.rgba, page.w, page.h, 2.0); +const doc = ocr.finishDoc(file.name, "md"); +``` + +Scanned pictures are cropped out of the rendered page just like the native +pipeline, so `images: "embedded"` inlines real figure bytes on this path too +(`ocr.finishDoc(name, "md", "embedded")`). + +Models resolve **device file → local `./models/` → Hugging Face**, so a page can +ship with no models and still work: `ocr.setProvidedModels({ "layout_heron_int8.onnx": buf })` +takes files the user picked, and anything not provided is fetched. ## Build @@ -49,8 +121,15 @@ wasm-bindgen --target web --out-dir crates/docling-wasm/www/pkg \ ## Demo -[`www/index.html`](./www/index.html) is a drop-a-file demo page over the -module (output selector, conversion timing, automated-test hook). After the +**Live: ** — deployed from +`www/` by [`.github/workflows/pages.yml`](../../.github/workflows/pages.yml) on +every push to `master`. No models are published with it, so the declarative +converters and text-layer PDFs work straight away and OCR starts once you give +the page models (device picker or Hugging Face — see below). + +[`www/index.html`](./www/index.html) is the whole thing on one page: drop a +file, pick the output (Markdown / JSON / DocLang), pick how images render, and +optionally turn on OCR for scanned pages. To run it locally, after the `wasm-bindgen` step above: ```bash @@ -58,6 +137,11 @@ python3 -m http.server -d crates/docling-wasm/www 8901 # open http://127.0.0.1:8901/ ``` +It is a plain static page — copy `www/` behind any web server (or into a mobile +app's webview) and it works as-is. The OCR half loads lazily, so a visitor who +only converts a DOCX never downloads ONNX Runtime, pdf.js or any model. PDFs +try their text layer first and fall back to OCR only when there isn't one. + Verified end-to-end in headless Chromium: Markdown/DOCX→md, DOCX→JSON, a corpus PDF→md through the text-layer path, and the scanned-PDF error path all exercised through the real wasm module. @@ -80,7 +164,19 @@ wasm output matching native — drift can only come from the runtime kernels. |---|---|---| | **1** — `ocr_image` | OCR a single scanned **image** | PP-OCRv3 recognition (~10 MB) | | **2** — `ScannedConverter` / `convert_scanned_image` | Scanned **PDF/image** → Markdown: layout + OCR + reading order, tables via geometric reconstruction | + RT-DETR layout (`layout_heron_int8.onnx`, ~68 MB) | -| **3** — `ScannedConverter.addPageTf` | Real **TableFormer** table structure instead of geometric | + `tableformer/{encoder,decoder_kv,bbox}.onnx` (~380 MB) | +| **3** — `ScannedConverter.addPageTf` | Real **TableFormer** table structure, on the tables that need it | + `tableformer/{encoder,decoder_kv,bbox}.onnx` (~380 MB) | + +Stage 3 is *selective*. TableFormer's encoder runs once per table region and +costs seconds, so each table is first reconstructed geometrically (free, from +the OCR cell positions) and the model is invoked only when that grid looks +unreliable — `docling_pdf::assemble::geometric_table_is_reliable`. The tell is +how `reconstruct_table` derives columns: it clusters cell *left edges*, which is +exact on a clean grid but splits one real column into several when entries are +not left-aligned, leaving a wide, mostly-empty table. So a grid is trusted only +when it is dense (≥60% of cells carry text) and has no column that just one row +uses; anything else goes to TableFormer. Well-formed tables therefore cost +nothing extra, and the pages that used to show spurious empty columns still get +the model. Stage 1 recognition wrapper: @@ -113,10 +209,9 @@ Stage 3 is the same but `conv.addPageTf(rgba, w, h, 2.0, layout, rec, tf)`, where `tf` is a stateful session over the three TableFormer graphs (the heavy cross-attention K/V and the growing decoder KV-cache stay on the JS side, so each decode step marshals only the last tag and gets back logits+hidden — see -[`www/pipeline.js`](./www/pipeline.js)'s `JsTfSession`). The demos wire all of -this up: [`www/ocr.html`](./www/ocr.html) (stage 1), -[`www/scan.html`](./www/scan.html) (stages 2–3, pdf.js + a TableFormer toggle + -a device model-file picker). +[`www/pipeline.js`](./www/pipeline.js)'s `JsTfSession`). +[`www/index.html`](./www/index.html) wires all three stages up behind the OCR +language selector, the TableFormer toggle and the model picker. ## Run it on your phone @@ -136,8 +231,8 @@ same-origin, from a CORS host, **or straight from files on your device**. Pages or githack.) Build the wasm `pkg/` first (see **Build** above). 3. **Provide the models to the page**, either: - - **from your device** — tap the *"Models from device"* picker in - `scan.html` and select `layout_heron_int8.onnx` and, for TableFormer, + - **from your device** — tap the model picker under *"Scanned PDFs and + images"* and select `layout_heron_int8.onnx` and, for TableFormer, `encoder.onnx`, `decoder_kv.onnx` (+`.data`), `bbox.onnx` (+`.data`). Each file is read to an `ArrayBuffer` on the client and used directly — no download, and a single allocation per file (a 225 MB encoder over diff --git a/crates/docling-wasm/src/digital.rs b/crates/docling-wasm/src/digital.rs new file mode 100644 index 00000000..a472e09e --- /dev/null +++ b/crates/docling-wasm/src/digital.rs @@ -0,0 +1,207 @@ +//! Browser pipeline for **digital** PDFs — the ones that carry a text layer +//! (#157). It is the scanned pipeline with the expensive half removed: the text +//! comes from the PDF itself rather than from OCR, so only the layout model +//! runs, and the result is both faster and exact (no recognition errors, no +//! mangled umlauts). +//! +//! This is what the native pipeline does for a digital PDF — it OCRs a page +//! only when `page.cells` is empty — so the same three steps happen here in the +//! same order: layout detection, region refinement against the real text cells, +//! then TableFormer (or the geometric fallback) per table region. +//! +//! Without it the browser had to choose between structure and fidelity: the +//! pure text-layer path (`convert`) is milliseconds but emits flat paragraphs, +//! because headings, tables and pictures are all things the layout model finds. +//! +//! ```js +//! const conv = new DigitalConverter(pdfBytes); // throws when there is no text layer +//! for (let i = 0; i < conv.page_count(); i++) { +//! // rasterize page i with pdf.js at 2 px/point, then: +//! await conv.add_page(i, rgba, w, h, 2.0, layout); +//! } +//! const markdown = conv.finish("bill.pdf", "md", "embedded"); +//! ``` + +use docling_pdf::assemble::{geometric_table_is_reliable, reconstruct_table}; +use docling_pdf::layout::{decode_layout, layout_input}; +use docling_pdf::pdfium_backend::PdfPage; +use docling_pdf::scanned::{ + assemble_page_with_tables, drop_duplicate_text_claims, finish_document, refine_regions, +}; +use image::RgbImage; +use wasm_bindgen::prelude::*; + +use crate::ocr::tensor_parts; +use crate::scanned::{render, LayoutSession}; +use crate::tableformer::TfSession; + +/// A digital PDF being converted page by page. Construct it from the file's +/// bytes (the text layer is parsed once, in Rust), then feed the rasterized +/// pages in order. +#[wasm_bindgen] +pub struct DigitalConverter { + /// One entry per page, carrying that page's text cells in PDF points. + pages: Vec, + out: Vec, +} + +#[wasm_bindgen] +impl DigitalConverter { + /// Parse the PDF's text layer. Fails when there is none — the caller should + /// fall back to the scanned pipeline, exactly as the demo page does. + #[wasm_bindgen(constructor)] + pub fn new(bytes: &[u8]) -> Result { + let pages = docling_pdf::textparse::pdf_text_pages(bytes); + if pages.iter().all(|p| p.cells.is_empty()) { + return Err(JsError::new( + "PDF has no embedded text layer (scanned/image-only?) — use the OCR pipeline", + )); + } + Ok(Self { + pages, + out: Vec::new(), + }) + } + + /// Pages the text parser found. + pub fn page_count(&self) -> usize { + self.pages.len() + } + + /// Convert page `index` (0-based) given its rendered bitmap: layout + /// detection plus geometric tables. `scale` is the raster's pixels per PDF + /// point (2.0 for pdf.js `{scale: 2}`). + pub async fn add_page( + &mut self, + index: usize, + rgba: &[u8], + px_w: u32, + px_h: u32, + scale: f32, + layout: &LayoutSession, + ) -> Result<(), JsError> { + self.add_page_impl(index, rgba, px_w, px_h, scale, layout, None) + .await + } + + /// [`add_page`](Self::add_page) with TableFormer for the table regions whose + /// geometric reconstruction looks unreliable. + #[wasm_bindgen(js_name = addPageTf)] + #[allow(clippy::too_many_arguments)] + pub async fn add_page_tf( + &mut self, + index: usize, + rgba: &[u8], + px_w: u32, + px_h: u32, + scale: f32, + layout: &LayoutSession, + tf: &TfSession, + ) -> Result<(), JsError> { + self.add_page_impl(index, rgba, px_w, px_h, scale, layout, Some(tf)) + .await + } + + /// Assemble the converted pages into a document and render it as `"md"` + /// (default), `"json"` or `"doclang"`, with `images` picking how pictures + /// render in Markdown. Resets the converter. + pub fn finish( + &mut self, + name: &str, + to: Option, + images: Option, + ) -> Result { + let doc = finish_document(name, std::mem::take(&mut self.out)); + render(&doc, to.as_deref(), images.as_deref()) + } +} + +impl DigitalConverter { + #[allow(clippy::too_many_arguments)] + async fn add_page_impl( + &mut self, + index: usize, + rgba: &[u8], + px_w: u32, + px_h: u32, + scale: f32, + layout: &LayoutSession, + tf: Option<&TfSession>, + ) -> Result<(), JsError> { + if rgba.len() != (px_w as usize) * (px_h as usize) * 4 { + return Err(JsError::new("rgba buffer size does not match dimensions")); + } + let mut page = self + .pages + .get(index) + .cloned() + .ok_or_else(|| JsError::new("page index is out of range"))?; + + let mut img = RgbImage::new(px_w, px_h); + for (i, px) in img.pixels_mut().enumerate() { + px.0 = [rgba[i * 4], rgba[i * 4 + 1], rgba[i * 4 + 2]]; + } + + // Layout: Rust preprocessing → JS inference → Rust decoding. The page + // geometry comes from the text parser, not the bitmap, so a raster at + // any scale lines up with the cells (which are in PDF points). + let (page_w, page_h) = (page.width, page.height); + let input = layout_input(&img); + let out = layout + .run(js_sys::Float32Array::from(input.as_slice())) + .await + .map_err(|e| JsError::new(&format!("layout session.run: {e:?}")))?; + let get = |k: &str| { + js_sys::Reflect::get(&out, &JsValue::from_str(k)) + .map_err(|_| JsError::new(&format!("layout result has no `{k}`"))) + }; + let (logits, q, c) = tensor_parts(&get("logits")?)?; + let (boxes, bq, four) = tensor_parts(&get("boxes")?)?; + if bq != q || four != 4 { + return Err(JsError::new("layout boxes dims must be [1, q, 4]")); + } + let regions = decode_layout(&logits, &boxes, q, c, page_w, page_h); + // Refine against the *real* text cells: unlike the OCR path, orphan-text + // recovery and the false-picture drop have something to work with here. + let regions = refine_regions(regions, &page.cells, page_w, page_h); + // The pdf.js raster can tease overlapping text detections out of the + // model (a block plus its own parts), and every overlap reads the same + // cells twice — drop the re-readers. + let regions = drop_duplicate_text_claims(regions, &page.cells); + + // Attach the raster so picture regions crop out of it, and record what + // it is scaled by (cells stay in points). + page.image = img; + page.scale = scale; + + let mut table_rows = Vec::with_capacity(regions.len()); + for r in ®ions { + if r.label != "table" { + table_rows.push(None); + continue; + } + // Same bargain as the scanned path: TableFormer costs seconds per + // region, so skip it wherever the free geometric reconstruction is + // already dense and well-formed. + let geometric = reconstruct_table(r, &page.cells); + match tf { + Some(tf) if !geometric_table_is_reliable(&geometric) => { + table_rows.push( + crate::tableformer::predict_table_rows( + tf, + &page.image, + [r.l, r.t, r.r, r.b], + &page.word_cells, + ) + .await, + ); + } + _ => table_rows.push(None), + } + } + + self.out + .push(assemble_page_with_tables(&page, regions, table_rows)); + Ok(()) + } +} diff --git a/crates/docling-wasm/src/lib.rs b/crates/docling-wasm/src/lib.rs index c80f37fc..71bb9a4a 100644 --- a/crates/docling-wasm/src/lib.rs +++ b/crates/docling-wasm/src/lib.rs @@ -16,9 +16,11 @@ //! const md = convert(new Uint8Array(await file.arrayBuffer()), file.name, "md"); //! ``` -use docling::{DocumentConverter, InputFormat, SourceDocument}; +use docling::{DocumentConverter, ImageMode, InputFormat, SourceDocument}; use wasm_bindgen::prelude::*; +#[cfg(feature = "ocr")] +mod digital; #[cfg(feature = "ocr")] mod ocr; #[cfg(feature = "ocr")] @@ -26,6 +28,8 @@ mod scanned; #[cfg(feature = "ocr")] mod tableformer; #[cfg(feature = "ocr")] +pub use digital::DigitalConverter; +#[cfg(feature = "ocr")] pub use ocr::ocr_image; #[cfg(feature = "ocr")] pub use scanned::{convert_scanned_image, ScannedConverter}; @@ -39,7 +43,12 @@ fn start() { /// The whole conversion body, host-testable (`JsError` can only be /// constructed on the wasm target, so the JS boundary stays a thin shim). -fn convert_impl(bytes: &[u8], filename: &str, to: Option<&str>) -> Result { +fn convert_impl( + bytes: &[u8], + filename: &str, + to: Option<&str>, + images: Option<&str>, +) -> Result { let ext = filename.rsplit('.').next().unwrap_or_default(); let format = InputFormat::from_extension(ext) .ok_or_else(|| format!("unknown or unsupported extension: {filename:?}"))?; @@ -47,8 +56,15 @@ fn convert_impl(bytes: &[u8], filename: &str, to: Option<&str>) -> Result Ok(result.document.export_to_markdown()), + // `Referenced` is deliberately unreachable here: it hands the caller + // loose image files to write next to the Markdown, which a page with no + // filesystem cannot do — the browser equivalent is `embedded`. + "md" | "markdown" => Ok(result + .document + .export_to_markdown_with_images(image_mode, "artifacts") + .0), "json" => Ok(result.document.export_to_json()), "doclang" => Ok(result.document.export_to_doclang()), other => Err(format!( @@ -57,13 +73,36 @@ fn convert_impl(bytes: &[u8], filename: &str, to: Option<&str>) -> Result`) or `embedded` +/// (`![Image](data:…;base64,…)`, self-contained — the only way to carry pixels +/// out of a page that cannot write files). +fn image_mode(images: Option<&str>) -> Result { + match images.unwrap_or("placeholder") { + "placeholder" => Ok(ImageMode::Placeholder), + "embedded" => Ok(ImageMode::Embedded), + other => Err(format!( + "unknown images={other:?} (expected \"placeholder\" or \"embedded\")" + )), + } +} + /// Convert a document (as bytes + filename, the extension drives format /// detection) to `to`: `"md"` (Markdown, default), `"json"` (docling-core's /// `DoclingDocument` wire format, schema 1.10.0) or `"doclang"` (docling's /// DocLang XML serialization). +/// +/// `images` controls how pictures render in Markdown — `"placeholder"` +/// (default) or `"embedded"` (base64 data URIs), the same option +/// docling-serve exposes. #[wasm_bindgen] -pub fn convert(bytes: &[u8], filename: &str, to: Option) -> Result { - convert_impl(bytes, filename, to.as_deref()).map_err(|e| JsError::new(&e)) +pub fn convert( + bytes: &[u8], + filename: &str, + to: Option, + images: Option, +) -> Result { + convert_impl(bytes, filename, to.as_deref(), images.as_deref()).map_err(|e| JsError::new(&e)) } /// The file extensions this build can convert, as a JSON string array — @@ -98,16 +137,16 @@ mod tests { #[test] fn markdown_roundtrip() { let md = b"# Title\n\nHello *world*\n"; - let out = convert_impl(md, "note.md", None).unwrap(); + let out = convert_impl(md, "note.md", None, None).unwrap(); assert!(out.contains("# Title")); - let json = convert_impl(md, "note.md", Some("json")).unwrap(); + let json = convert_impl(md, "note.md", Some("json"), None).unwrap(); assert!(json.contains("\"schema_name\"")); } #[test] fn ml_formats_rejected() { // Images still need the full ML pipeline. - let err = convert_impl(&[0x89, b'P', b'N', b'G'], "scan.png", None).unwrap_err(); + let err = convert_impl(&[0x89, b'P', b'N', b'G'], "scan.png", None, None).unwrap_err(); assert!( err.contains("unknown or unsupported") || err.contains("pdf"), "should reject the ML-only format: {err}" @@ -129,7 +168,7 @@ mod tests { "/../../tests/data/pdf/sources/code_and_formula.pdf" )) .expect("corpus pdf"); - let out = convert_impl(&bytes, "code_and_formula.pdf", None).unwrap(); + let out = convert_impl(&bytes, "code_and_formula.pdf", None, None).unwrap(); assert!(!out.trim().is_empty(), "text layer should extract"); } @@ -141,7 +180,7 @@ mod tests { if docling::PDF_ML_COMPILED { return; } - let err = convert_impl(b"%PDF-1.4\n%%EOF", "scan.pdf", None).unwrap_err(); + let err = convert_impl(b"%PDF-1.4\n%%EOF", "scan.pdf", None, None).unwrap_err(); assert!( err.contains("text layer") || err.contains("OCR"), "should point at the missing text layer: {err}" @@ -156,10 +195,29 @@ mod tests { "/../docling/tests/data/docx/sources/docx_lists.docx" )) .expect("corpus docx"); - let out = convert_impl(&bytes, "docx_lists.docx", None).unwrap(); + let out = convert_impl(&bytes, "docx_lists.docx", None, None).unwrap(); assert!(!out.trim().is_empty()); } + /// `images=embedded` inlines picture bytes as data URIs (docling-serve's + /// option, and the only way a page with no filesystem can carry pixels + /// out); the default stays docling's `` placeholder. + #[test] + fn embedded_images_inline_as_data_uris() { + let bytes = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../docling/tests/data/docx/sources/word_image_anchors.docx" + )) + .expect("corpus docx with images"); + let placeholder = convert_impl(&bytes, "word_image_anchors.docx", None, None).unwrap(); + assert!(placeholder.contains(""), "{placeholder}"); + let embedded = + convert_impl(&bytes, "word_image_anchors.docx", None, Some("embedded")).unwrap(); + assert!(embedded.contains("](data:image/"), "expected a data URI"); + let err = convert_impl(&bytes, "word_image_anchors.docx", None, Some("nope")).unwrap_err(); + assert!(err.contains("unknown images="), "{err}"); + } + #[test] fn extensions_json_parses() { let v: Vec = serde_json::from_str(&supported_extensions()).unwrap(); diff --git a/crates/docling-wasm/src/ocr.rs b/crates/docling-wasm/src/ocr.rs index 3b0d030e..234d4803 100644 --- a/crates/docling-wasm/src/ocr.rs +++ b/crates/docling-wasm/src/ocr.rs @@ -27,7 +27,7 @@ //! }, //! }); //! ``` -//! (see `www/ocr.html` for the complete wiring, including output-name +//! (see `www/index.html` for the complete wiring, including output-name //! discovery and model/dict caching.) use docling_core::{DoclingDocument, Node}; diff --git a/crates/docling-wasm/src/scanned.rs b/crates/docling-wasm/src/scanned.rs index 6e322737..e743eb5e 100644 --- a/crates/docling-wasm/src/scanned.rs +++ b/crates/docling-wasm/src/scanned.rs @@ -4,7 +4,9 @@ //! reconstruction and reading-order assembly in Rust — the same code as the //! native pipeline (`docling_pdf::{layout, ocr_prep, scanned}`). TableFormer //! and the enrichment models are stage 3; table regions fall back to the -//! geometric reconstruction the native `--no-table-former` flag uses. +//! geometric reconstruction the native `--no-table-former` flag uses. Picture +//! regions are cropped out of the page bitmap, like the native pipeline, so +//! `images = "embedded"` inlines real figure bytes. //! //! Pages arrive as raw RGBA bitmaps: for a scanned PDF the host page renders //! them with pdf.js (`page.getViewport({scale: 2})` — 2 px per PDF point, @@ -16,10 +18,11 @@ //! for (const bitmap of pages) { //! await conv.add_page(bitmap.data, bitmap.width, bitmap.height, 2.0, layout, rec); //! } -//! const markdown = conv.finish("scan.pdf", "md"); +//! const markdown = conv.finish("scan.pdf", "md", "embedded"); //! ``` -//! (`www/scan.html` is the complete wiring.) +//! (`www/index.html` is the complete wiring.) +use docling_pdf::assemble::{geometric_table_is_reliable, reconstruct_table}; use docling_pdf::layout::{decode_layout, layout_input, SIDE}; use docling_pdf::ocr_prep::{ batch_input, decode_row, dict_chars, normalize_polarity, prep_region_lines, prep_table_words, @@ -33,14 +36,6 @@ use wasm_bindgen::prelude::*; use crate::ocr::{tensor_parts, RecSession}; use crate::tableformer::TfSession; -#[wasm_bindgen] -extern "C" { - // On-page diagnostic sink (defined by the host — worker.js forwards it to - // the main thread, which shows it; console isn't reachable on a phone). - #[wasm_bindgen(js_name = __docling_diag)] - fn diag(s: &str); -} - #[wasm_bindgen] extern "C" { /// The JS-side layout session: a wrapper around an `ort.InferenceSession` @@ -188,19 +183,6 @@ impl ScannedConverter { let regions = decode_layout(&logits, &boxes, q, c, page_w, page_h); let regions = refine_regions(regions, &[], page_w, page_h); - // Diagnostic: the region-label histogram, so it's visible (browser - // console) whether the layout model flagged any `table` region on this - // page — tables only render when it does (geometric or TableFormer). - { - let mut hist: std::collections::BTreeMap<&str, usize> = - std::collections::BTreeMap::new(); - for r in ®ions { - *hist.entry(r.label).or_default() += 1; - } - let summary: Vec = hist.iter().map(|(l, n)| format!("{l}×{n}")).collect(); - diag(&format!("layout regions: {}", summary.join(", "))); - } - // OCR the text regions (same gather/batch/decode as native ocr_page). let (bboxes, lines) = prep_region_lines(&img, ®ions, scale); let texts = self.ocr_lines(rec, &lines).await?; @@ -234,26 +216,34 @@ impl ScannedConverter { let table_rows = if let Some(tf) = tf { let mut rows = Vec::with_capacity(regions.len()); for r in ®ions { - if r.label == "table" { - rows.push( - crate::tableformer::predict_table_rows( - tf, - &img, - [r.l, r.t, r.r, r.b], - &cells, - ) - .await, - ); - } else { + if r.label != "table" { rows.push(None); + continue; } + // TableFormer costs seconds per region (the fp32 encoder runs + // once per table), so spend it only where it buys something: + // when the free geometric reconstruction already yields a dense, + // well-formed grid it is what TableFormer would agree with, and + // `None` tells assemble to keep it. + let geometric = reconstruct_table(r, &cells); + if geometric_table_is_reliable(&geometric) { + rows.push(None); + continue; + } + rows.push( + crate::tableformer::predict_table_rows(tf, &img, [r.l, r.t, r.r, r.b], &cells) + .await, + ); } rows } else { Vec::new() // assemble_page_with_tables resizes to all-None }; - let page = PdfPage::from_cells(page_w, page_h, scale, cells); + // Hand the page bitmap over: assemble crops each `picture` region out + // of it, so the browser pipeline produces the same figure bytes the + // native one does (`images=embedded` then inlines them). + let page = PdfPage::from_cells_with_image(page_w, page_h, scale, cells, img); self.pages .push(assemble_page_with_tables(&page, regions, table_rows)); Ok(()) @@ -268,16 +258,39 @@ impl ScannedConverter { } /// Assemble the accumulated pages into the final document and render it - /// as `"md"` (default) or `"json"`. Resets the converter. - pub fn finish(&mut self, name: &str, to: Option) -> Result { + /// as `"md"` (default), `"json"` or `"doclang"` — the same three the + /// declarative [`crate::convert`] entry point offers. `images` picks how + /// cropped figures render in Markdown (`"placeholder"` | `"embedded"`), + /// like [`crate::convert`]. Resets the converter. + pub fn finish( + &mut self, + name: &str, + to: Option, + images: Option, + ) -> Result { let doc = finish_document(name, std::mem::take(&mut self.pages)); - match to.as_deref().unwrap_or("md") { - "md" | "markdown" => Ok(doc.export_to_markdown()), - "json" => Ok(doc.export_to_json()), - other => Err(JsError::new(&format!( - "unknown output format {other:?} (expected \"md\" or \"json\")" - ))), + render(&doc, to.as_deref(), images.as_deref()) + } +} + +/// Render an assembled document in one of the three output grammars, with the +/// same `images` choice the declarative path offers — picture regions are +/// cropped out of the rendered page, so `embedded` has real bytes to inline. +pub(crate) fn render( + doc: &docling_core::DoclingDocument, + to: Option<&str>, + images: Option<&str>, +) -> Result { + match to.unwrap_or("md") { + "md" | "markdown" => { + let mode = crate::image_mode(images).map_err(|e| JsError::new(&e))?; + Ok(doc.export_to_markdown_with_images(mode, "artifacts").0) } + "json" => Ok(doc.export_to_json()), + "doclang" => Ok(doc.export_to_doclang()), + other => Err(JsError::new(&format!( + "unknown output format {other:?} (expected \"md\", \"json\" or \"doclang\")" + ))), } } @@ -292,6 +305,7 @@ pub async fn convert_scanned_image( layout: &LayoutSession, rec: &RecSession, to: Option, + images: Option, ) -> Result { let img = image::load_from_memory(bytes) .map_err(|e| JsError::new(&format!("decode image: {e}")))? @@ -299,7 +313,7 @@ pub async fn convert_scanned_image( let (w, h) = img.dimensions(); let mut conv = ScannedConverter::new(dict); conv.add_page(img.as_raw(), w, h, 1.0, layout, rec).await?; - conv.finish(name, to) + conv.finish(name, to, images) } // Silence the unused warning for SIDE re-export path (the JS side sizes its diff --git a/crates/docling-wasm/src/tableformer.rs b/crates/docling-wasm/src/tableformer.rs index 4b23862d..bd65ca42 100644 --- a/crates/docling-wasm/src/tableformer.rs +++ b/crates/docling-wasm/src/tableformer.rs @@ -9,7 +9,7 @@ //! encoder's constant cross-attention K/V and `enc_out`, and the growing //! decoder KV-cache, entirely on the JS side. Each decode step sends only the //! last tag (one int) and gets back `logits` + `hidden` (525 floats). See -//! `www/scan.html` for the JS wiring (the `decoder_kv` graph: N_LAYERS=6, +//! `www/index.html` for the JS wiring (the `decoder_kv` graph: N_LAYERS=6, //! KV_HEADS=8, head_dim=64, cross length 784). use docling_pdf::pdfium_backend::TextCell; diff --git a/crates/docling-wasm/www/index.html b/crates/docling-wasm/www/index.html index 169a9b6c..1e95168e 100644 --- a/crates/docling-wasm/www/index.html +++ b/crates/docling-wasm/www/index.html @@ -3,92 +3,560 @@ - docling.rs — in-browser document conversion (wasm) + + + docling.rs — document conversion in your browser (wasm) + + -

docling.rs → wasm: convert documents in your browser

-

- DOCX / HTML / Markdown / XLSX / PPTX / CSV / EPUB / ODF / LaTeX / - digital PDF / … are converted to Markdown or docling JSON - entirely client-side — the file never leaves this page. - PDFs convert via their embedded text layer (flat paragraphs, like the - native --no-ocr); scanned PDFs, images and audio need the - native ML pipeline and are not part of the wasm build. +

docling.rs in your browser

+

+ Documents convert to Markdown, docling JSON or DocLang XML + entirely client-side — nothing is uploaded, no server is + involved. Everything below runs from this one page.

+ +
Drop a file here, or click to choose
+ +
-
-
Drop a document here, or click to pick a file
- + +
+ + + scanned pages need models — see below +
+ + 

 
+  
What converts without any setup

+  

+    DOCX, PPTX, XLSX, HTML, Markdown, CSV, AsciiDoc, EPUB, ODF, LaTeX, JATS,
+    email, MHTML, JSON, DocLang — and PDFs that carry a text layer
+    (the same extraction the native --no-ocr flag does). These need
+    no models and no network: the converters are compiled into the wasm module.
+  

+
+  
Scanned PDFs and images (optional models)

+  

+    A scanned page has no text layer, so it needs the ML pipeline: RT-DETR
+    layout detection + PP-OCR recognition (+ TableFormer for tables), running on
+    ONNX Runtime Web
+    with all pre/post-processing in Rust/wasm — the same code as the native
+    pipeline. Provide the models any one of these three ways:
+  

+  
    
+    
  1. 
+      From your device — pick the .onnx files:
+      
+      
    Layout needs layout_heron_int8.onnx; TableFormer also needs
+      encoder.onnx, decoder_kv.onnx + .data,
+      bbox.onnx + .data. Each file is read
+      locally — no download, and it avoids the memory spike a 225 MB fetch
+      causes on a phone.
+    
    2. 
+    
  2. 
+      From a models/ folder next to this page —
+      served same-origin, fetched automatically:
+      
    cd crates/docling-wasm/www/
+curl -fsSL https://raw.githubusercontent.com/docling-project/docling.rs/master/scripts/install/download_dependencies.sh | sh
    
+    
    3. 
+    
  3. 
+      From Hugging Face — used automatically when neither of the
+      above has the file. (GitHub Release assets can't be fetched cross-origin:
+      they send no Access-Control-Allow-Origin.)
+    
    4. 
+  

+  

+    Resolution order per file is device → local models/ →
+    Hugging Face, so any mix works. The recognition model (~10 MB)
+    streams from Hugging Face on demand; layout is ~68 MB and TableFormer
+    ~380 MB, so on mobile prefer picking files from the device.
+  

+
   
 
diff --git a/crates/docling-wasm/www/ocr.html b/crates/docling-wasm/www/ocr.html
deleted file mode 100644
index 7716c8cd..00000000
--- a/crates/docling-wasm/www/ocr.html
+++ /dev/null
@@ -1,219 +0,0 @@
-
-
-
-  
-  
-  
-  
-  docling.rs — in-browser OCR (wasm + ONNX Runtime Web)
-  
-
-
-  
docling.rs → wasm: OCR a scanned image in your browser

-  

-    Stage 1 of #157:
-    the PP-OCRv3 recognition model (~10 MB) runs via
-    ONNX Runtime Web;
-    line segmentation and CTC decoding run in Rust/wasm — the same code as the
-    native pipeline. The image never leaves this page. The model + dictionary
-    are fetched once and cached by the browser. Single-column scans work best
-    (layout detection is stage 2).
-  

-  
drop a scanned image here (PNG/JPEG/TIFF/WebP) or click to pick

-  
-  

-    Recognition language:
-    
-    
-  

-
-  

-
-  
-
-
diff --git a/crates/docling-wasm/www/pipeline.js b/crates/docling-wasm/www/pipeline.js
index c6d54015..bf15940c 100644
--- a/crates/docling-wasm/www/pipeline.js
+++ b/crates/docling-wasm/www/pipeline.js
@@ -8,7 +8,7 @@
 // DOM-free: the only host concern is onStatus(msg, spinning) for progress.
 
 import * as ort from "https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.mjs";
-import init, { ScannedConverter, convert_scanned_image } from "./pkg/docling_wasm.js";
+import init, { DigitalConverter, ScannedConverter, convert_scanned_image } from "./pkg/docling_wasm.js";
 
 // Multi-threaded wasm when cross-origin isolated (coi.js); else one thread.
 ort.env.wasm.numThreads = self.crossOriginIsolated
@@ -100,7 +100,7 @@ export function createOcr({ onStatus }) {
   const status = (msg, spinning = true) => onStatus && onStatus(msg, spinning);
 
   // Model files the user picked from the device (basename → ArrayBuffer),
-  // used instead of any network fetch — see scan.html's file picker. Reading a
+  // used instead of any network fetch — see index.html's model picker. Reading a
   // File to an ArrayBuffer is a single allocation, so it also sidesteps the
   // double-buffering peak fetchProgress hits on the 225 MB encoder.
   let provided = {};
@@ -262,27 +262,51 @@ export function createOcr({ onStatus }) {
     const tfSess = useTf ? await ensureTf() : null;
     cur = { conv: new ScannedConverter(dict), rec, tf: tfSess };
   }
-  async function addPage(rgba, w, h, scale) {
+
+  // A digital PDF (one with a text layer) needs the layout model but no
+  // recognition: the text comes out of the file, so this is both much faster
+  // and exact. Throws when there is no text layer — the host then falls back to
+  // startDoc. Returns the page count, since the parser already knows it.
+  async function startDigital(bytes, useTf) {
+    const conv = new DigitalConverter(new Uint8Array(bytes));
+    const tfSess = useTf ? await ensureTf() : null;
+    cur = { conv, tf: tfSess, digital: true };
+    return conv.page_count();
+  }
+  async function addPage(rgba, w, h, scale, index) {
+    // Guard the single in-flight document: without it a stray addPage/finish
+    // after (or before) the lifecycle reads `cur` as null and the host sees an
+    // opaque "Cannot read properties of null".
+    if (!cur) throw new Error("addPage called with no document open (startDoc first)");
+    if (cur.digital) {
+      const args = [index, rgba, w, h, scale, layout];
+      return cur.tf
+        ? cur.conv.addPageTf(...args, cur.tf)
+        : cur.conv.add_page(...args);
+    }
     if (cur.tf) {
       await cur.conv.addPageTf(rgba, w, h, scale, layout, cur.rec, cur.tf);
     } else {
       await cur.conv.add_page(rgba, w, h, scale, layout, cur.rec);
     }
   }
-  function finishDoc(name) {
-    const md = cur.conv.finish(name, "md");
+  function finishDoc(name, to, images) {
+    if (!cur) throw new Error("finishDoc called with no document open (startDoc first)");
+    const md = cur.conv.finish(name, to || "md", images || "placeholder");
     cur = null;
     return md;
   }
 
   // Standalone image: the wasm side decodes it (no canvas needed).
-  async function convertImage(bytes, name, lang) {
+  async function convertImage(bytes, name, lang, to, images) {
     const { dict, rec } = await recFor(lang);
-    return convert_scanned_image(new Uint8Array(bytes), name, dict, layout, rec, "md");
+    return convert_scanned_image(
+      new Uint8Array(bytes), name, dict, layout, rec, to || "md", images || "placeholder",
+    );
   }
 
   return {
-    boot, warmup, recFor, startDoc, addPage, finishDoc, convertImage, setProvidedModels,
+    boot, warmup, recFor, startDoc, startDigital, addPage, finishDoc, convertImage, setProvidedModels,
     get layoutKind() { return layoutKind; },
   };
 }
diff --git a/crates/docling-wasm/www/scan.html b/crates/docling-wasm/www/scan.html
deleted file mode 100644
index 48623064..00000000
--- a/crates/docling-wasm/www/scan.html
+++ /dev/null
@@ -1,347 +0,0 @@
-
-
-
-  
-  
-  
-  
-  docling.rs — scanned PDFs in the browser (lite profile)
-  
-
-
-  
docling.rs → wasm: scanned PDF / image, full lite profile

-  

-    Stage 2 of #157:
-    RT-DETR layout detection + PP-OCRv3 recognition via ONNX Runtime Web;
-    pdf.js rasterizes PDF pages at the native pipeline's 2 px/point;
-    region refinement, OCR and reading-order assembly run in Rust/wasm — the
-    same code as the native pipeline. Tables come out via the geometric
-    reconstruction (TableFormer is stage 3). Nothing leaves this page.
-  

-  

-    The layout model is served from this page's own origin — fetch it once into ./models/
-    

-      Unlike the recognition model (fetched from Hugging Face, which sends CORS
-      headers), the RT-DETR layout export lives on this repo's GitHub Release,
-      and GitHub Release assets can't be fetched cross-origin from a
-      browser (no Access-Control-Allow-Origin). So this
-      page loads the layout model same-origin, from a
-      models/ directory next to it. Run the installer from
-      this www/ directory — it writes ./models/
-      relative to the current directory:
-    

-    
cd crates/docling-wasm/www/
-curl -fsSL https://raw.githubusercontent.com/docling-project/docling.rs/master/scripts/install/download_dependencies.sh | sh

-    

-      That populates ./models/ (this page reads
-      ./models/layout_heron_int8.onnx, ~68 MB, falling back to
-      ./models/layout_heron.onnx, ~172 MB — int8 needs a
-      release that hosts it, otherwise fp32 is fetched). These are the exact
-      conformance-validated models the native pipeline uses; both decode
-      identically here (same pixel_values → logits, pred_boxes
-      contract). The recognition model still streams from Hugging Face on demand.
-    

-  

-  
drop a scanned PDF or image here, or click to pick

-  
-  

-    Recognition language:
-    
-    
-    
-  

-  

-    Models from device (optional — skips the download):
-    
-    
pick layout_heron_int8.onnx and, for TableFormer, encoder.onnx,
-    decoder_kv.onnx(+.data), bbox.onnx(+.data).
-  

-
-  

-  

-
-  
-
-
diff --git a/crates/docling-wasm/www/worker.js b/crates/docling-wasm/www/worker.js
index f4dffe9b..96ab9b31 100644
--- a/crates/docling-wasm/www/worker.js
+++ b/crates/docling-wasm/www/worker.js
@@ -7,17 +7,15 @@
 //
 // RPC: every request carries an id; the reply is {type:"ok", id, ...data} or
 // {type:"error", id, msg}. Progress is a broadcast {type:"status", msg,
-// spinning}. Requests: boot{lang} | rec{lang} | doc-start{lang} |
-// doc-page{rgba,w,h,scale} | doc-finish{name} | convert-image{bytes,name,lang}.
+// spinning}. Requests: set-models{models} | boot{lang,layoutOnly} | rec{lang} |
+// doc-start{lang,useTf} | doc-start-digital{bytes,useTf} |
+// doc-page{rgba,w,h,scale,index} | doc-finish{name,to,images} |
+// convert-image{bytes,name,lang,to,images}.
 
 import { createOcr, THREADS } from "./pipeline.js";
 
 const post = (type, extra) => self.postMessage({ type, ...extra });
 
-// On-page diagnostic sink the wasm calls (see src/scanned.rs) — forward to the
-// main thread, which renders it (a phone can't open the console).
-globalThis.__docling_diag = (msg) => post("diag", { msg });
-
 const ocr = createOcr({
   onStatus: (msg, spinning) => post("status", { msg, spinning }),
 });
@@ -30,8 +28,12 @@ async function handle(m) {
     case "boot": {
       const kind = await ocr.boot();
       if (!kind) return { noLayout: true };
-      await ocr.recFor(m.lang);
-      await ocr.warmup(m.lang);
+      // A digital PDF never recognises anything, so its boot skips the
+      // recognition model entirely; the scanned path loads it lazily anyway.
+      if (!m.layoutOnly) {
+        await ocr.recFor(m.lang);
+        await ocr.warmup(m.lang);
+      }
       return { kind, threads: THREADS };
     }
     case "rec":
@@ -41,13 +43,15 @@ async function handle(m) {
     case "doc-start":
       await ocr.startDoc(m.lang, m.useTf);
       return {};
+    case "doc-start-digital":
+      return { pages: await ocr.startDigital(m.bytes, m.useTf) };
     case "doc-page":
-      await ocr.addPage(new Uint8Array(m.rgba), m.w, m.h, m.scale);
+      await ocr.addPage(new Uint8Array(m.rgba), m.w, m.h, m.scale, m.index);
       return {};
     case "doc-finish":
-      return { md: ocr.finishDoc(m.name) };
+      return { md: ocr.finishDoc(m.name, m.to, m.images) };
     case "convert-image":
-      return { md: await ocr.convertImage(m.bytes, m.name, m.lang) };
+      return { md: await ocr.convertImage(m.bytes, m.name, m.lang, m.to, m.images) };
     default:
       throw new Error(`unknown request ${m.type}`);
   }