From 5a837fca8e676249ffd71cb80f1554abcf0f5a7a Mon Sep 17 00:00:00 2001 From: artiz Date: Fri, 24 Jul 2026 10:16:58 +0000 Subject: [PATCH 01/40] wasm: in-browser OCR via ONNX Runtime Web interop (#157 stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of the browser ML pipeline: scanned images OCR fully client-side. docling-pdf grows an `ocr-prep` feature (pure Rust + the image crate, no ort/pdfium): the ONNX-free half of the PP-OCRv3 recognition pipeline — line segmentation, crop prep, deterministic width-batching, CTC decode, dictionary handling — moved verbatim from ocr.rs into a public ocr_prep module. ocr.rs now consumes it, so there is exactly ONE implementation of the pre/post-processing; the native path is byte-identical after the refactor (verified: the scanned-tiff conversion diffs empty against the pre-refactor output, full docling-pdf suite green). Any wasm-vs-native drift can therefore only come from the runtime's kernels — the #157 conformance story depends on that property. docling-wasm (default `ocr` feature): `ocr_image(bytes, dict, session, to)` — decode the image in Rust, segment/prep/batch via ocr_prep, delegate each width-batch to a JS `RecSession` wrapper around an ort-web InferenceSession (`run(n, h, w, data) -> {data, dims}`), CTC- decode, and export a DoclingDocument as Markdown or docling JSON. No layout model yet (stage 2): the whole image is one text region split by the ink-projection profile — right for single-column scans, degraded on complex layouts, and said so in the docs. www/ocr.html: complete drop-an-image demo — ort-web from CDN, the ~10 MB en_PP-OCRv3 model + dictionary fetched from their public hosting and browser-cached, timing readout. Compiles for wasm32-unknown-unknown and the host (rlib tests); clippy clean; the existing wasm CI check is unaffected. Refs #157 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- Cargo.lock | 5 + crates/docling-pdf/Cargo.toml | 4 + crates/docling-pdf/src/lib.rs | 2 + crates/docling-pdf/src/ocr.rs | 180 +++------------------ crates/docling-pdf/src/ocr_prep.rs | 247 +++++++++++++++++++++++++++++ crates/docling-wasm/Cargo.toml | 20 +++ crates/docling-wasm/README.md | 26 +++ crates/docling-wasm/src/lib.rs | 5 + crates/docling-wasm/src/ocr.rs | 134 ++++++++++++++++ crates/docling-wasm/www/ocr.html | 102 ++++++++++++ 10 files changed, 568 insertions(+), 157 deletions(-) create mode 100644 crates/docling-pdf/src/ocr_prep.rs create mode 100644 crates/docling-wasm/src/ocr.rs create mode 100644 crates/docling-wasm/www/ocr.html diff --git a/Cargo.lock b/Cargo.lock index 654b7a7b..e4645db1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1529,8 +1529,13 @@ version = "0.49.0" dependencies = [ "console_error_panic_hook", "docling", + "docling-core", + "docling-pdf", + "image", + "js-sys", "serde_json", "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] diff --git a/crates/docling-pdf/Cargo.toml b/crates/docling-pdf/Cargo.toml index 4a837aa2..8cde9c3f 100644 --- a/crates/docling-pdf/Cargo.toml +++ b/crates/docling-pdf/Cargo.toml @@ -18,7 +18,11 @@ default = ["ml"] # (`convert_text_layer` — lopdf content-stream parsing + the docling-parse # line sanitizer, the same extraction `--no-ocr` uses), which compiles for # wasm32-unknown-unknown. +# The ONNX-free OCR pre/post half (`ocr_prep`) alone — what the wasm build +# (#79 phase 2) pulls: pure Rust + the `image` crate, no ort/pdfium. +ocr-prep = ["dep:image"] ml = [ + "ocr-prep", "dep:pdfium-render", "dep:ort", "dep:image", diff --git a/crates/docling-pdf/src/lib.rs b/crates/docling-pdf/src/lib.rs index acaf23ef..776ea0bc 100644 --- a/crates/docling-pdf/src/lib.rs +++ b/crates/docling-pdf/src/lib.rs @@ -27,6 +27,8 @@ pub mod layout; mod mets; #[cfg(feature = "ml")] mod ocr; +#[cfg(feature = "ocr-prep")] +pub mod ocr_prep; pub mod pdfium_backend; mod reading_order; #[cfg(feature = "ml")] diff --git a/crates/docling-pdf/src/ocr.rs b/crates/docling-pdf/src/ocr.rs index 83b9058c..64e781a2 100644 --- a/crates/docling-pdf/src/ocr.rs +++ b/crates/docling-pdf/src/ocr.rs @@ -5,88 +5,25 @@ //! is recognised and decoded with CTC — producing [`TextCell`]s the normal //! layout assembly then consumes. This avoids a separate text-detection model. -use std::collections::BTreeMap; - -use image::{imageops, imageops::FilterType, Rgb, RgbImage}; +use image::{imageops, RgbImage}; use ort::session::Session; use ort::value::Tensor; use crate::layout::Region; +// The ONNX-free half (line prep, batching, CTC decode) lives in `ocr_prep` +// so the wasm build shares it verbatim (#79 phase 2). +use crate::ocr_prep::{ + batch_input, decode_row, dict_chars, prep_line, segment_lines, width_batches, PrepLine, + REC_HEIGHT, +}; use crate::pdfium_backend::TextCell; -const REC_HEIGHT: u32 = 48; - -/// Cap on lines per recognition run: bounds peak input-tensor memory -/// (16 × 3 × 48 × 2400 px ≈ 22 MB f32) without costing measurable batching -/// benefit — same-width groups are rarely larger. -const REC_BATCH: usize = 16; - -/// A text-line crop prepared for recognition: resized to the fixed model -/// height, normalised to `[-1, 1]`, laid out CHW. -struct PrepLine { - /// Width after the aspect-preserving resize to `REC_HEIGHT`. - w: usize, - /// `3 * REC_HEIGHT * w` values. - data: Vec, -} - -/// Prepare one line crop, or `None` for a degenerate (zero-sized) crop. -fn prep_line(line: &RgbImage) -> Option { - let (w, h) = line.dimensions(); - if w == 0 || h == 0 { - return None; - } - let new_w = ((w as f32) * REC_HEIGHT as f32 / h as f32) - .round() - .clamp(8.0, 2400.0) as u32; - let resized = imageops::resize(line, new_w, REC_HEIGHT, FilterType::Triangle); - let n = (REC_HEIGHT * new_w) as usize; - // Normalise to [-1, 1]: (x/255 - 0.5) / 0.5. - let mut data = vec![0f32; 3 * n]; - for (i, px) in resized.pixels().enumerate() { - data[i] = px[0] as f32 / 127.5 - 1.0; - data[n + i] = px[1] as f32 / 127.5 - 1.0; - data[2 * n + i] = px[2] as f32 / 127.5 - 1.0; - } - Some(PrepLine { - w: new_w as usize, - data, - }) -} - pub struct OcrModel { rec: Session, /// CTC classes: index 0 = blank, 1..=6623 = dictionary, 6624 = space. chars: Vec, } -/// Greedy CTC decode of one row's `(T, C)` probabilities. -fn decode_row(chars: &[String], probs: &[f32], nc: usize) -> String { - let mut out = String::new(); - let mut prev = 0usize; - for row in probs.chunks_exact(nc) { - let mut best = 0usize; - let mut bestv = row[0]; - for (c, &v) in row.iter().enumerate().skip(1) { - if v > bestv { - bestv = v; - best = c; - } - } - if best != prev && best != 0 { - if let Some(ch) = chars.get(best) { - out.push_str(ch); - } - } - prev = best; - } - out -} - -fn luma(p: &Rgb) -> f32 { - 0.299 * p[0] as f32 + 0.587 * p[1] as f32 + 0.114 * p[2] as f32 -} - /// Layout labels whose content is recognised as running text. fn is_text_label(label: &str) -> bool { matches!( @@ -194,10 +131,10 @@ impl OcrModel { .map_err(|e| format!("ocr: load {rec_path}: {e}"))?; let dict = std::fs::read_to_string(&dict_path) .map_err(|e| format!("ocr: read dict {dict_path}: {e}"))?; - let mut chars = vec![String::new()]; // blank at 0 - chars.extend(dict.lines().map(|s| s.to_string())); - chars.push(" ".to_string()); - Ok(Self { rec, chars }) + Ok(Self { + rec, + chars: dict_chars(&dict), + }) } /// Recognise a batch of prepared *same-width* lines in one session run. @@ -208,13 +145,14 @@ impl OcrModel { /// the scanned corpus), whereas width-padding leaks into the real /// timesteps through the model's global-attention blocks and measurably /// changes low-confidence characters. - fn recognize_batch(&mut self, w: usize, batch: &[&PrepLine]) -> Result, String> { - let n = batch.len(); - let hw = REC_HEIGHT as usize * w; - let mut data = vec![0f32; n * 3 * hw]; - for (i, pl) in batch.iter().enumerate() { - data[i * 3 * hw..(i + 1) * 3 * hw].copy_from_slice(&pl.data); - } + fn recognize_batch( + &mut self, + w: usize, + chunk: &[usize], + lines: &[PrepLine], + ) -> Result, String> { + let n = chunk.len(); + let data = batch_input(w, chunk, lines); let input = Tensor::from_array(([n, 3, REC_HEIGHT as usize, w], data)) .map_err(|e| format!("ocr: input tensor: {e}"))?; let outputs = self @@ -277,19 +215,11 @@ impl OcrModel { } } - // Group page-order line indices by exact width (BTreeMap: run order is - // deterministic) and recognise each group batched. - let mut by_width: BTreeMap> = BTreeMap::new(); - for (ix, pl) in lines.iter().enumerate() { - by_width.entry(pl.w).or_default().push(ix); - } + // Deterministic width-batching (shared with the wasm path). let mut texts = vec![String::new(); lines.len()]; - for (w, ixs) in by_width { - for chunk in ixs.chunks(REC_BATCH) { - let batch: Vec<&PrepLine> = chunk.iter().map(|&i| &lines[i]).collect(); - for (&i, text) in chunk.iter().zip(self.recognize_batch(w, &batch)?) { - texts[i] = text; - } + for (w, chunk) in width_batches(&lines) { + for (&i, text) in chunk.iter().zip(self.recognize_batch(w, &chunk, &lines)?) { + texts[i] = text; } } @@ -305,67 +235,3 @@ impl OcrModel { Ok(cells) } } - -/// Split a region crop into text lines via a horizontal ink-projection profile. -/// Returns tight `(l, t, r, b)` boxes in crop pixels. -fn segment_lines(crop: &RgbImage) -> Vec<(u32, u32, u32, u32)> { - let (w, h) = crop.dimensions(); - if w == 0 || h == 0 { - return Vec::new(); - } - let mean: f32 = crop.pixels().map(luma).sum::() / (w * h) as f32; - let thresh = mean * 0.7; // ink = noticeably darker than the page average - let min_ink = ((w as f32) * 0.005).max(1.0) as u32; - - let mut profile = vec![0u32; h as usize]; - for y in 0..h { - let mut row = 0u32; - for x in 0..w { - if luma(crop.get_pixel(x, y)) < thresh { - row += 1; - } - } - profile[y as usize] = row; - } - - // Maximal runs of text rows, separated by (near-)blank rows. - let mut runs: Vec<(u32, u32)> = Vec::new(); - let mut start: Option = None; - for y in 0..h { - let text = profile[y as usize] >= min_ink; - if text && start.is_none() { - start = Some(y); - } else if !text { - if let Some(s) = start.take() { - if y - s >= 4 { - runs.push((s, y)); - } - } - } - } - if let Some(s) = start { - if h - s >= 4 { - runs.push((s, h)); - } - } - - // Tighten each line to its horizontal ink bounds. - runs.into_iter() - .map(|(t, b)| { - let (mut l, mut r) = (w, 0u32); - for y in t..b { - for x in 0..w { - if luma(crop.get_pixel(x, y)) < thresh { - l = l.min(x); - r = r.max(x + 1); - } - } - } - if l >= r { - (0, t, w, b) - } else { - (l, t, r, b) - } - }) - .collect() -} diff --git a/crates/docling-pdf/src/ocr_prep.rs b/crates/docling-pdf/src/ocr_prep.rs new file mode 100644 index 00000000..1af8ef80 --- /dev/null +++ b/crates/docling-pdf/src/ocr_prep.rs @@ -0,0 +1,247 @@ +//! ONNX-free half of the PP-OCRv3 recognition pipeline: everything before +//! and after the `session.run` call — line segmentation, crop preparation, +//! width-batching, CTC decoding, dictionary handling. +//! +//! Split out of `ocr.rs` (which keeps the `ort` session) so the browser build +//! can reuse it (issue #79 phase 2): `docling-wasm` runs these exact +//! functions and delegates only the inference call to ONNX Runtime Web on +//! the JS side. Keeping one implementation is what makes the wasm output +//! byte-comparable to the native CPU path — any drift then comes from the +//! runtime, not from pre/post-processing. + +use image::{imageops, imageops::FilterType, Rgb, RgbImage}; + +/// PP-OCRv3's fixed input height. +pub const REC_HEIGHT: u32 = 48; + +/// Cap on lines per recognition run: bounds peak input-tensor memory +/// (16 × 3 × 48 × 2400 px ≈ 22 MB f32) without costing measurable batching +/// benefit — same-width groups are rarely larger. +pub const REC_BATCH: usize = 16; + +/// A text-line crop prepared for recognition: resized to the fixed model +/// height, normalised to `[-1, 1]`, laid out CHW. +pub struct PrepLine { + /// Width after the aspect-preserving resize to [`REC_HEIGHT`]. + pub w: usize, + /// `3 * REC_HEIGHT * w` values. + pub data: Vec, +} + +/// Prepare one line crop, or `None` for a degenerate (zero-sized) crop. +pub fn prep_line(line: &RgbImage) -> Option { + let (w, h) = line.dimensions(); + if w == 0 || h == 0 { + return None; + } + let new_w = ((w as f32) * REC_HEIGHT as f32 / h as f32) + .round() + .clamp(8.0, 2400.0) as u32; + let resized = imageops::resize(line, new_w, REC_HEIGHT, FilterType::Triangle); + let n = (REC_HEIGHT * new_w) as usize; + // Normalise to [-1, 1]: (x/255 - 0.5) / 0.5. + let mut data = vec![0f32; 3 * n]; + for (i, px) in resized.pixels().enumerate() { + data[i] = px[0] as f32 / 127.5 - 1.0; + data[n + i] = px[1] as f32 / 127.5 - 1.0; + data[2 * n + i] = px[2] as f32 / 127.5 - 1.0; + } + Some(PrepLine { + w: new_w as usize, + data, + }) +} + +/// The CTC class table for a recognition dictionary file: index 0 = blank, +/// then one class per dictionary line, then the space class. +pub fn dict_chars(dict: &str) -> Vec { + let mut chars = vec![String::new()]; // blank at 0 + chars.extend(dict.lines().map(|s| s.to_string())); + chars.push(" ".to_string()); + chars +} + +/// Greedy CTC decode of one row's `(T, C)` probabilities. +pub fn decode_row(chars: &[String], probs: &[f32], nc: usize) -> String { + let mut out = String::new(); + let mut prev = 0usize; + for row in probs.chunks_exact(nc) { + let mut best = 0usize; + let mut bestv = row[0]; + for (c, &v) in row.iter().enumerate().skip(1) { + if v > bestv { + bestv = v; + best = c; + } + } + if best != prev && best != 0 { + if let Some(ch) = chars.get(best) { + out.push_str(ch); + } + } + prev = best; + } + out +} + +pub(crate) fn luma(p: &Rgb) -> f32 { + 0.299 * p[0] as f32 + 0.587 * p[1] as f32 + 0.114 * p[2] as f32 +} + +/// Split a region crop into text lines via a horizontal ink-projection profile. +/// Returns tight `(l, t, r, b)` boxes in crop pixels. +pub fn segment_lines(crop: &RgbImage) -> Vec<(u32, u32, u32, u32)> { + let (w, h) = crop.dimensions(); + if w == 0 || h == 0 { + return Vec::new(); + } + let mean: f32 = crop.pixels().map(luma).sum::() / (w * h) as f32; + let thresh = mean * 0.7; // ink = noticeably darker than the page average + let min_ink = ((w as f32) * 0.005).max(1.0) as u32; + + let mut profile = vec![0u32; h as usize]; + for y in 0..h { + let mut row = 0u32; + for x in 0..w { + if luma(crop.get_pixel(x, y)) < thresh { + row += 1; + } + } + profile[y as usize] = row; + } + + // Maximal runs of text rows, separated by (near-)blank rows. + let mut runs: Vec<(u32, u32)> = Vec::new(); + let mut start: Option = None; + for y in 0..h { + let text = profile[y as usize] >= min_ink; + if text && start.is_none() { + start = Some(y); + } else if !text { + if let Some(s) = start.take() { + if y - s >= 4 { + runs.push((s, y)); + } + } + } + } + if let Some(s) = start { + if h - s >= 4 { + runs.push((s, h)); + } + } + + // Tighten each line to its horizontal ink bounds. + runs.into_iter() + .map(|(t, b)| { + let (mut l, mut r) = (w, 0u32); + for y in t..b { + for x in 0..w { + if luma(crop.get_pixel(x, y)) < thresh { + l = l.min(x); + r = r.max(x + 1); + } + } + } + if l >= r { + (0, t, w, b) + } else { + (l, t, r, b) + } + }) + .collect() +} + +/// Whole-image line preparation for the browser OCR path (no layout model: +/// the page itself is the single text region). Returns page-order prepared +/// lines; callers that need geometry use [`segment_lines`] directly. +pub fn prep_page_lines(img: &RgbImage) -> Vec { + segment_lines(img) + .into_iter() + .filter_map(|(l, t, r, b)| { + let line = imageops::crop_imm(img, l, t, r - l, b - t).to_image(); + prep_line(&line) + }) + .collect() +} + +/// Deterministic recognition batching: page-order line indices grouped by +/// exact width (equal widths share a run — bit-identical to one-at-a-time +/// recognition, see `ocr.rs`), each group split into [`REC_BATCH`] chunks. +pub fn width_batches(lines: &[PrepLine]) -> Vec<(usize, Vec)> { + let mut by_width: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for (ix, pl) in lines.iter().enumerate() { + by_width.entry(pl.w).or_default().push(ix); + } + let mut out = Vec::new(); + for (w, ixs) in by_width { + for chunk in ixs.chunks(REC_BATCH) { + out.push((w, chunk.to_vec())); + } + } + out +} + +/// Pack one width-batch into the model's `(N, 3, H, W)` input buffer. +pub fn batch_input(w: usize, chunk: &[usize], lines: &[PrepLine]) -> Vec { + let hw = REC_HEIGHT as usize * w; + let mut data = vec![0f32; chunk.len() * 3 * hw]; + for (i, &ix) in chunk.iter().enumerate() { + data[i * 3 * hw..(i + 1) * 3 * hw].copy_from_slice(&lines[ix].data); + } + data +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A synthetic "page": white background with two black text bars. + fn page() -> RgbImage { + let mut img = RgbImage::from_pixel(200, 100, Rgb([255, 255, 255])); + for y in 20..30 { + for x in 10..190 { + img.put_pixel(x, y, Rgb([0, 0, 0])); + } + } + for y in 60..72 { + for x in 10..120 { + img.put_pixel(x, y, Rgb([0, 0, 0])); + } + } + img + } + + #[test] + fn segments_and_preps_page_lines() { + let lines = prep_page_lines(&page()); + assert_eq!(lines.len(), 2); + for pl in &lines { + assert_eq!(pl.data.len(), 3 * REC_HEIGHT as usize * pl.w); + } + // Different aspect ratios → different widths → separate batches. + let batches = width_batches(&lines); + assert_eq!(batches.len(), 2); + let (w0, chunk0) = &batches[0]; + assert_eq!( + batch_input(*w0, chunk0, &lines).len(), + 3 * REC_HEIGHT as usize * w0 + ); + } + + #[test] + fn ctc_decode_collapses_repeats_and_blanks() { + // 3 classes: blank, "a", "b"; timesteps a a blank b b → "ab". + let chars = dict_chars("a\nb"); + assert_eq!(chars.len(), 4); // blank, a, b, space + let probs = [ + 0.1, 0.8, 0.1, 0.0, // a + 0.1, 0.8, 0.1, 0.0, // a (repeat collapses) + 0.9, 0.05, 0.05, 0.0, // blank + 0.1, 0.1, 0.8, 0.0, // b + 0.1, 0.1, 0.8, 0.0, // b (repeat collapses) + ]; + assert_eq!(decode_row(&chars, &probs, 4), "ab"); + } +} diff --git a/crates/docling-wasm/Cargo.toml b/crates/docling-wasm/Cargo.toml index 762b90cb..1183f0ae 100644 --- a/crates/docling-wasm/Cargo.toml +++ b/crates/docling-wasm/Cargo.toml @@ -16,10 +16,30 @@ publish = false # wasm-bindgen needs a cdylib; rlib keeps `cargo test` runnable on the host. crate-type = ["cdylib", "rlib"] +[features] +# Browser OCR (#157 stage 1): the ONNX-free half of the PP-OCRv3 recognition +# pipeline compiled in, with inference delegated to ONNX Runtime Web on the +# JS side. Pure Rust — the ort/pdfium native stack stays out. +default = ["ocr"] +ocr = [ + "dep:docling-core", + "dep:docling-pdf", + "dep:image", + "dep:js-sys", + "dep:wasm-bindgen-futures", +] + [dependencies] # The pure-Rust declarative slice only: no pdfium/ONNX (pdf, asr), no HTTP # client (fetch-images) — exactly the set that compiles for wasm32 (#79). docling = { path = "../docling", version = "0.49.0", default-features = false, features = ["pdf-text"] } +docling-core = { path = "../docling-core", version = "0.49.0", optional = true } +# ocr_prep only: line prep + CTC decode shared with the native pipeline. +docling-pdf = { path = "../docling-pdf", version = "0.49.0", default-features = false, features = ["ocr-prep"], optional = true } +# Input decoding for the OCR path; codecs mirror the browser's usual suspects. +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "bmp", "tiff", "webp"], optional = true } +js-sys = { version = "0.3", optional = true } +wasm-bindgen-futures = { version = "0.4", optional = true } serde_json = "1" wasm-bindgen = "0.2" # Panic messages land in the browser console instead of "unreachable executed". diff --git a/crates/docling-wasm/README.md b/crates/docling-wasm/README.md index b23622fb..9833c4b0 100644 --- a/crates/docling-wasm/README.md +++ b/crates/docling-wasm/README.md @@ -62,6 +62,32 @@ Verified end-to-end in headless Chromium: Markdown/DOCX→md, DOCX→JSON, a corpus PDF→md through the text-layer path, and the scanned-PDF error path all exercised through the real wasm module. +## In-browser OCR (#157 stage 1, experimental) + +With the default `ocr` feature, the module also exports `ocr_image`: OCR a +scanned **image** entirely client-side. Line segmentation, crop preparation +and CTC decoding run in Rust (`docling_pdf::ocr_prep` — the same code as the +native pipeline); the PP-OCRv3 recognition inference (~10 MB model) is +delegated to [ONNX Runtime Web](https://onnxruntime.ai/docs/tutorials/web/) +through a small JS wrapper you construct around an `ort.InferenceSession`: + +```js +const rec = { + run: async (n, h, w, data) => { + const results = await session.run({ [session.inputNames[0]]: + new ort.Tensor("float32", data, [n, 3, h, w]) }); + const t = results[session.outputNames[0]]; + return { data: t.data, dims: Array.from(t.dims) }; + }, +}; +const markdown = await ocr_image(imageBytes, dictText, rec, "md"); +``` + +[`www/ocr.html`](./www/ocr.html) is the complete demo (model/dict fetched +from their public hosting and browser-cached). Whole-image projection +segmentation only — best on single-column scans; layout detection is +stage 2, scanned PDFs (pdf.js rasterization) arrive with it. + ## Host-side tests `cargo test -p docling-wasm` runs the conversion body natively (the diff --git a/crates/docling-wasm/src/lib.rs b/crates/docling-wasm/src/lib.rs index 99aec907..699a7070 100644 --- a/crates/docling-wasm/src/lib.rs +++ b/crates/docling-wasm/src/lib.rs @@ -19,6 +19,11 @@ use docling::{DocumentConverter, InputFormat, SourceDocument}; use wasm_bindgen::prelude::*; +#[cfg(feature = "ocr")] +mod ocr; +#[cfg(feature = "ocr")] +pub use ocr::ocr_image; + #[wasm_bindgen(start)] fn start() { // Panics surface as readable messages in the browser console instead of diff --git a/crates/docling-wasm/src/ocr.rs b/crates/docling-wasm/src/ocr.rs new file mode 100644 index 00000000..0ae34e56 --- /dev/null +++ b/crates/docling-wasm/src/ocr.rs @@ -0,0 +1,134 @@ +//! Browser OCR — stage 1 of #157 (`#79 phase 2: ML pipeline in the browser`). +//! +//! Scanned **images** convert fully client-side: all pre/post-processing +//! (line segmentation, crop preparation, width-batching, CTC decoding) runs +//! in Rust via `docling_pdf::ocr_prep` — the *same* code the native pipeline +//! uses — and only the PP-OCRv3 recognition inference is delegated to +//! [ONNX Runtime Web](https://onnxruntime.ai/docs/tutorials/web/) on the JS +//! side through the [`RecSession`] interop interface. One shared +//! implementation is what keeps the wasm output comparable to the native CPU +//! path; drift can then only come from the runtime's kernels. +//! +//! Without a layout model (that's stage 2), the whole image is treated as a +//! single text region and split into lines by the ink-projection profile — +//! fine for typical single-column scans, degraded on complex layouts. +//! +//! ```js +//! import * as ort from "onnxruntime-web"; +//! import init, { ocr_image } from "./pkg/docling_wasm.js"; +//! await init(); +//! const session = await ort.InferenceSession.create("ocr_rec_en.onnx"); +//! const dict = await (await fetch("en_dict.txt")).text(); +//! const md = await ocr_image(imageBytes, dict, { +//! run: async (n, h, w, data) => { +//! const out = (await session.run({ x: new ort.Tensor("float32", data, [n, 3, h, w]) })).softmax_2.data +//! ?? Object.values(await session.run({ x: new ort.Tensor("float32", data, [n, 3, h, w]) }))[0]; +//! // return the output tensor: { data: Float32Array, dims: [n, t, c] } +//! }, +//! }); +//! ``` +//! (see `www/ocr.html` for the complete wiring, including output-name +//! discovery and model/dict caching.) + +use docling_core::{DoclingDocument, Node}; +use docling_pdf::ocr_prep::{ + batch_input, decode_row, dict_chars, prep_page_lines, width_batches, REC_HEIGHT, +}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + /// The JS-side recognition session: a wrapper object around an + /// `ort.InferenceSession` exposing one method, + /// `run(count, height, width, data)`, which feeds the `(N,3,H,W)` CHW + /// float buffer to the model and resolves to + /// `{ data: Float32Array, dims: [n, t, c] }` — the recognition + /// probabilities tensor. + pub type RecSession; + + #[wasm_bindgen(method, catch)] + async fn run( + this: &RecSession, + count: u32, + height: u32, + width: u32, + data: js_sys::Float32Array, + ) -> Result; +} + +/// OCR a scanned image entirely in the browser: `bytes` is the image file +/// (PNG/JPEG/…), `dict` the recognition dictionary text (`en_dict.txt` for +/// the default English model), `session` the JS inference wrapper. Returns +/// Markdown (default) or docling JSON per `to`, one paragraph per recognized +/// line. +#[wasm_bindgen] +pub async fn ocr_image( + bytes: &[u8], + dict: &str, + session: &RecSession, + to: Option, +) -> Result { + let img = image::load_from_memory(bytes) + .map_err(|e| JsError::new(&format!("decode image: {e}")))? + .to_rgb8(); + let lines = prep_page_lines(&img); + let chars = dict_chars(dict); + let mut texts = vec![String::new(); lines.len()]; + for (w, chunk) in width_batches(&lines) { + let input = batch_input(w, &chunk, &lines); + let out = session + .run( + chunk.len() as u32, + REC_HEIGHT, + w as u32, + js_sys::Float32Array::from(input.as_slice()), + ) + .await + .map_err(|e| JsError::new(&format!("session.run: {e:?}")))?; + let (probs, t_len, nc) = tensor_parts(&out)?; + if probs.len() < chunk.len() * t_len * nc { + return Err(JsError::new("session.run returned a short tensor")); + } + for (i, &ix) in chunk.iter().enumerate() { + texts[ix] = decode_row(&chars, &probs[i * t_len * nc..(i + 1) * t_len * nc], nc); + } + } + let mut doc = DoclingDocument::new("image"); + doc.nodes.extend( + texts + .into_iter() + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()) + .map(|text| Node::Paragraph { text }), + ); + match to.as_deref().unwrap_or("md") { + "md" | "markdown" => Ok(doc.export_to_markdown()), + "json" => Ok(doc.export_to_json()), + other => Err(JsError::new(&format!( + "unknown output format {other:?} (expected \"md\" or \"json\")" + ))), + } +} + +/// Pull `{ data: Float32Array, dims: [n, t, c] }` out of the JS result. +fn tensor_parts(out: &JsValue) -> Result<(Vec, usize, usize), JsError> { + let get = |k: &str| { + js_sys::Reflect::get(out, &JsValue::from_str(k)) + .map_err(|_| JsError::new(&format!("session.run result has no `{k}`"))) + }; + let data: js_sys::Float32Array = get("data")? + .dyn_into() + .map_err(|_| JsError::new("`data` is not a Float32Array"))?; + let dims: js_sys::Array = get("dims")? + .dyn_into() + .map_err(|_| JsError::new("`dims` is not an array"))?; + if dims.length() != 3 { + return Err(JsError::new("`dims` must be [n, t, c]")); + } + let t_len = dims.get(1).as_f64().unwrap_or(0.0) as usize; + let nc = dims.get(2).as_f64().unwrap_or(0.0) as usize; + if t_len == 0 || nc == 0 { + return Err(JsError::new("`dims` must be [n, t, c] with t, c > 0")); + } + Ok((data.to_vec(), t_len, nc)) +} diff --git a/crates/docling-wasm/www/ocr.html b/crates/docling-wasm/www/ocr.html new file mode 100644 index 00000000..37c14666 --- /dev/null +++ b/crates/docling-wasm/www/ocr.html @@ -0,0 +1,102 @@ + + + + + + docling.rs — in-browser OCR (wasm + ONNX Runtime Web) + + + +

docling.rs → wasm: OCR a scanned image in your browser

+

+ Stage 1 of #157: + the PP-OCRv3 recognition model (~10 MB) runs via + ONNX Runtime Web; + line segmentation and CTC decoding run in Rust/wasm — the same code as the + native pipeline. The image never leaves this page. The model + dictionary + are fetched once and cached by the browser. Single-column scans work best + (layout detection is stage 2). +

+
drop a scanned image here (PNG/JPEG/TIFF/WebP) or click to pick
+ +

loading wasm + model …

+
+ + + + From 76add75301a5658f3cb0e8bc5193c278892bc45a Mon Sep 17 00:00:00 2001 From: artiz Date: Fri, 24 Jul 2026 10:39:01 +0000 Subject: [PATCH 02/40] wasm: lite-profile scanned pipeline in the browser (#157 stage 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanned PDFs and images now convert fully client-side through the real pipeline shape: RT-DETR layout detection + PP-OCRv3 recognition via ONNX Runtime Web, everything else in Rust — the native pipeline's own functions, no second implementation. docling-pdf (all under the existing ocr-prep feature, wasm-safe): - layout.rs: input packing (640x640 CHW) and RT-DETR decoding (sigmoid top-q, center->corners, per-label thresholds downstream) extracted as layout_input/decode_layout; the native batch path now decodes through the same function per page. - ocr_prep: prep_region_lines — the exact region-crop/line-split/prep gathering native ocr_page does — moved out and shared; ocr.rs consumes it. - new scanned module: the Worker::process refinement chain (label thresholds -> resolve -> orphans -> false-picture drops -> contained drops), page assembly with the geometric table fallback (the lite profile: TableFormer is stage 3), and cross-page continuation merging, re-exposed without the ml feature. Native path byte-identical after the extraction — the scanned-tiff conversion diffs empty against the pre-refactor output (rebuilt and re-verified after a disk-full broke the first attempt), full docling-pdf/docling suites green. docling-wasm: - ScannedConverter: feed pdf.js-rendered RGBA pages ({scale: 2} — the native RENDER_SCALE) one by one, finish() assembles the document with the same continuation merging as native; page_count() for progress. - convert_scanned_image: one-shot image path (its own page at scale 1, like native). - LayoutSession interop mirrors RecSession: run(data) -> {logits: {data, dims}, boxes: {data, dims}}. - www/scan.html: complete demo — pdf.js from CDN, layout int8 (~165 MB, models-v1 release, browser-cached) + rec model + dict, per-page progress, timing. Stage 3 (TableFormer + enrichment in the browser) remains: the autoregressive decoder loop is deeply ort-typed and needs its own careful extraction pass; table regions meanwhile take the geometric reconstruction, matching --no-table-former. Refs #157 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- crates/docling-pdf/src/layout.rs | 110 ++++++++++------ crates/docling-pdf/src/lib.rs | 2 + crates/docling-pdf/src/ocr.rs | 55 +------- crates/docling-pdf/src/ocr_prep.rs | 60 +++++++++ crates/docling-pdf/src/scanned.rs | 54 ++++++++ crates/docling-wasm/README.md | 30 ++++- crates/docling-wasm/src/lib.rs | 4 + crates/docling-wasm/src/ocr.rs | 19 +-- crates/docling-wasm/src/scanned.rs | 196 +++++++++++++++++++++++++++++ crates/docling-wasm/www/scan.html | 141 +++++++++++++++++++++ 10 files changed, 572 insertions(+), 99 deletions(-) create mode 100644 crates/docling-pdf/src/scanned.rs create mode 100644 crates/docling-wasm/src/scanned.rs create mode 100644 crates/docling-wasm/www/scan.html diff --git a/crates/docling-pdf/src/layout.rs b/crates/docling-pdf/src/layout.rs index 782a9684..895ef7f5 100644 --- a/crates/docling-pdf/src/layout.rs +++ b/crates/docling-pdf/src/layout.rs @@ -47,13 +47,12 @@ pub struct Region { pub b: f32, } -#[cfg(feature = "ml")] /// Base confidence threshold (docling-ibm-models `base_threshold`): the raw /// RT-DETR floor before docling's `LayoutPostprocessor` applies its stricter /// per-label thresholds ([`label_threshold`]). const THRESHOLD: f32 = 0.3; -#[cfg(feature = "ml")] -const SIDE: u32 = 640; +/// RT-DETR's fixed square input side. +pub const SIDE: u32 = 640; /// Per-label confidence threshold, ported from docling's /// `LayoutPostprocessor.CONFIDENCE_THRESHOLDS`. The raw predictor keeps every @@ -222,46 +221,81 @@ impl LayoutModel { let logits = &logits[p * num_queries * num_classes..(p + 1) * num_queries * num_classes]; let boxes = &boxes[p * num_queries * 4..(p + 1) * num_queries * 4]; - - // sigmoid over every (query, class); take the top `num_queries` scores. - let mut scored: Vec<(f32, usize)> = (0..num_queries * num_classes) - .map(|idx| (sigmoid(logits[idx]), idx)) - .collect(); - scored.sort_unstable_by(|a, b| b.0.total_cmp(&a.0)); - scored.truncate(num_queries); - - let mut regions = Vec::new(); - for (score, idx) in scored { - if score <= THRESHOLD { - continue; - } - let label_id = idx % num_classes; - let q = idx / num_classes; - let cx = boxes[q * 4]; - let cy = boxes[q * 4 + 1]; - let w = boxes[q * 4 + 2]; - let h = boxes[q * 4 + 3]; - // center_to_corners, then scale normalized coords to page points. - let l = (cx - w / 2.0) * page_w; - let t = (cy - h / 2.0) * page_h; - let r = (cx + w / 2.0) * page_w; - let b = (cy + h / 2.0) * page_h; - regions.push(Region { - label: LABELS.get(label_id).copied().unwrap_or("text"), - score, - l, - t, - r, - b, - }); - } - all.push(regions); + all.push(decode_layout( + logits, + boxes, + num_queries, + num_classes, + *page_w, + *page_h, + )); } Ok(all) } } -#[cfg(feature = "ml")] fn sigmoid(x: f32) -> f32 { 1.0 / (1.0 + (-x).exp()) } + +/// Pack one page image into the model's `(1, 3, SIDE, SIDE)` input: resize +/// (aspect ignored, RT-DETR convention), rescale to `[0,1]`, CHW. Shared +/// with the browser build (#157), which delegates only the session call. +#[cfg(feature = "ocr-prep")] +pub fn layout_input(img: &image::RgbImage) -> Vec { + let n = (SIDE * SIDE) as usize; + let mut data = vec![0f32; 3 * n]; + let resized = image::imageops::resize(img, SIDE, SIDE, image::imageops::FilterType::Triangle); + for (i, px) in resized.pixels().enumerate() { + data[i] = px[0] as f32 / 255.0; + data[n + i] = px[1] as f32 / 255.0; + data[2 * n + i] = px[2] as f32 / 255.0; + } + data +} + +/// Decode one page's raw RT-DETR outputs into scored [`Region`]s in page +/// points — sigmoid over every (query, class), top-`num_queries` kept, boxes +/// converted center→corners and scaled. Shared with the browser build; the +/// native batch path calls it per page, so both decode identically. +pub fn decode_layout( + logits: &[f32], + boxes: &[f32], + num_queries: usize, + num_classes: usize, + page_w: f32, + page_h: f32, +) -> Vec { + let mut scored: Vec<(f32, usize)> = (0..num_queries * num_classes) + .map(|idx| (sigmoid(logits[idx]), idx)) + .collect(); + scored.sort_unstable_by(|a, b| b.0.total_cmp(&a.0)); + scored.truncate(num_queries); + + let mut regions = Vec::new(); + for (score, idx) in scored { + if score <= THRESHOLD { + continue; + } + let label_id = idx % num_classes; + let q = idx / num_classes; + let cx = boxes[q * 4]; + let cy = boxes[q * 4 + 1]; + let w = boxes[q * 4 + 2]; + let h = boxes[q * 4 + 3]; + // center_to_corners, then scale normalized coords to page points. + let l = (cx - w / 2.0) * page_w; + let t = (cy - h / 2.0) * page_h; + let r = (cx + w / 2.0) * page_w; + let b = (cy + h / 2.0) * page_h; + regions.push(Region { + label: LABELS.get(label_id).copied().unwrap_or("text"), + score, + l, + t, + r, + b, + }); + } + regions +} diff --git a/crates/docling-pdf/src/lib.rs b/crates/docling-pdf/src/lib.rs index 776ea0bc..2bf85eb6 100644 --- a/crates/docling-pdf/src/lib.rs +++ b/crates/docling-pdf/src/lib.rs @@ -33,6 +33,8 @@ pub mod pdfium_backend; mod reading_order; #[cfg(feature = "ml")] pub mod resample; +#[cfg(feature = "ocr-prep")] +pub mod scanned; #[cfg(feature = "ml")] pub mod tableformer; pub mod textparse; diff --git a/crates/docling-pdf/src/ocr.rs b/crates/docling-pdf/src/ocr.rs index 64e781a2..0a2b3f91 100644 --- a/crates/docling-pdf/src/ocr.rs +++ b/crates/docling-pdf/src/ocr.rs @@ -5,7 +5,7 @@ //! is recognised and decoded with CTC — producing [`TextCell`]s the normal //! layout assembly then consumes. This avoids a separate text-detection model. -use image::{imageops, RgbImage}; +use image::RgbImage; use ort::session::Session; use ort::value::Tensor; @@ -13,8 +13,7 @@ use crate::layout::Region; // The ONNX-free half (line prep, batching, CTC decode) lives in `ocr_prep` // so the wasm build shares it verbatim (#79 phase 2). use crate::ocr_prep::{ - batch_input, decode_row, dict_chars, prep_line, segment_lines, width_batches, PrepLine, - REC_HEIGHT, + batch_input, decode_row, dict_chars, prep_region_lines, width_batches, PrepLine, REC_HEIGHT, }; use crate::pdfium_backend::TextCell; @@ -24,21 +23,6 @@ pub struct OcrModel { chars: Vec, } -/// Layout labels whose content is recognised as running text. -fn is_text_label(label: &str) -> bool { - matches!( - label, - "text" - | "title" - | "section_header" - | "list_item" - | "caption" - | "footnote" - | "code" - | "formula" - ) -} - /// OCR recognition language: which PP-OCRv3 model + dictionary pair runs. /// /// The default is **English** (`models/ocr_rec_en.onnx` + `models/en_dict.txt`): @@ -183,37 +167,10 @@ impl OcrModel { regions: &[Region], scale: f32, ) -> Result, String> { - let (iw, ih) = img.dimensions(); - // Gather every line crop on the page first, so equal-width lines can - // share a recognition run regardless of which region they came from. - let mut bboxes: Vec<(f32, f32, f32, f32)> = Vec::new(); - let mut lines: Vec = Vec::new(); - for region in regions { - if !is_text_label(region.label) { - continue; - } - let l = (region.l * scale).max(0.0) as u32; - let t = (region.t * scale).max(0.0) as u32; - let r = ((region.r * scale).max(0.0) as u32).min(iw); - let b = ((region.b * scale).max(0.0) as u32).min(ih); - if r <= l || b <= t { - continue; - } - let crop = imageops::crop_imm(img, l, t, r - l, b - t).to_image(); - for (lx, ly, rx, ry) in segment_lines(&crop) { - let line = imageops::crop_imm(&crop, lx, ly, rx - lx, ry - ly).to_image(); - let Some(pl) = prep_line(&line) else { - continue; - }; - bboxes.push(( - (l + lx) as f32 / scale, - (t + ly) as f32 / scale, - (l + rx) as f32 / scale, - (t + ry) as f32 / scale, - )); - lines.push(pl); - } - } + // Gather every line crop on the page first (shared with the browser + // path), so equal-width lines can share a recognition run regardless + // of which region they came from. + let (bboxes, lines) = prep_region_lines(img, regions, scale); // Deterministic width-batching (shared with the wasm path). let mut texts = vec![String::new(); lines.len()]; diff --git a/crates/docling-pdf/src/ocr_prep.rs b/crates/docling-pdf/src/ocr_prep.rs index 1af8ef80..0c5e812b 100644 --- a/crates/docling-pdf/src/ocr_prep.rs +++ b/crates/docling-pdf/src/ocr_prep.rs @@ -152,6 +152,66 @@ pub fn segment_lines(crop: &RgbImage) -> Vec<(u32, u32, u32, u32)> { .collect() } +/// Layout labels whose content is recognised as running text. +pub fn is_text_label(label: &str) -> bool { + matches!( + label, + "text" + | "title" + | "section_header" + | "list_item" + | "caption" + | "footnote" + | "code" + | "formula" + ) +} + +/// A line's page-point bounding box, `(l, t, r, b)`. +pub type LineBox = (f32, f32, f32, f32); + +/// Gather every text-region line crop on a page, in page order: crop each +/// text region (page points × `scale` → image px), split it into lines, prep +/// each line, and keep the line's page-point bbox. The exact gathering the +/// native `ocr_page` does — shared so the browser path produces the same +/// cells given the same probabilities. +pub fn prep_region_lines( + img: &RgbImage, + regions: &[crate::layout::Region], + scale: f32, +) -> (Vec, Vec) { + let (iw, ih) = img.dimensions(); + let mut bboxes = Vec::new(); + let mut lines = Vec::new(); + for region in regions { + if !is_text_label(region.label) { + continue; + } + let l = (region.l * scale).max(0.0) as u32; + let t = (region.t * scale).max(0.0) as u32; + let r = ((region.r * scale).max(0.0) as u32).min(iw); + let b = ((region.b * scale).max(0.0) as u32).min(ih); + if r <= l || b <= t { + continue; + } + let crop = imageops::crop_imm(img, l, t, r - l, b - t).to_image(); + for (lx, ly, rx, ry) in segment_lines(&crop) { + let line = imageops::crop_imm(&crop, lx, ly, rx - lx, ry - ly).to_image(); + let Some(pl) = prep_line(&line) else { + continue; + }; + bboxes.push(( + (l + lx) as f32 / scale, + (t + ly) as f32 / scale, + (l + rx) as f32 / scale, + (t + ry) as f32 / scale, + )); + lines.push(pl); + } + } + (bboxes, lines) +} + /// Whole-image line preparation for the browser OCR path (no layout model: /// the page itself is the single text region). Returns page-order prepared /// lines; callers that need geometry use [`segment_lines`] directly. diff --git a/crates/docling-pdf/src/scanned.rs b/crates/docling-pdf/src/scanned.rs new file mode 100644 index 00000000..9aad62bc --- /dev/null +++ b/crates/docling-pdf/src/scanned.rs @@ -0,0 +1,54 @@ +//! Scanned-page assembly facade for the browser build (#157 stage 2). +//! +//! The native `Worker::process` chain, re-exposed without the `ml` feature so +//! `docling-wasm` can run it around JS-delegated inference: refine the raw +//! layout detections exactly like the native pipeline, then assemble the page +//! with the geometric table fallback (the lite profile — TableFormer is +//! stage 3) and no enrichments. One shared implementation, one behavior. + +use docling_core::{DoclingDocument, Node}; + +/// One assembled page: its nodes and `(anchor, href)` hyperlink pairs. +pub type AssembledPage = (Vec, Vec<(String, String)>); + +use crate::layout::{label_threshold, Region}; +use crate::pdfium_backend::{PdfPage, TextCell}; + +/// The native pipeline's region-refinement chain, in its exact order: +/// per-label score thresholds → overlap resolution → orphan text regions for +/// detector-missed cells → false-picture drops → contained-regular drops. +/// `cells` is the page's (possibly empty, pre-OCR) text-cell set. +pub fn refine_regions( + regions: Vec, + cells: &[TextCell], + page_w: f32, + page_h: f32, +) -> Vec { + let mut regions = regions; + regions.retain(|r| r.score >= label_threshold(r.label)); + let mut regions = crate::assemble::resolve(regions); + crate::assemble::add_orphan_regions(&mut regions, cells); + crate::assemble::drop_false_pictures(&mut regions, cells, page_w, page_h); + crate::assemble::drop_contained_regulars(&mut regions); + regions +} + +/// Assemble one refined page — geometric tables (no TableFormer), no +/// enrichments — into its nodes and hyperlink pairs. +pub fn assemble_page(page: &PdfPage, regions: Vec) -> AssembledPage { + let table_rows = vec![None; regions.len()]; + let enrich = vec![None; regions.len()]; + crate::assemble::assemble_page(page, regions, &table_rows, &enrich) +} + +/// Stitch per-page results into a document: cross-page paragraph +/// continuations merge exactly like the native pipeline's final pass. +pub fn finish_document(name: &str, pages: Vec) -> DoclingDocument { + let mut doc = DoclingDocument::new(name); + for (nodes, links) in pages { + doc.nodes.extend(nodes); + doc.links.extend(links); + } + crate::assemble::merge_continuations(&mut doc.nodes); + doc +} diff --git a/crates/docling-wasm/README.md b/crates/docling-wasm/README.md index 9833c4b0..4d28eaa5 100644 --- a/crates/docling-wasm/README.md +++ b/crates/docling-wasm/README.md @@ -83,10 +83,32 @@ const rec = { const markdown = await ocr_image(imageBytes, dictText, rec, "md"); ``` -[`www/ocr.html`](./www/ocr.html) is the complete demo (model/dict fetched -from their public hosting and browser-cached). Whole-image projection -segmentation only — best on single-column scans; layout detection is -stage 2, scanned PDFs (pdf.js rasterization) arrive with it. +[`www/ocr.html`](./www/ocr.html) is the recognition-only demo (model/dict +fetched from their public hosting and browser-cached). + +**Stage 2 — the lite profile** (`ScannedConverter` / `convert_scanned_image`): +RT-DETR layout detection (int8, ~165 MB from the `models-v1` release) + +region-cropped OCR + reading-order assembly, with tables via the geometric +reconstruction (the native `--no-table-former` path — TableFormer is +stage 3). Scanned **PDFs** convert too: the host page rasterizes pages with +pdf.js at `{scale: 2}` (2 px/point — the native pipeline's `RENDER_SCALE`) +and feeds the bitmaps page by page: + +```js +const conv = new ScannedConverter(dictText); +for (let p = 1; p <= pdf.numPages; p++) { + const viewport = page.getViewport({ scale: 2 }); + // … render to canvas, then: + await conv.add_page(rgba, canvas.width, canvas.height, 2.0, layout, rec); +} +const markdown = conv.finish(file.name, "md"); +``` + +[`www/scan.html`](./www/scan.html) is the complete demo (pdf.js + both +models). Region refinement, OCR gathering/batching/decoding and page +assembly are the native pipeline's own functions +(`docling_pdf::{layout, ocr_prep, scanned}`) — the wasm path adds no second +implementation. ## Host-side tests diff --git a/crates/docling-wasm/src/lib.rs b/crates/docling-wasm/src/lib.rs index 699a7070..286a8138 100644 --- a/crates/docling-wasm/src/lib.rs +++ b/crates/docling-wasm/src/lib.rs @@ -22,7 +22,11 @@ use wasm_bindgen::prelude::*; #[cfg(feature = "ocr")] mod ocr; #[cfg(feature = "ocr")] +mod scanned; +#[cfg(feature = "ocr")] pub use ocr::ocr_image; +#[cfg(feature = "ocr")] +pub use scanned::{convert_scanned_image, ScannedConverter}; #[wasm_bindgen(start)] fn start() { diff --git a/crates/docling-wasm/src/ocr.rs b/crates/docling-wasm/src/ocr.rs index 0ae34e56..67396677 100644 --- a/crates/docling-wasm/src/ocr.rs +++ b/crates/docling-wasm/src/ocr.rs @@ -47,7 +47,7 @@ extern "C" { pub type RecSession; #[wasm_bindgen(method, catch)] - async fn run( + pub async fn run( this: &RecSession, count: u32, height: u32, @@ -110,8 +110,10 @@ pub async fn ocr_image( } } -/// Pull `{ data: Float32Array, dims: [n, t, c] }` out of the JS result. -fn tensor_parts(out: &JsValue) -> Result<(Vec, usize, usize), JsError> { +/// Pull `{ data: Float32Array, dims: [..] }` out of a JS tensor object; +/// returns `(data, dims[len-2], dims[len-1])` — the trailing two dims are +/// what every consumer indexes by. +pub(crate) fn tensor_parts(out: &JsValue) -> Result<(Vec, usize, usize), JsError> { let get = |k: &str| { js_sys::Reflect::get(out, &JsValue::from_str(k)) .map_err(|_| JsError::new(&format!("session.run result has no `{k}`"))) @@ -122,13 +124,14 @@ fn tensor_parts(out: &JsValue) -> Result<(Vec, usize, usize), JsError> { let dims: js_sys::Array = get("dims")? .dyn_into() .map_err(|_| JsError::new("`dims` is not an array"))?; - if dims.length() != 3 { - return Err(JsError::new("`dims` must be [n, t, c]")); + let len = dims.length(); + if len < 2 { + return Err(JsError::new("`dims` must have at least 2 entries")); } - let t_len = dims.get(1).as_f64().unwrap_or(0.0) as usize; - let nc = dims.get(2).as_f64().unwrap_or(0.0) as usize; + let t_len = dims.get(len - 2).as_f64().unwrap_or(0.0) as usize; + let nc = dims.get(len - 1).as_f64().unwrap_or(0.0) as usize; if t_len == 0 || nc == 0 { - return Err(JsError::new("`dims` must be [n, t, c] with t, c > 0")); + return Err(JsError::new("trailing `dims` entries must be > 0")); } Ok((data.to_vec(), t_len, nc)) } diff --git a/crates/docling-wasm/src/scanned.rs b/crates/docling-wasm/src/scanned.rs new file mode 100644 index 00000000..cdfc8045 --- /dev/null +++ b/crates/docling-wasm/src/scanned.rs @@ -0,0 +1,196 @@ +//! Browser scanned-document pipeline — stage 2 of #157 (the *lite profile*): +//! RT-DETR layout detection + PP-OCRv3 recognition, both via ONNX Runtime Web +//! on the JS side; region refinement, region-cropped OCR, geometric table +//! reconstruction and reading-order assembly in Rust — the same code as the +//! native pipeline (`docling_pdf::{layout, ocr_prep, scanned}`). TableFormer +//! and the enrichment models are stage 3; table regions fall back to the +//! geometric reconstruction the native `--no-table-former` flag uses. +//! +//! Pages arrive as raw RGBA bitmaps: for a scanned PDF the host page renders +//! them with pdf.js (`page.getViewport({scale: 2})` — 2 px per PDF point, +//! matching the native pipeline's `RENDER_SCALE`); a standalone image is its +//! own page at scale 1, exactly like the native image path. +//! +//! ```js +//! const conv = new ScannedConverter(dictText); +//! for (const bitmap of pages) { +//! await conv.add_page(bitmap.data, bitmap.width, bitmap.height, 2.0, layout, rec); +//! } +//! const markdown = conv.finish("scan.pdf", "md"); +//! ``` +//! (`www/scan.html` is the complete wiring.) + +use docling_pdf::layout::{decode_layout, layout_input, SIDE}; +use docling_pdf::ocr_prep::{ + batch_input, decode_row, dict_chars, prep_region_lines, width_batches, REC_HEIGHT, +}; +use docling_pdf::pdfium_backend::{PdfPage, TextCell}; +use docling_pdf::scanned::{assemble_page, finish_document, refine_regions}; +use image::RgbImage; +use wasm_bindgen::prelude::*; + +use crate::ocr::{tensor_parts, RecSession}; + +#[wasm_bindgen] +extern "C" { + /// The JS-side layout session: a wrapper around an `ort.InferenceSession` + /// over the RT-DETR layout model exposing `run(data)` — feed the + /// `(1, 3, 640, 640)` CHW float buffer, resolve to + /// `{ logits: {data, dims: [1, q, c]}, boxes: {data, dims: [1, q, 4]} }`. + pub type LayoutSession; + + #[wasm_bindgen(method, catch)] + pub async fn run(this: &LayoutSession, data: js_sys::Float32Array) -> Result; +} + +/// Multi-page scanned-document converter (lite profile). Feed pages in +/// order, then [`finish`](Self::finish) — cross-page paragraph continuations +/// merge exactly like the native pipeline. +#[wasm_bindgen] +pub struct ScannedConverter { + chars: Vec, + pages: Vec, +} + +#[wasm_bindgen] +impl ScannedConverter { + /// `dict` is the recognition dictionary text (`en_dict.txt` for the + /// default English model). + #[wasm_bindgen(constructor)] + pub fn new(dict: &str) -> Self { + Self { + chars: dict_chars(dict), + pages: Vec::new(), + } + } + + /// Convert one page: `rgba` is the rendered bitmap (canvas ImageData), + /// `scale` its pixels-per-PDF-point (2.0 for pdf.js `{scale: 2}`; 1.0 + /// for a standalone image). + pub async fn add_page( + &mut self, + rgba: &[u8], + px_w: u32, + px_h: u32, + scale: f32, + layout: &LayoutSession, + rec: &RecSession, + ) -> Result<(), JsError> { + if rgba.len() != (px_w as usize) * (px_h as usize) * 4 { + return Err(JsError::new("rgba buffer size does not match dimensions")); + } + let mut img = RgbImage::new(px_w, px_h); + for (i, px) in img.pixels_mut().enumerate() { + px.0 = [rgba[i * 4], rgba[i * 4 + 1], rgba[i * 4 + 2]]; + } + let (page_w, page_h) = (px_w as f32 / scale, px_h as f32 / scale); + + // Layout: Rust preprocessing → JS inference → Rust decoding. + let input = layout_input(&img); + let out = layout + .run(js_sys::Float32Array::from(input.as_slice())) + .await + .map_err(|e| JsError::new(&format!("layout session.run: {e:?}")))?; + let get = |k: &str| { + js_sys::Reflect::get(&out, &JsValue::from_str(k)) + .map_err(|_| JsError::new(&format!("layout result has no `{k}`"))) + }; + let (logits, q, c) = tensor_parts(&get("logits")?)?; + let (boxes, bq, four) = tensor_parts(&get("boxes")?)?; + if bq != q || four != 4 { + return Err(JsError::new("layout boxes dims must be [1, q, 4]")); + } + let regions = decode_layout(&logits, &boxes, q, c, page_w, page_h); + let regions = refine_regions(regions, &[], page_w, page_h); + + // OCR the text regions (same gather/batch/decode as native ocr_page). + let (bboxes, lines) = prep_region_lines(&img, ®ions, scale); + let mut texts = vec![String::new(); lines.len()]; + for (w, chunk) in width_batches(&lines) { + let data = batch_input(w, &chunk, &lines); + let out = rec + .run( + chunk.len() as u32, + REC_HEIGHT, + w as u32, + js_sys::Float32Array::from(data.as_slice()), + ) + .await + .map_err(|e| JsError::new(&format!("rec session.run: {e:?}")))?; + let (probs, t_len, nc) = tensor_parts(&out)?; + if probs.len() < chunk.len() * t_len * nc { + return Err(JsError::new("rec session.run returned a short tensor")); + } + for (i, &ix) in chunk.iter().enumerate() { + texts[ix] = decode_row( + &self.chars, + &probs[i * t_len * nc..(i + 1) * t_len * nc], + nc, + ); + } + } + let mut cells = Vec::new(); + for ((l, t, r, b), text) in bboxes.into_iter().zip(texts) { + let text = text.trim().to_string(); + if text.is_empty() { + continue; + } + cells.push(TextCell { text, l, t, r, b }); + } + + let page = PdfPage { + width: page_w, + height: page_h, + scale, + cells, + code_cells: Vec::new(), + word_cells: Vec::new(), + links: Vec::new(), + }; + self.pages.push(assemble_page(&page, regions)); + Ok(()) + } + + /// Number of pages converted so far (progress display). + pub fn page_count(&self) -> usize { + self.pages.len() + } + + /// Assemble the accumulated pages into the final document and render it + /// as `"md"` (default) or `"json"`. Resets the converter. + pub fn finish(&mut self, name: &str, to: Option) -> Result { + let doc = finish_document(name, std::mem::take(&mut self.pages)); + match to.as_deref().unwrap_or("md") { + "md" | "markdown" => Ok(doc.export_to_markdown()), + "json" => Ok(doc.export_to_json()), + other => Err(JsError::new(&format!( + "unknown output format {other:?} (expected \"md\" or \"json\")" + ))), + } + } +} + +/// One-shot scanned-image conversion through the full lite profile (layout + +/// OCR + assembly) — the browser counterpart of the native image path +/// (a standalone image is its own page at scale 1). +#[wasm_bindgen] +pub async fn convert_scanned_image( + bytes: &[u8], + name: &str, + dict: &str, + layout: &LayoutSession, + rec: &RecSession, + to: Option, +) -> Result { + let img = image::load_from_memory(bytes) + .map_err(|e| JsError::new(&format!("decode image: {e}")))? + .to_rgba8(); + let (w, h) = img.dimensions(); + let mut conv = ScannedConverter::new(dict); + conv.add_page(img.as_raw(), w, h, 1.0, layout, rec).await?; + conv.finish(name, to) +} + +// Silence the unused warning for SIDE re-export path (the JS side sizes its +// tensor from the buffer length, but the constant documents the contract). +const _: u32 = SIDE; diff --git a/crates/docling-wasm/www/scan.html b/crates/docling-wasm/www/scan.html new file mode 100644 index 00000000..9a7e12d0 --- /dev/null +++ b/crates/docling-wasm/www/scan.html @@ -0,0 +1,141 @@ + + + + + + docling.rs — scanned PDFs in the browser (lite profile) + + + +

docling.rs → wasm: scanned PDF / image, full lite profile

+

+ Stage 2 of #157: + RT-DETR layout detection (int8, ~165 MB — cached after the first load) + + PP-OCRv3 recognition via ONNX Runtime Web; pdf.js rasterizes PDF pages + at the native pipeline's 2 px/point; region refinement, OCR and + reading-order assembly run in Rust/wasm — the same code as the native + pipeline. Tables come out via the geometric reconstruction + (TableFormer is stage 3). Nothing leaves this page. +

+
drop a scanned PDF or image here, or click to pick
+ +

loading wasm …

+
+ + + + From ebd2cb6e18273278a38102cc59f37ee69c8e9e25 Mon Sep 17 00:00:00 2001 From: artiz Date: Fri, 24 Jul 2026 10:49:27 +0000 Subject: [PATCH 03/40] =?UTF-8?q?wasm:=20fix=20workspace=20clippy=20?= =?UTF-8?q?=E2=80=94=20PdfPage::from=5Fcells=20absorbs=20the=20ml=20image?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workspace-wide clippy job unifies features: docling-cli enables docling-pdf/ml alongside docling-wasm, so PdfPage grows its ml-gated `image` field and docling-wasm's struct literal stopped compiling (E0063). A cfg-aware constructor (from_cells, under ocr-prep) spells the field only when it exists; the browser pipeline builds its pages through it. Verified with the CI command verbatim (cargo clippy --workspace --all-targets -- -D warnings) plus the wasm32 target check. Refs #157 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- crates/docling-pdf/src/pdfium_backend.rs | 22 ++++++++++++++++++++++ crates/docling-wasm/src/scanned.rs | 10 +--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/crates/docling-pdf/src/pdfium_backend.rs b/crates/docling-pdf/src/pdfium_backend.rs index 10688d32..f2d8a932 100644 --- a/crates/docling-pdf/src/pdfium_backend.rs +++ b/crates/docling-pdf/src/pdfium_backend.rs @@ -53,6 +53,28 @@ pub struct PdfPage { pub links: Vec, } +impl PdfPage { + /// A page built from recognized cells alone — the browser pipeline's + /// shape (#157), where the bitmap lives on the JS side. Exists so callers + /// compile identically with and without the `ml` feature: under a + /// feature-unified workspace build the struct carries the `image` field, + /// which a plain literal in a non-`ml` consumer can't spell. + #[cfg(feature = "ocr-prep")] + pub fn from_cells(width: f32, height: f32, scale: f32, cells: Vec) -> Self { + Self { + width, + height, + scale, + cells, + code_cells: Vec::new(), + word_cells: Vec::new(), + #[cfg(feature = "ml")] + image: RgbImage::new(0, 0), + links: Vec::new(), + } + } +} + /// A PDF link annotation: its rectangle (top-left page coordinates, matching /// [`TextCell`]) and target URI. #[derive(Debug, Clone)] diff --git a/crates/docling-wasm/src/scanned.rs b/crates/docling-wasm/src/scanned.rs index cdfc8045..746df69e 100644 --- a/crates/docling-wasm/src/scanned.rs +++ b/crates/docling-wasm/src/scanned.rs @@ -138,15 +138,7 @@ impl ScannedConverter { cells.push(TextCell { text, l, t, r, b }); } - let page = PdfPage { - width: page_w, - height: page_h, - scale, - cells, - code_cells: Vec::new(), - word_cells: Vec::new(), - links: Vec::new(), - }; + let page = PdfPage::from_cells(page_w, page_h, scale, cells); self.pages.push(assemble_page(&page, regions)); Ok(()) } From 16a09557ca37614eca60c2a4223ec99f5fe5a042 Mon Sep 17 00:00:00 2001 From: artiz Date: Fri, 24 Jul 2026 12:40:50 +0000 Subject: [PATCH 04/40] wasm: polarity normalization for dark-mode captures (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First real-browser test fed ocr.html a dark-theme screenshot and got "(no text found)": the whole scan pipeline assumes dark ink on light paper — the line segmentation thresholds pixels *darker* than the page mean, so light-on-dark input yields garbage. New ocr_prep::normalize_polarity inverts a predominantly dark page (mean luma below mid-gray) before segmentation and inference; both browser entry points (ocr_image, ScannedConverter::add_page — layout assumes scan polarity too) run it. The native pipeline deliberately does NOT: its input is scanned paper and the conformance baseline stays pinned. Test: the inverted synthetic page misfires raw and recovers exactly after normalization; a light page passes through untouched. Refs #157 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- crates/docling-pdf/src/ocr_prep.rs | 37 ++++++++++++++++++++++++++++++ crates/docling-wasm/src/ocr.rs | 13 +++++++---- crates/docling-wasm/src/scanned.rs | 6 ++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/docling-pdf/src/ocr_prep.rs b/crates/docling-pdf/src/ocr_prep.rs index 0c5e812b..7665309d 100644 --- a/crates/docling-pdf/src/ocr_prep.rs +++ b/crates/docling-pdf/src/ocr_prep.rs @@ -212,6 +212,26 @@ pub fn prep_region_lines( (bboxes, lines) } +/// Normalize an image to the scan polarity every stage assumes — dark ink +/// on light paper (the segmentation threshold and the recognition model's +/// training data both bake it in): a predominantly dark page (mean luma +/// below mid-gray — a dark-mode screenshot, an inverted scan) is inverted. +/// Browser-path helper; the native pipeline never calls it (its input is +/// scanned paper, and the conformance baseline stays untouched). +pub fn normalize_polarity(mut img: RgbImage) -> RgbImage { + let (w, h) = img.dimensions(); + if w == 0 || h == 0 { + return img; + } + let mean: f32 = img.pixels().map(luma).sum::() / (w * h) as f32; + if mean < 128.0 { + for px in img.pixels_mut() { + px.0 = [255 - px.0[0], 255 - px.0[1], 255 - px.0[2]]; + } + } + img +} + /// Whole-image line preparation for the browser OCR path (no layout model: /// the page itself is the single text region). Returns page-order prepared /// lines; callers that need geometry use [`segment_lines`] directly. @@ -290,6 +310,23 @@ mod tests { ); } + #[test] + fn dark_mode_pages_normalize_to_scan_polarity() { + // The same two-bar page, inverted (light text on dark) — the raw + // segmentation misfires (it thresholds the dark *background* as ink); + // polarity normalization recovers the true structure. + let mut dark = page(); + for px in dark.pixels_mut() { + px.0 = [255 - px.0[0], 255 - px.0[1], 255 - px.0[2]]; + } + assert_ne!(prep_page_lines(&dark).len(), 2); + let fixed = normalize_polarity(dark); + assert_eq!(prep_page_lines(&fixed).len(), 2); + // A light page passes through untouched. + let light = page(); + assert_eq!(normalize_polarity(light.clone()), light); + } + #[test] fn ctc_decode_collapses_repeats_and_blanks() { // 3 classes: blank, "a", "b"; timesteps a a blank b b → "ab". diff --git a/crates/docling-wasm/src/ocr.rs b/crates/docling-wasm/src/ocr.rs index 67396677..3b0d030e 100644 --- a/crates/docling-wasm/src/ocr.rs +++ b/crates/docling-wasm/src/ocr.rs @@ -32,7 +32,8 @@ use docling_core::{DoclingDocument, Node}; use docling_pdf::ocr_prep::{ - batch_input, decode_row, dict_chars, prep_page_lines, width_batches, REC_HEIGHT, + batch_input, decode_row, dict_chars, normalize_polarity, prep_page_lines, width_batches, + REC_HEIGHT, }; use wasm_bindgen::prelude::*; @@ -68,9 +69,13 @@ pub async fn ocr_image( session: &RecSession, to: Option, ) -> Result { - let img = image::load_from_memory(bytes) - .map_err(|e| JsError::new(&format!("decode image: {e}")))? - .to_rgb8(); + // Screenshots and dark-mode captures arrive light-on-dark; the whole + // pipeline assumes scan polarity, so normalize first. + let img = normalize_polarity( + image::load_from_memory(bytes) + .map_err(|e| JsError::new(&format!("decode image: {e}")))? + .to_rgb8(), + ); let lines = prep_page_lines(&img); let chars = dict_chars(dict); let mut texts = vec![String::new(); lines.len()]; diff --git a/crates/docling-wasm/src/scanned.rs b/crates/docling-wasm/src/scanned.rs index 746df69e..6c561604 100644 --- a/crates/docling-wasm/src/scanned.rs +++ b/crates/docling-wasm/src/scanned.rs @@ -22,7 +22,8 @@ use docling_pdf::layout::{decode_layout, layout_input, SIDE}; use docling_pdf::ocr_prep::{ - batch_input, decode_row, dict_chars, prep_region_lines, width_batches, REC_HEIGHT, + batch_input, decode_row, dict_chars, normalize_polarity, prep_region_lines, width_batches, + REC_HEIGHT, }; use docling_pdf::pdfium_backend::{PdfPage, TextCell}; use docling_pdf::scanned::{assemble_page, finish_document, refine_regions}; @@ -83,6 +84,9 @@ impl ScannedConverter { for (i, px) in img.pixels_mut().enumerate() { px.0 = [rgba[i * 4], rgba[i * 4 + 1], rgba[i * 4 + 2]]; } + // Dark-mode screenshots invert scan polarity; normalize before both + // layout and OCR (each assumes dark ink on light paper). + let img = normalize_polarity(img); let (page_w, page_h) = (px_w as f32 / scale, px_h as f32 / scale); // Layout: Rust preprocessing → JS inference → Rust decoding. From 47c1783328c16218d084d0a377f3440b62166681 Mon Sep 17 00:00:00 2001 From: artiz Date: Fri, 24 Jul 2026 12:46:28 +0000 Subject: [PATCH 05/40] wasm: recognition-language selector in the browser demos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The demo pages hardcoded the English PP-OCRv3 model, so any non-Latin scan (e.g. Cyrillic) decoded to mojibake through the wrong dictionary. The wasm API is language-agnostic — model and dict are injected from JS — so the fix is demo-side only: both ocr.html and scan.html get a recognition-language -

loading wasm + model …

+

+ Recognition language: + + +

+
before the module script. + +if (typeof window === "undefined") { + // --- service-worker context ----------------------------------------------- + self.addEventListener("install", () => self.skipWaiting()); + self.addEventListener("activate", (e) => e.waitUntil(self.clients.claim())); + + self.addEventListener("fetch", (event) => { + const req = event.request; + // `only-if-cached` is only valid with same-origin `no-cors`; leave it alone. + if (req.cache === "only-if-cached" && req.mode !== "same-origin") return; + + event.respondWith( + fetch(req) + .then((res) => { + if (res.status === 0) return res; // opaque response — pass through + const headers = new Headers(res.headers); + headers.set("Cross-Origin-Embedder-Policy", "credentialless"); + headers.set("Cross-Origin-Opener-Policy", "same-origin"); + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers, + }); + }) + .catch((e) => console.error(e)), + ); + }); +} else { + // --- page context: register self, reload once to gain isolation ----------- + (async () => { + if (window.crossOriginIsolated) return; // already isolated — nothing to do + if (!window.isSecureContext || !navigator.serviceWorker) return; // can't + const reg = await navigator.serviceWorker + .register(document.currentScript.src) + .catch(() => null); + if (!reg) return; + // A fresh registration doesn't control this load yet; reload so the SW can + // stamp COOP/COEP on the document and crossOriginIsolated flips true. + if (!navigator.serviceWorker.controller) window.location.reload(); + })(); +} diff --git a/crates/docling-wasm/www/ocr.html b/crates/docling-wasm/www/ocr.html index 6646e895..7716c8cd 100644 --- a/crates/docling-wasm/www/ocr.html +++ b/crates/docling-wasm/www/ocr.html @@ -3,6 +3,10 @@ + + docling.rs — in-browser OCR (wasm + ONNX Runtime Web)