diff --git a/README.md b/README.md index 2430d08f..059d9a56 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,13 @@ The CLI streams Markdown by default (`--no-stream` opts back into buffering; loading/running the TableFormer table-structure model, falling back to simple geometric table reconstruction from cell positions — no model load, no per-table inference, which can noticeably speed up parsing (especially in -streaming mode) at the cost of table fidelity. +streaming mode) at the cost of table fidelity. `--no-ocr` goes further and +skips layout detection, OCR, and TableFormer entirely — no ML inference at +all, only the PDF's embedded text cells grouped into flat paragraphs by +reading order (no headings/lists/tables/pictures). It's the fastest PDF path +by a wide margin, but a scanned/image-only PDF (no embedded text layer) comes +back empty rather than erroring, so a caller can detect that and re-convert +without the flag. ## Node.js / Bun bindings diff --git a/crates/fleischwolf-cli/src/main.rs b/crates/fleischwolf-cli/src/main.rs index adf76e95..d6aa577d 100644 --- a/crates/fleischwolf-cli/src/main.rs +++ b/crates/fleischwolf-cli/src/main.rs @@ -3,7 +3,7 @@ //! A stand-in for `docling.cli.main`; the full Typer-style CLI (batch mode, //! pipeline options) is a later phase. //! -//! Usage: fleischwolf [--strict] [--to md|json] [--images MODE] [--fetch-images] [--no-stream] [--no-table-former] +//! Usage: fleischwolf [--strict] [--to md|json] [--images MODE] [--fetch-images] [--no-stream] [--no-table-former] [--no-ocr] //! --to md|json output format (default: md). `json` emits docling-core's //! native DoclingDocument JSON (export_to_dict). //! --images MODE picture handling for Markdown (mirrors docling's @@ -24,6 +24,13 @@ //! geometric reconstruction from cell positions. Faster //! (no model load, no per-table inference) at the cost of //! table fidelity — helps most in streaming mode. +//! --no-ocr skip layout detection, OCR, and TableFormer entirely for +//! PDF/image input — no model load or inference at all. +//! Emits the embedded text layer as flat paragraphs in +//! reading order (no headings/lists/tables/pictures). The +//! fastest option, but a scanned/image-only PDF (no +//! embedded text layer) yields no text — convert those +//! without this flag. use std::io::{self, Write}; use std::path::Path; @@ -38,6 +45,7 @@ fn main() -> ExitCode { let mut fetch_images = false; let mut no_stream = false; let mut no_table_former = false; + let mut no_ocr = false; let mut bench_warm: Option = None; let mut path: Option = None; let mut args = std::env::args().skip(1); @@ -47,6 +55,7 @@ fn main() -> ExitCode { "--fetch-images" => fetch_images = true, "--no-stream" => no_stream = true, "--no-table-former" => no_table_former = true, + "--no-ocr" => no_ocr = true, "--to" => to = args.next().unwrap_or_default(), "--images" => images = args.next().unwrap_or_default(), // Hidden benchmarking aid: load the PDF/image pipeline once, then time @@ -81,7 +90,7 @@ fn main() -> ExitCode { }; let Some(path) = path else { - eprintln!("usage: fleischwolf [--strict] [--to md|json] [--images MODE] [--fetch-images] [--no-stream] [--no-table-former] "); + eprintln!("usage: fleischwolf [--strict] [--to md|json] [--images MODE] [--fetch-images] [--no-stream] [--no-table-former] [--no-ocr] "); return ExitCode::from(2); }; @@ -94,7 +103,7 @@ fn main() -> ExitCode { }; if let Some(runs) = bench_warm { - return match bench_warm_conversion(&source, runs, no_table_former) { + return match bench_warm_conversion(&source, runs, no_table_former, no_ocr) { Ok(avg) => { // Bare seconds on stdout for the benchmark harness; a human line on stderr. println!("{avg:.6}"); @@ -114,7 +123,8 @@ fn main() -> ExitCode { let converter = DocumentConverter::new() .strict(strict) .fetch_images(fetch_images) - .no_table_former(no_table_former); + .no_table_former(no_table_former) + .no_ocr(no_ocr); // Stream Markdown by default: print each chunk as the converter produces it // (page by page for PDF). JSON needs the whole tree, and the referenced image @@ -201,10 +211,12 @@ fn bench_warm_conversion( source: &SourceDocument, runs: usize, no_table_former: bool, + no_ocr: bool, ) -> Result { let mut pipeline = Pipeline::new() .map_err(|e| e.to_string())? - .no_table_former(no_table_former); + .no_table_former(no_table_former) + .no_ocr(no_ocr); let once = |p: &mut Pipeline| -> Result<(), String> { match source.format { InputFormat::Pdf => p diff --git a/crates/fleischwolf-pdf/src/lib.rs b/crates/fleischwolf-pdf/src/lib.rs index 2eb8dc8f..85d8bb89 100644 --- a/crates/fleischwolf-pdf/src/lib.rs +++ b/crates/fleischwolf-pdf/src/lib.rs @@ -82,23 +82,33 @@ type PageOut = (Vec, Vec<(String, String)>); /// parallel page-worker owns its own `Worker` so inference runs concurrently /// without sharing an ONNX session (`ort`'s `Session::run` is `&mut self`). struct Worker { - layout: layout::LayoutModel, + /// `None` when `no_ocr` skips layout entirely — no model load, no inference. + layout: Option, ocr: Option, /// TableFormer structure model; `None` when its ONNX graphs aren't present - /// (the assembler then falls back to geometric table reconstruction). + /// (the assembler then falls back to geometric table reconstruction) or + /// when `no_table_former`/`no_ocr` skip it. tables: Option, + /// Skip layout, OCR, and TableFormer; reconstruct text purely from the PDF's + /// embedded text layer. See [`Pipeline::no_ocr`]. + no_ocr: bool, } impl Worker { - fn load(intra: usize, no_table_former: bool) -> Result { + fn load(intra: usize, no_table_former: bool, no_ocr: bool) -> Result { Ok(Self { - layout: layout::LayoutModel::load_with(intra).map_err(PdfError::Layout)?, + layout: if no_ocr { + None + } else { + Some(layout::LayoutModel::load_with(intra).map_err(PdfError::Layout)?) + }, ocr: None, - tables: if no_table_former { + tables: if no_table_former || no_ocr { None } else { tableformer::TableFormer::load_with(intra) }, + no_ocr, }) } @@ -106,8 +116,25 @@ impl Worker { /// into its nodes and links. Pure given the page (mutates only the worker's /// lazily-loaded OCR model), so it is safe to run concurrently across pages. fn process(&mut self, n: usize, page: &mut PdfPage) -> Result { + if self.no_ocr { + // Fastest path: no layout/OCR/TableFormer inference at all. The PDF's + // embedded text cells (if any) become flat, line-grouped paragraphs in + // reading order via the same orphan-region machinery that normally + // rescues text the detector missed — here it rescues *all* of it. + // Pages with no embedded text layer (scanned/image-only) yield nothing; + // convert those without `no_ocr`. + let mut regions = Vec::new(); + assemble::add_orphan_regions(&mut regions, &page.cells); + let table_rows = vec![None; regions.len()]; + return Ok(timing::timed("assemble_page", || { + assemble::assemble_page(page, regions, &table_rows) + })); + } let regions = timing::timed("layout.predict", || { - self.layout.predict(&page.image, page.width, page.height) + self.layout + .as_mut() + .expect("layout model loaded unless no_ocr") + .predict(&page.image, page.width, page.height) }) .map_err(|e| PdfError::Layout(format!("page {}: {e}", n + 1)))?; // Resolve overlapping detections once, before OCR. @@ -215,6 +242,8 @@ pub struct Pipeline { /// Skip loading/running TableFormer; table regions fall back to geometric /// reconstruction. See [`Pipeline::no_table_former`]. no_table_former: bool, + /// Skip layout, OCR, and TableFormer entirely. See [`Pipeline::no_ocr`]. + no_ocr: bool, } impl Pipeline { @@ -228,6 +257,7 @@ impl Pipeline { target_workers: pdf_worker_count(), parallel_min: pdf_parallel_min(), no_table_former: false, + no_ocr: false, }) } @@ -242,10 +272,27 @@ impl Pipeline { self } + /// Skip layout detection, OCR, and TableFormer entirely — no model load, no + /// inference of any kind. The PDF's embedded text cells are grouped by line + /// and emitted as plain paragraphs in reading order: no headings, lists, + /// tables, code blocks, or pictures, since that structure comes from the + /// layout model. The fastest possible PDF path, but pages with no embedded + /// text layer (scanned/image-only PDFs) yield no text at all — convert those + /// without this flag. Implies `no_table_former`. No effect if a worker is + /// already loaded; set this before the first conversion. + pub fn no_ocr(mut self, disable: bool) -> Self { + self.no_ocr = disable; + self + } + /// The full-intra serial worker, loaded on first use. fn primary(&mut self) -> Result<&mut Worker, PdfError> { if self.primary.is_none() { - self.primary = Some(Worker::load(intra_threads(), self.no_table_former)?); + self.primary = Some(Worker::load( + intra_threads(), + self.no_table_former, + self.no_ocr, + )?); } Ok(self.primary.as_mut().unwrap()) } @@ -280,8 +327,9 @@ impl Pipeline { name: &str, ) -> Result { let mut doc = DoclingDocument::new(name); + let render_image = !self.no_ocr; let worker = self.primary()?; - pdfium_backend::for_each_page(bytes, password, |n, _total, mut page| { + pdfium_backend::for_each_page(bytes, password, render_image, |n, _total, mut page| { let (nodes, links) = worker.process(n, &mut page)?; doc.nodes.extend(nodes); doc.links.extend(links); @@ -304,6 +352,7 @@ impl Pipeline { ) -> Result { self.ensure_pool()?; let n_workers = self.pool.len(); + let render_image = !self.no_ocr; let (work_tx, work_rx) = sync_channel::<(usize, PdfPage)>(n_workers * 2); let work_rx: Arc>> = Arc::new(Mutex::new(work_rx)); let results: Arc>> = Arc::new(Mutex::new(Vec::new())); @@ -335,11 +384,12 @@ impl Pipeline { // Render on this thread and feed the workers; backpressure blocks here // when the channel is full. Dropping `work_tx` afterwards signals the // workers (recv → Err) to finish. - let render = pdfium_backend::for_each_page(bytes, password, |i, _total, page| { - work_tx - .send((i, page)) - .map_err(|_| PdfError::Pdfium("page-worker channel closed".into())) - }); + let render = + pdfium_backend::for_each_page(bytes, password, render_image, |i, _total, page| { + work_tx + .send((i, page)) + .map_err(|_| PdfError::Pdfium("page-worker channel closed".into())) + }); drop(work_tx); if let Err(e) = render { let mut slot = first_err.lock().unwrap(); @@ -412,8 +462,9 @@ impl Pipeline { F: FnMut(Vec, Vec<(String, String)>) -> Result<(), PdfError>, { let mut asm = assemble::StreamAssembler::new(); + let render_image = !self.no_ocr; let worker = self.primary()?; - pdfium_backend::for_each_page(bytes, password, |n, _total, mut page| { + pdfium_backend::for_each_page(bytes, password, render_image, |n, _total, mut page| { let (nodes, links) = worker.process(n, &mut page)?; emit(asm.push(nodes), links) })?; @@ -437,6 +488,7 @@ impl Pipeline { { self.ensure_pool()?; let n_workers = self.pool.len(); + let render_image = !self.no_ocr; let (work_tx, work_rx) = sync_channel::<(usize, PdfPage)>(n_workers * 2); let work_rx: Arc>> = Arc::new(Mutex::new(work_rx)); // Workers and the renderer report here; the calling thread drains it in @@ -467,12 +519,16 @@ impl Pipeline { { let res_tx = res_tx.clone(); s.spawn(move || { - let render = - pdfium_backend::for_each_page(bytes, password, |i, _total, page| { + let render = pdfium_backend::for_each_page( + bytes, + password, + render_image, + |i, _total, page| { work_tx .send((i, page)) .map_err(|_| PdfError::Pdfium("page-worker channel closed".into())) - }); + }, + ); drop(work_tx); // signal workers to finish if let Err(e) = render { let _ = res_tx.send(Err(e)); @@ -527,9 +583,10 @@ impl Pipeline { } let intra = pdf_intra(); let no_table_former = self.no_table_former; + let no_ocr = self.no_ocr; let loaded: Vec> = std::thread::scope(|s| { let handles: Vec<_> = (0..need) - .map(|_| s.spawn(move || Worker::load(intra, no_table_former))) + .map(|_| s.spawn(move || Worker::load(intra, no_table_former, no_ocr))) .collect(); handles.into_iter().map(|h| h.join().unwrap()).collect() }); @@ -587,53 +644,62 @@ pub fn convert( password: Option<&str>, name: &str, ) -> Result { - convert_with_options(bytes, password, name, false) + convert_with_options(bytes, password, name, false, false) } /// Like [`convert`], but optionally skips loading/running TableFormer (see -/// [`Pipeline::no_table_former`]). +/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see +/// [`Pipeline::no_ocr`]). pub fn convert_with_options( bytes: &[u8], password: Option<&str>, name: &str, no_table_former: bool, + no_ocr: bool, ) -> Result { Pipeline::new()? .no_table_former(no_table_former) + .no_ocr(no_ocr) .convert(bytes, password, name) } /// Convenience one-shot image conversion (loads the pipeline per call). pub fn convert_image(bytes: &[u8], name: &str) -> Result { - convert_image_with_options(bytes, name, false) + convert_image_with_options(bytes, name, false, false) } /// Like [`convert_image`], but optionally skips loading/running TableFormer (see -/// [`Pipeline::no_table_former`]). +/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see +/// [`Pipeline::no_ocr`]). pub fn convert_image_with_options( bytes: &[u8], name: &str, no_table_former: bool, + no_ocr: bool, ) -> Result { Pipeline::new()? .no_table_former(no_table_former) + .no_ocr(no_ocr) .convert_image(bytes, name) } /// Convert pre-segmented pages (image + already-known text cells, e.g. METS/hOCR /// scans) through the shared layout + assembly pipeline. pub fn convert_pages(pages: Vec, name: &str) -> Result { - convert_pages_with_options(pages, name, false) + convert_pages_with_options(pages, name, false, false) } /// Like [`convert_pages`], but optionally skips loading/running TableFormer (see -/// [`Pipeline::no_table_former`]). +/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see +/// [`Pipeline::no_ocr`]). pub fn convert_pages_with_options( pages: Vec, name: &str, no_table_former: bool, + no_ocr: bool, ) -> Result { Pipeline::new()? .no_table_former(no_table_former) + .no_ocr(no_ocr) .process_pages(pages, name) } diff --git a/crates/fleischwolf-pdf/src/mets.rs b/crates/fleischwolf-pdf/src/mets.rs index de6b139e..62aae539 100644 --- a/crates/fleischwolf-pdf/src/mets.rs +++ b/crates/fleischwolf-pdf/src/mets.rs @@ -20,15 +20,17 @@ use crate::pdfium_backend::{PdfPage, TextCell}; use crate::{convert_pages_with_options, PdfError}; pub fn convert_mets_gbs(bytes: &[u8], name: &str) -> Result { - convert_mets_gbs_with_options(bytes, name, false) + convert_mets_gbs_with_options(bytes, name, false, false) } /// Like [`convert_mets_gbs`], but optionally skips loading/running TableFormer -/// (see [`crate::Pipeline::no_table_former`]). +/// (see [`crate::Pipeline::no_table_former`]) and/or layout+OCR+TableFormer +/// entirely (see [`crate::Pipeline::no_ocr`]). pub fn convert_mets_gbs_with_options( bytes: &[u8], name: &str, no_table_former: bool, + no_ocr: bool, ) -> Result { let mut html: BTreeMap = BTreeMap::new(); let mut tiff: BTreeMap> = BTreeMap::new(); @@ -90,7 +92,7 @@ pub fn convert_mets_gbs_with_options( "mets: no hOCR/TIFF page pairs found in archive".into(), )); } - convert_pages_with_options(pages, name, no_table_former) + convert_pages_with_options(pages, name, no_table_former, no_ocr) } /// Parse an hOCR page: the `ocr_page` bbox gives the page geometry (cells are in diff --git a/crates/fleischwolf-pdf/src/pdfium_backend.rs b/crates/fleischwolf-pdf/src/pdfium_backend.rs index 429223b0..27662b78 100644 --- a/crates/fleischwolf-pdf/src/pdfium_backend.rs +++ b/crates/fleischwolf-pdf/src/pdfium_backend.rs @@ -138,7 +138,7 @@ impl PdfDocument { let mut pages = Vec::new(); for (i, page) in doc.pages().iter().enumerate() { let rc = rust.as_mut().and_then(|v| v.get_mut(i).map(std::mem::take)); - pages.push(extract_page(&page, &ffi, i as i32, rc)?); + pages.push(extract_page(&page, &ffi, i as i32, rc, true)?); } Ok(PdfDocument { pages }) } @@ -173,8 +173,20 @@ pub fn page_count(bytes: &[u8], password: Option<&str>) -> Result(bytes: &[u8], password: Option<&str>, mut f: F) -> Result<(), E> +pub fn for_each_page( + bytes: &[u8], + password: Option<&str>, + render_image: bool, + mut f: F, +) -> Result<(), E> where E: From, F: FnMut(usize, usize, PdfPage) -> Result<(), E>, @@ -187,7 +199,7 @@ where let total = pages.len() as usize; for (i, page) in pages.iter().enumerate() { let rc = rust.as_mut().and_then(|v| v.get_mut(i).map(std::mem::take)); - let extracted = extract_page(&page, &ffi, i as i32, rc)?; + let extracted = extract_page(&page, &ffi, i as i32, rc, render_image)?; f(i, total, extracted)?; } Ok(()) @@ -198,52 +210,69 @@ fn extract_page( ffi: &FfiText<'_>, index: i32, rust_cells: Option, + render_image: bool, ) -> Result { let width = page.width().value; let height = page.height().value; - let (mut cells, mut code_cells, mut word_cells) = - crate::timing::timed("ffi.page_cells", || ffi.page_cells(index, height)); - if cells.is_empty() { - cells = segment_cells(&page.text()?, height); - } // Default: use the pure-Rust text parser instead of pdfium's text layer // (override with `DOCLING_PDFIUM_TEXT`). Prose line cells always come from the // parser; word and code cells do too unless `DOCLING_PDFIUM_WORDS` keeps them // on pdfium (the parser's word grouping reproduces docling-parse's, which // TableFormer matches against — roadmap item 6). A page the parser couldn't // read (no text layer) keeps pdfium's cells. - if let Some(rc) = rust_cells { - if !rc.prose.is_empty() { - cells = rc.prose; - } - if use_parser_words() && !rc.words.is_empty() { - word_cells = rc.words; - } - if use_parser_code() && !rc.code.is_empty() { - code_cells = rc.code; - } + let rc = rust_cells.unwrap_or_default(); + let need_pdfium_prose = rc.prose.is_empty(); + let need_pdfium_words = !use_parser_words() || rc.words.is_empty(); + let need_pdfium_code = !use_parser_code() || rc.code.is_empty(); + + // The parser covers prose/words/code from one shared glyph pass, so on the + // common (parser-succeeded) page all three are already satisfied and this + // pdfium FFI call — otherwise fully discarded below — is skipped outright. + let (mut cells, mut code_cells, mut word_cells) = + if need_pdfium_prose || need_pdfium_words || need_pdfium_code { + let (mut cells, code_cells, word_cells) = + crate::timing::timed("ffi.page_cells", || ffi.page_cells(index, height)); + if cells.is_empty() { + cells = segment_cells(&page.text()?, height); + } + (cells, code_cells, word_cells) + } else { + (Vec::new(), Vec::new(), Vec::new()) + }; + if !rc.prose.is_empty() { + cells = rc.prose; + } + if use_parser_words() && !rc.words.is_empty() { + word_cells = rc.words; + } + if use_parser_code() && !rc.code.is_empty() { + code_cells = rc.code; } - // docling renders at 1.5× the target scale and downsamples "to make it - // sharper" (pypdfium2 → PIL BICUBIC). Replicate exactly: the TableFormer - // model is pixel-sensitive, so the page bitmap must match byte-for-byte. - // `CatmullRom` is the same a=-0.5 cubic kernel as PIL's BICUBIC. - const SUPERSAMPLE: f32 = 1.5; - let tw = (width * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32; - let th = (height * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32; - let cfg = PdfRenderConfig::new() - .set_target_width(tw) - .set_target_height(th); - let big = crate::timing::timed("pdfium.render", || { - page.render_with_config(&cfg) - .map(|b| b.as_image().into_rgb8()) - })?; - let dw = (width * RENDER_SCALE).round().max(1.0) as u32; - let dh = (height * RENDER_SCALE).round().max(1.0) as u32; - let image = crate::timing::timed("image.resize", || { - image::imageops::resize(&big, dw, dh, image::imageops::FilterType::CatmullRom) - }); + let image = if render_image { + // docling renders at 1.5× the target scale and downsamples "to make it + // sharper" (pypdfium2 → PIL BICUBIC). Replicate exactly: the TableFormer + // model is pixel-sensitive, so the page bitmap must match byte-for-byte. + // `CatmullRom` is the same a=-0.5 cubic kernel as PIL's BICUBIC. + const SUPERSAMPLE: f32 = 1.5; + let tw = (width * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32; + let th = (height * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32; + let cfg = PdfRenderConfig::new() + .set_target_width(tw) + .set_target_height(th); + let big = crate::timing::timed("pdfium.render", || { + page.render_with_config(&cfg) + .map(|b| b.as_image().into_rgb8()) + })?; + let dw = (width * RENDER_SCALE).round().max(1.0) as u32; + let dh = (height * RENDER_SCALE).round().max(1.0) as u32; + crate::timing::timed("image.resize", || { + image::imageops::resize(&big, dw, dh, image::imageops::FilterType::CatmullRom) + }) + } else { + RgbImage::new(1, 1) + }; Ok(PdfPage { width, diff --git a/crates/fleischwolf/src/converter.rs b/crates/fleischwolf/src/converter.rs index 21880170..77ea97c9 100644 --- a/crates/fleischwolf/src/converter.rs +++ b/crates/fleischwolf/src/converter.rs @@ -45,6 +45,7 @@ pub struct DocumentConverter { strict: bool, fetch_images: bool, no_table_former: bool, + no_ocr: bool, } impl DocumentConverter { @@ -61,6 +62,7 @@ impl DocumentConverter { strict: false, fetch_images: false, no_table_former: false, + no_ocr: false, } } @@ -106,6 +108,20 @@ impl DocumentConverter { self } + /// Skip layout detection, OCR, and TableFormer entirely for PDF/image/METS + /// sources — no model load, no inference of any kind. + /// + /// Off by default. When enabled, the PDF's embedded text cells are grouped by + /// line and emitted as plain paragraphs in reading order: no headings, lists, + /// tables, code blocks, or pictures, since that structure comes from the + /// layout model. The fastest possible PDF path, but pages with no embedded + /// text layer (scanned/image-only PDFs) yield no text at all — convert those + /// without this flag. Implies [`no_table_former`](Self::no_table_former). + pub fn no_ocr(mut self, disable: bool) -> Self { + self.no_ocr = disable; + self + } + /// Convert a source document to Markdown **incrementally**, returning an /// iterator of Markdown chunks (with picture placeholders). /// @@ -157,6 +173,7 @@ impl DocumentConverter { image_mode, self.strict, self.no_table_former, + self.no_ocr, )) } @@ -216,18 +233,21 @@ impl DocumentConverter { None, &source.name, self.no_table_former, + self.no_ocr, ) .map_err(|e| ConversionError::Parse(e.to_string()))?, InputFormat::Image => fleischwolf_pdf::convert_image_with_options( &source.bytes, &source.name, self.no_table_former, + self.no_ocr, ) .map_err(|e| ConversionError::Parse(e.to_string()))?, InputFormat::MetsGbs => fleischwolf_pdf::convert_mets_gbs_with_options( &source.bytes, &source.name, self.no_table_former, + self.no_ocr, ) .map_err(|e| ConversionError::Parse(e.to_string()))?, other => return Err(ConversionError::UnsupportedFormat(other)), diff --git a/crates/fleischwolf/src/stream.rs b/crates/fleischwolf/src/stream.rs index 696a2966..2e44a8a5 100644 --- a/crates/fleischwolf/src/stream.rs +++ b/crates/fleischwolf/src/stream.rs @@ -76,10 +76,19 @@ pub(crate) fn spawn( image_mode: ImageMode, strict: bool, no_table_former: bool, + no_ocr: bool, ) -> MarkdownStream { let (tx, rx) = sync_channel::>(CHANNEL_DEPTH); let handle = std::thread::spawn(move || { - run(converter, source, image_mode, strict, no_table_former, &tx) + run( + converter, + source, + image_mode, + strict, + no_table_former, + no_ocr, + &tx, + ) }); MarkdownStream { rx: Some(rx), @@ -96,12 +105,13 @@ fn run( image_mode: ImageMode, strict: bool, no_table_former: bool, + no_ocr: bool, tx: &std::sync::mpsc::SyncSender>, ) { match source.format { // PDF is the format with internal page-level parallelism, so it gets the // true streaming path: emit each page's Markdown in order as it completes. - InputFormat::Pdf => run_pdf(&source, image_mode, strict, no_table_former, tx), + InputFormat::Pdf => run_pdf(&source, image_mode, strict, no_table_former, no_ocr, tx), // Every other backend builds the whole `DoclingDocument` synchronously, so // there is no latency to stream away; serialize it through the same chunk // API for a uniform interface (one chunk plus the trailing newline). @@ -114,20 +124,22 @@ fn run_pdf( image_mode: ImageMode, strict: bool, no_table_former: bool, + no_ocr: bool, tx: &std::sync::mpsc::SyncSender>, ) { // The PDF pipeline builds its document from `DoclingDocument::new` defaults, so // tables use the padded GitHub serializer (compact_tables = false), matching the // buffered PDF path. let mut streamer = MarkdownStreamer::new(strict, image_mode, false); - let mut pipeline = - match fleischwolf_pdf::Pipeline::new().map(|p| p.no_table_former(no_table_former)) { - Ok(p) => p, - Err(e) => { - let _ = tx.send(Err(ConversionError::Parse(e.to_string()))); - return; - } - }; + let mut pipeline = match fleischwolf_pdf::Pipeline::new() + .map(|p| p.no_table_former(no_table_former).no_ocr(no_ocr)) + { + Ok(p) => p, + Err(e) => { + let _ = tx.send(Err(ConversionError::Parse(e.to_string()))); + return; + } + }; let result = pipeline.convert_streaming(&source.bytes, None, &source.name, |nodes, links| { let chunk = streamer.push(&nodes, &links);