From 5522d021b689ec7c5b4e5feba798e5c5e691c108 Mon Sep 17 00:00:00 2001 From: artiz Date: Sat, 25 Jul 2026 08:19:43 +0000 Subject: [PATCH 1/2] vlm: don't fail the whole page on malformed DocLang from the model (#153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VLM pipeline routes non-DocTags model output through the DocLang XML reader, which is a strict parser: a model that leaves a / unclosed produces markup it rejects with `expected 'heading' tag, not 'doclang'`, and that error propagated all the way out — dropping the entire normal_4pages fixture from the #153 corpus run ("rust side failed"). The DocTags path never hard-fails on hostile model output (#152: best-effort document out); the DocLang fallback now matches it. parse_doclang_fragments tries the strict reader first (well-formed DocLang keeps its heading levels and table structure) and, on a parse error, salvages the fragments through the tolerant DocTags parser, which degrades broken/unknown tags to text and never errors. Two regression tests cover both branches. Refs #153 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- crates/docling/src/vlm.rs | 57 ++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/crates/docling/src/vlm.rs b/crates/docling/src/vlm.rs index 2a86cfa3..6dcca6b8 100644 --- a/crates/docling/src/vlm.rs +++ b/crates/docling/src/vlm.rs @@ -184,11 +184,7 @@ pub fn convert_vlm( doc.name = source.name.clone(); doc } else { - let body: Vec = fragments.iter().map(|f| prose_fallback(f)).collect(); - let xml = format!("\n{}\n", body.join("\n")); - let synthetic = - SourceDocument::from_bytes(&source.name, InputFormat::XmlDoclang, xml.into_bytes()); - DoclangBackend.convert(&synthetic)? + parse_doclang_fragments(&source.name, &fragments) }; if doc.nodes.is_empty() { // The request loop succeeded, so this is a content problem, not a @@ -338,6 +334,30 @@ fn looks_like_doctags(fragment: &str) -> bool { || fragment.contains("`/`` unclosed or nests a stray root produces markup it +/// rejects outright (`expected 'heading' tag, not 'doclang' …`), which — when +/// propagated — fails the entire conversion on one bad page. The DocTags path +/// never does that (#152: hostile model output, best-effort document out), so +/// neither should this one: on a parse error, salvage the fragments through the +/// tolerant DocTags parser, which degrades broken/unknown tags to text and +/// never errors. Well-formed DocLang still takes the strict reader, keeping its +/// heading levels and table structure. +fn parse_doclang_fragments(name: &str, fragments: &[String]) -> DoclingDocument { + let body: Vec = fragments.iter().map(|f| prose_fallback(f)).collect(); + let xml = format!("\n{}\n", body.join("\n")); + let synthetic = SourceDocument::from_bytes(name, InputFormat::XmlDoclang, xml.into_bytes()); + let mut doc = match DoclangBackend.convert(&synthetic) { + Ok(doc) => doc, + Err(_) => docling_core::doctags::parse_pages(fragments.iter().map(String::as_str)), + }; + doc.name = name.to_string(); + doc +} + /// DocLang-path fallback for a model that ignored the markup instruction /// entirely (plain prose / Markdown, no tags): one `` per non-empty /// line, instead of silently dropping the page's content. @@ -396,7 +416,32 @@ fn encode_png(image: &image::RgbImage) -> Result, String> { #[cfg(test)] mod tests { - use super::{looks_like_doctags, prose_fallback, strip_wrappers}; + use super::{looks_like_doctags, parse_doclang_fragments, prose_fallback, strip_wrappers}; + + /// A misbehaving model can emit malformed DocLang — here an unclosed + /// `` — that the strict XML reader rejects with + /// `expected 'heading' tag, not 'doclang'`. That hard error dropped the + /// whole `normal_4pages` fixture from the #153 corpus run; the fallback + /// must salvage the page's text instead of failing the conversion. + #[test] + fn malformed_doclang_degrades_instead_of_erroring() { + let frag = "Unclosed title\nBody line.".to_string(); + let md = parse_doclang_fragments("normal_4pages.pdf", &[frag]).export_to_markdown(); + // Best-effort salvage via the tolerant DocTags parser: the text that + // the strict reader would have thrown away with the whole page survives. + assert!(md.contains("Body line."), "md: {md:?}"); + assert!(md.contains("Unclosed title"), "md: {md:?}"); + } + + /// Well-formed DocLang still takes the strict reader, which keeps the + /// heading *level* (the tolerant fallback would flatten it to text). + #[test] + fn wellformed_doclang_keeps_structure() { + let frag = "Sec\nPara.".to_string(); + let md = parse_doclang_fragments("d.pdf", &[frag]).export_to_markdown(); + assert!(md.contains("## Sec"), "md: {md:?}"); + assert!(md.contains("Para."), "md: {md:?}"); + } #[test] fn wrapper_stripping() { From 1ad8e486c6931390a47dc288ff798a126c353368 Mon Sep 17 00:00:00 2001 From: artiz Date: Sat, 25 Jul 2026 08:25:30 +0000 Subject: [PATCH 2/2] =?UTF-8?q?vlm:=20lock=20DocTags=E2=86=92Markdown=20pa?= =?UTF-8?q?rity=20for=20picture=5Fclassification=20(#153)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit picture_classification scored 1.3% similarity in the live VLM corpus run. Root-causing it deterministically: our DocTags parser reproduces docling's groundtruth Markdown for that fixture byte-for-byte (pictures with captions before the image placeholder, multi-page , reading order), so the divergence is live-model / render variance — which the harness itself flags as expected ("triage as render, not parser") — not a defect on our side. Add an end-to-end regression test through convert_vlm that feeds the model's exact DocTags for the document and asserts the Markdown matches the groundtruth, so our side of the comparison stays provably correct. Image leg, so it needs no pdfium and runs in CI everywhere. Refs #153 Signed-off-by: artiz Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EY5KAiquN4YpVf2PXEQkVT --- crates/docling/tests/vlm.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/docling/tests/vlm.rs b/crates/docling/tests/vlm.rs index 99b74366..e145fe01 100644 --- a/crates/docling/tests/vlm.rs +++ b/crates/docling/tests/vlm.rs @@ -169,6 +169,38 @@ fn vlm_parses_doctags_answers() { assert!(md.contains("| H"), "md: {md:?}"); } +/// Conformance lock for #153: the `picture_classification` corpus fixture +/// scored 1.3% similarity in the live VLM run — a divergence that (per the +/// harness's own note) is render/model variance, *not* a parser defect. Feed +/// the model's exact DocTags for that document through `convert_vlm` and assert +/// the Markdown matches docling's groundtruth byte-for-byte, so our side of the +/// comparison stays provably correct (pictures + captions before the image +/// placeholder, multi-page ``, reading order). Image leg → no +/// pdfium, runs everywhere. +#[test] +fn vlm_doctags_matches_groundtruth_picture_classification() { + let doctags = std::fs::read_to_string( + repo_root().join("tests/data/pdf/groundtruth/picture_classification.doctags.txt"), + ) + .expect("doctags fixture"); + let want = std::fs::read_to_string( + repo_root().join("tests/data/pdf/groundtruth/picture_classification.md"), + ) + .expect("markdown groundtruth"); + // The model emits one DocTags blob per document (root wrapper + internal + // ); the image leg is a single request → single answer. + let (endpoint, served, handle) = mock_openai(vec![doctags]); + let source = SourceDocument::from_bytes("page.png", InputFormat::Image, image_bytes()); + let doc = convert_vlm(&source, &opts(endpoint)).expect("vlm conversion"); + handle.join().expect("mock server"); + assert_eq!(served.load(Ordering::SeqCst), 1); + assert_eq!( + doc.export_to_markdown().trim(), + want.trim(), + "VLM DocTags → Markdown must match docling's groundtruth for picture_classification" + ); +} + #[test] fn vlm_rejects_non_visual_formats() { let source = SourceDocument::from_bytes("x.md", InputFormat::Md, b"# hi".to_vec());