feat(pdf): make the docling-parse line sanitizer the default

Flip the prose reconstruction to dp_lines (the docling-parse sanitizer port) by
default; `DOCLING_LEGACY_LINES` restores the old gap-heuristic path. Routed
through a single `use_dp_lines()` helper (pdfium_backend), used by page_cells and
by assemble's cell-join + clean_text branches.

PDF conformance default 3/14 → **4/14** (multi_page now byte-exact), with the rest
markedly closer: 2206 200→168, redp5110 300→243, 2305v1 110→64, normal_4pages
108→82, 2203 309→263, table_mislabeled 113→102. The 3 previously-exact PDFs stay
exact. DOCX/HTML/etc. are unaffected (the sanitizer is PDF-only). Updated the
clean_text unit test for the preserve-spacing default.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: artiz

crates/fleischwolf-pdf/src/assemble.rs

@@ -122,7 +122,7 @@ fn clean_text(text: &str) -> String {
         .replace(['\u{201c}', '\u{201d}'], "\"") // “ ” → "
         .replace(['\u{2013}', '\u{2014}'], "-") // – — → -
         .replace('\u{2026}', "..."); // … → ...
-    let out = if std::env::var("DOCLING_DP_LINES").is_ok() {
+    let out = if crate::pdfium_backend::use_dp_lines() {
         // The docling-parse sanitizer already placed the correct spacing (e.g.
         // justified double spaces); preserve internal runs of spaces, only
         // normalizing line breaks/tabs and trimming the ends.
@@ -226,7 +226,7 @@ fn region_text(region: &Region, cells: &[TextCell]) -> String {
     // label and its value, `LABEL` | `: value` → `LABEL : value`. The legacy
     // reconstruction instead joins same-band cells with a space only across a real
     // gap, because it can split a word into abutting segments (`الت`|`ي` → `التي`).
-    let dp = std::env::var("DOCLING_DP_LINES").is_ok();
+    let dp = crate::pdfium_backend::use_dp_lines();
     let mut joined = String::new();
     let mut prev: Option<&&TextCell> = None;
     for c in &inside {
@@ -578,7 +578,8 @@ mod tests {
             "Graph's \"x\""
         );
         assert_eq!(clean_text("a\u{2026}"), "a...");
-        // Whitespace collapses; a normal space-join is preserved.
-        assert_eq!(clean_text("a   b\nc"), "a b c");
+        // The dp default (the docling-parse sanitizer) preserves internal spacing
+        // it placed deliberately; line breaks/tabs normalize to a space, ends trim.
+        assert_eq!(clean_text("a   b\nc"), "a   b c");
     }
 }

crates/fleischwolf-pdf/src/pdfium_backend.rs

@@ -55,6 +55,13 @@ pub struct PdfDocument {
 /// Bind to the pdfium dynamic library. Honors `PDFIUM_DYNAMIC_LIB_PATH` (a
 /// directory or file), else the directory of the current exe, else the system
 /// library — mirroring how a deployment ships `libpdfium` alongside the binary.
+/// Whether to use the docling-parse line sanitizer ([`crate::dp_lines`]) for prose
+/// reconstruction — the default. Set `DOCLING_LEGACY_LINES` to fall back to the
+/// older gap-heuristic `lines_from_glyphs`.
+pub(crate) fn use_dp_lines() -> bool {
+    std::env::var("DOCLING_LEGACY_LINES").is_err()
+}
+
 fn bind() -> Result<Pdfium, PdfiumError> {
     if let Ok(path) = std::env::var("PDFIUM_DYNAMIC_LIB_PATH") {
         let name = Pdfium::pdfium_platform_library_name_at_path(&path);
@@ -222,7 +229,7 @@ impl<'a> FfiText<'a> {
         let out = if tp.is_null() {
             empty()
         } else {
-            let dp = std::env::var("DOCLING_DP_LINES").is_ok();
+            let dp = use_dp_lines();
             let g = glyphs(b, tp, dp);
             b.FPDFText_ClosePage(tp);
             // Prose line cells: the docling-parse-style sanitizer (behind a flag