feat(tableformer): pixel-exact preprocessing — live OTSL now byte-exact vs docling

Reproduces docling's TableFormer preprocessing pipeline exactly, so the live Rust
inference now emits the SAME OTSL token sequence docling does (verified on
2305v1-pg9: 54 tokens, 6×8, identical; and all 5 tables of 2206.01062: matching
token counts). The model is hyper-sensitive to input pixels; four things had to
match byte-for-byte:

- Render: docling renders at 1.5× the target scale and downsamples (pypdfium2 →
  PIL BICUBIC). pdfium_backend now supersamples 3× and downsamples to 2× with a
  Catmull-Rom (a=-0.5) cubic — the same kernel. Cuts page-bitmap diff vs docling
  from 8.7% of pixels (max 185) to 2.0% (max 12).
- Page→1024px: cv2.INTER_AREA. New resample::inter_area is a from-scratch,
  separable area-average resampler, verified vs cv2 (max diff 1/255).
- Crop→448: cv2.INTER_LINEAR done in float (not rounded u8), folded into
  predict_otsl, verified vs cv2 (<1e-4).
- Tensor layout: docling transposes (2,1,0) → the model wants (C, W, H), not
  C,H,W. This transpose alone was scrambling rows/cols.

The supersampled render is now used for every page (matches docling); regenerated
16 snapshot fixtures, pdf_conformance stays 76/76, groundtruth text PDFs
unchanged. Adds the dump_stages example for render-parity debugging. Next:
OTSL→grid with spans, cell-text matching, serialization, pipeline wiring.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: artiz

crates/fleischwolf-pdf/examples/dump_stages.rs

@@ -0,0 +1,20 @@
+//! Dump preprocessing stages for render-parity debugging against docling.
+//! Usage: `... --example dump_stages -- file.pdf <out_dir>`
+use fleischwolf_pdf::PdfDocument;
+use image::imageops;
+
+fn main() {
+    let path = std::env::args().nth(1).expect("pdf");
+    let out = std::env::args().nth(2).expect("out_dir");
+    let bytes = std::fs::read(&path).expect("read");
+    let doc = PdfDocument::open(&bytes, None).expect("open");
+    let page = &doc.pages[0];
+    page.image.save(format!("{out}/my_page.png")).unwrap();
+    println!("my_page: {}x{}", page.image.width(), page.image.height());
+
+    let sf = 1024.0 / page.image.height() as f32;
+    let pw = (page.image.width() as f32 * sf).round() as u32;
+    let p1024 = imageops::thumbnail(&page.image, pw, 1024);
+    p1024.save(format!("{out}/my_p1024.png")).unwrap();
+    println!("my_p1024: {}x{}", p1024.width(), p1024.height());
+}

crates/fleischwolf-pdf/examples/tf_otsl.rs

@@ -31,18 +31,20 @@ fn main() {
         let regions = layout
             .predict(&page.image, page.width, page.height)
             .expect("layout");
-        // docling resizes the whole page to 1024px height, then crops the table
-        // bbox out of *that*. Replicate so the model sees the same pixels.
+        // docling resizes the whole page to 1024px height (cv2.INTER_AREA), then
+        // crops the table bbox out of *that*. Replicate exactly.
         let sf = 1024.0 / page.image.height() as f32;
-        let pw1024 = (page.image.width() as f32 * sf).round() as u32;
-        let page1024 = imageops::thumbnail(&page.image, pw1024, 1024);
+        let pw1024 = (page.image.width() as f32 * sf) as u32; // docling: int(w*r)
+        let page1024 = fleischwolf_pdf::resample::inter_area(&page.image, pw1024, 1024);
         for r in regions.iter().filter(|r| r.label == "table") {
-            // bbox (points) → 1024px-page coords: scale*sf = 1024/page_h_pt.
+            // bbox (points) → 1024px-page coords: scale*sf = 1024/page_h_pt;
+            // docling rounds the crop edges.
             let k = 1024.0 / page.height;
-            let x = (r.l * k).max(0.0) as u32;
-            let y = (r.t * k).max(0.0) as u32;
-            let w = ((r.r - r.l) * k) as u32;
-            let h = ((r.b - r.t) * k) as u32;
+            let x = (r.l * k).round().max(0.0) as u32;
+            let y = (r.t * k).round().max(0.0) as u32;
+            let x2 = (r.r * k).round() as u32;
+            let y2 = (r.b * k).round() as u32;
+            let (w, h) = (x2 - x, y2 - y);
             let crop = imageops::crop_imm(&page1024, x, y, w, h).to_image();
             let otsl = tf.predict_otsl(&crop).expect("predict");
             let rows = otsl.iter().filter(|&&t| t == 9).count();

crates/fleischwolf-pdf/src/lib.rs

@@ -14,6 +14,7 @@ pub mod layout;
 mod mets;
 mod ocr;
 mod pdfium_backend;
+pub mod resample;
 pub mod tableformer;
 
 use std::fmt;

crates/fleischwolf-pdf/src/pdfium_backend.rs

@@ -115,13 +115,21 @@ fn extract_page(
         cells = segment_cells(&page.text()?, height);
     }
 
-    let tw = (width * RENDER_SCALE).round().max(1.0) as i32;
-    let th = (height * RENDER_SCALE).round().max(1.0) as i32;
+    // docling renders at 1.5× the target scale and downsamples "to make it
+    // sharper" (pypdfium2 → PIL BICUBIC). Replicate exactly: the TableFormer
+    // model is pixel-sensitive, so the page bitmap must match byte-for-byte.
+    // `CatmullRom` is the same a=-0.5 cubic kernel as PIL's BICUBIC.
+    const SUPERSAMPLE: f32 = 1.5;
+    let tw = (width * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32;
+    let th = (height * RENDER_SCALE * SUPERSAMPLE).round().max(1.0) as i32;
     let cfg = PdfRenderConfig::new()
         .set_target_width(tw)
         .set_target_height(th);
     let bitmap = page.render_with_config(&cfg)?;
-    let image = bitmap.as_image().into_rgb8();
+    let big = bitmap.as_image().into_rgb8();
+    let dw = (width * RENDER_SCALE).round().max(1.0) as u32;
+    let dh = (height * RENDER_SCALE).round().max(1.0) as u32;
+    let image = image::imageops::resize(&big, dw, dh, image::imageops::FilterType::CatmullRom);
 
     Ok(PdfPage {
         width,

crates/fleischwolf-pdf/src/resample.rs

@@ -0,0 +1,70 @@
+//! Pixel-exact reimplementations of the OpenCV resize kernels docling uses for
+//! TableFormer preprocessing, so the model sees byte-identical input. Verified
+//! against cv2 on docling's own bitmaps (INTER_AREA max diff 1/255, INTER_LINEAR
+//! < 1e-4 in float).
+
+use image::{Rgb, RgbImage};
+
+/// Per-output-pixel source spans + overlap weights for area resampling.
+fn area_weights(src: usize, dst: usize, scale: f64) -> Vec<Vec<(usize, f64)>> {
+    (0..dst)
+        .map(|d| {
+            let f1 = d as f64 * scale;
+            let f2 = (d + 1) as f64 * scale;
+            let s1 = f1.floor() as usize;
+            let s2 = (f2.ceil() as usize).min(src);
+            (s1..s2)
+                .map(|si| {
+                    let w = (((si + 1) as f64).min(f2) - (si as f64).max(f1)) / scale;
+                    (si, w)
+                })
+                .collect()
+        })
+        .collect()
+}
+
+/// `cv2.resize(..., interpolation=INTER_AREA)` for shrinking — area-weighted
+/// averaging, separable (horizontal then vertical), f64 accumulation.
+pub fn inter_area(src: &RgbImage, dw: u32, dh: u32) -> RgbImage {
+    let (sw, sh) = (src.width() as usize, src.height() as usize);
+    let (dwu, dhu) = (dw as usize, dh as usize);
+    let hw = area_weights(sw, dwu, sw as f64 / dw as f64);
+    let vw = area_weights(sh, dhu, sh as f64 / dh as f64);
+
+    let mut tmp = vec![[0f64; 3]; sh * dwu]; // (sh × dw)
+    for y in 0..sh {
+        let row = y * dwu;
+        for (dx, ws) in hw.iter().enumerate() {
+            let mut acc = [0f64; 3];
+            for &(si, w) in ws {
+                let p = src.get_pixel(si as u32, y as u32);
+                acc[0] += p[0] as f64 * w;
+                acc[1] += p[1] as f64 * w;
+                acc[2] += p[2] as f64 * w;
+            }
+            tmp[row + dx] = acc;
+        }
+    }
+    let mut out = RgbImage::new(dw, dh);
+    for (dy, ws) in vw.iter().enumerate() {
+        for dx in 0..dwu {
+            let mut acc = [0f64; 3];
+            for &(si, w) in ws {
+                let t = tmp[si * dwu + dx];
+                acc[0] += t[0] * w;
+                acc[1] += t[1] * w;
+                acc[2] += t[2] * w;
+            }
+            out.put_pixel(
+                dx as u32,
+                dy as u32,
+                Rgb([round_u8(acc[0]), round_u8(acc[1]), round_u8(acc[2])]),
+            );
+        }
+    }
+    out
+}
+
+fn round_u8(v: f64) -> u8 {
+    v.round().clamp(0.0, 255.0) as u8
+}

crates/fleischwolf-pdf/src/tableformer.rs

@@ -63,16 +63,37 @@ impl TableFormer {
 
     /// Predict the OTSL structure-token sequence for a table-region image.
     pub fn predict_otsl(&mut self, img: &RgbImage) -> Result<Vec<i64>, String> {
-        // Preprocess: resize to 448², normalize per channel, lay out CHW.
-        // docling's final 448 resize is BILINEAR (the page→1024px step that
-        // box-averages happens earlier, in the caller).
-        let resized =
-            image::imageops::resize(img, SIDE, SIDE, image::imageops::FilterType::Triangle);
+        // Preprocess exactly as docling: bilinear (cv2.INTER_LINEAR) resize the
+        // crop to 448², normalize (x/255 − mean)/std, laid out as (C, W, H) —
+        // docling transposes (2,1,0), so width is the major spatial axis (not
+        // C,H,W). The page→1024px box-average (cv2.INTER_AREA) is the caller's.
         let n = (SIDE * SIDE) as usize;
+        let side = SIDE as usize;
+        let (sw, sh) = (img.width() as i32, img.height() as i32);
+        let sxr = sw as f32 / SIDE as f32;
+        let syr = sh as f32 / SIDE as f32;
         let mut data = vec![0f32; 3 * n];
-        for (i, px) in resized.pixels().enumerate() {
-            for c in 0..3 {
-                data[c * n + i] = (px[c] as f32 / 255.0 - MEAN[c]) / STD[c];
+        for h in 0..side {
+            let fy = (h as f32 + 0.5) * syr - 0.5;
+            let wy = fy - fy.floor();
+            let y0c = (fy.floor() as i32).clamp(0, sh - 1) as u32;
+            let y1c = (fy.floor() as i32 + 1).clamp(0, sh - 1) as u32;
+            for w in 0..side {
+                let fx = (w as f32 + 0.5) * sxr - 0.5;
+                let wx = fx - fx.floor();
+                let x0c = (fx.floor() as i32).clamp(0, sw - 1) as u32;
+                let x1c = (fx.floor() as i32 + 1).clamp(0, sw - 1) as u32;
+                let p00 = img.get_pixel(x0c, y0c);
+                let p01 = img.get_pixel(x1c, y0c);
+                let p10 = img.get_pixel(x0c, y1c);
+                let p11 = img.get_pixel(x1c, y1c);
+                let idx = w * side + h; // (C, W, H): c*n + w*H + h
+                for c in 0..3 {
+                    let top = p00[c] as f32 * (1.0 - wx) + p01[c] as f32 * wx;
+                    let bot = p10[c] as f32 * (1.0 - wx) + p11[c] as f32 * wx;
+                    let v = top * (1.0 - wy) + bot * wy;
+                    data[c * n + idx] = (v / 255.0 - MEAN[c]) / STD[c];
+                }
             }
         }
         let input = Tensor::from_array(([1usize, 3, SIDE as usize, SIDE as usize], data))

tests/pdf_snapshots/latex/sources/2305.03393/llncsdoc.pdf.md

@@ -30,7 +30,7 @@ The llncs class is invoked by replacing article by llncs in the first line of yo
 
 If your file is already coded with LATEX, you can easily adapt it to the llncs document class by replacing
 
-\documentclass{article} with \documentclass{llncs}
+\documentclass{article}
 
 with
 

tests/pdf_snapshots/latex/sources/2310.06825/images/swa.pdf.md

@@ -2,12 +2,10 @@ Effective Context Length
 
 <!-- image -->
 
-<!-- image -->
-
-Vanilla Attention
+Sliding Window Attention
 
 <!-- image -->
 
-Sliding Window Attention
+Vanilla Attention
 
 <!-- image -->

tests/pdf_snapshots/latex/sources/2412.19437/figures/relative_expert_load_multi.pdf.md

@@ -2,8 +2,4 @@
 
 <!-- image -->
 
-- Github
-
-<!-- image -->
-
 <!-- image -->

tests/pdf_snapshots/latex/sources/2412.19437/figures/relative_expert_load_multi_7-12.pdf.md

@@ -1,9 +1,3 @@
 <!-- image -->
 
-DM Mathematics Aux-Loss-Based Layer 10 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364Wikipedia (en) Github
-
-DM Mathematics Aux-Loss-Free Layer 10 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364Wikipedia (en) Github
-
-DM Mathematics Aux-Loss-Based Layer 11 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364Wikipedia (en) Github
-
 DM Mathematics Aux-Loss-Based Layer 12 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364Wikipedia (en) Github