Skip to content

Commit 11f1fe0

Browse files
artizclaude
andcommitted
feat(pdf): ordered lists + wrap dehyphenation — multi_page EXACT via dp path
- Ordered list items: detect an `N.` enumeration marker at the start of a list item and render it as an ordered item (`N. text`) instead of `- N. text`. This is un-gated, helping the whole corpus (2305v1 114→64, 2203 285→263, table_mislabeled 111→102, redp5110 252→243). - Wrap dehyphenation (dp path): a line cell ending in a hyphen/dash followed by a lowercase continuation joins without the dash or a space (`platforms—` + `reflects` → `platformsreflects`). The dash is still raw at join time (clean_text normalizes em/en dashes later), so match -, ‐, –, — all. With DOCLING_DP_LINES: multi_page 54→**EXACT**, taking the dp path to **4/14** (vs legacy 3/14). Default path unchanged. Debug dumps now show loose box + font. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent b60a5c5 commit 11f1fe0

2 files changed

Lines changed: 57 additions & 12 deletions

File tree

crates/fleischwolf-pdf/examples/dump_regions.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ fn main() {
4343
// raw line cells in extraction order (to inspect RTL ordering)
4444
if std::env::var("DUMP_CELLS").is_ok() {
4545
for (ci, c) in page.cells.iter().enumerate() {
46-
let snip: String = c.text.chars().take(50).collect();
46+
let snip: String = c.text.chars().take(300).collect();
4747
println!(
4848
" CELL[{ci}] t={:6.1} l={:6.1} r={:6.1} | {}",
4949
c.t, c.l, c.r, snip

crates/fleischwolf-pdf/src/assemble.rs

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,19 @@ fn order_regions<T>(items: &mut [T], page_w: f32, reg: impl Fn(&T) -> &Region) {
100100
/// tighten punctuation spacing: docling preserves the PDF's own spaces (it keeps
101101
/// `{ ahn }`, `Name 1 .`, `[ 9 ]`), and a geometric gap heuristic diverges from
102102
/// it more than a plain single-space join does.
103+
/// An ordered-list enumeration marker at the start of a list item: leading ASCII
104+
/// digits followed by `.`, e.g. `1. Undo/Redo` → `(1, "Undo/Redo")`. Returns
105+
/// `None` when the text doesn't start with `digits.`.
106+
fn parse_ordered_marker(s: &str) -> Option<(u64, String)> {
107+
let digits: String = s.chars().take_while(|c| c.is_ascii_digit()).collect();
108+
if digits.is_empty() {
109+
return None;
110+
}
111+
let rest = s[digits.len()..].strip_prefix('.')?;
112+
let number = digits.parse().ok()?;
113+
Some((number, rest.trim_start().to_string()))
114+
}
115+
103116
fn clean_text(text: &str) -> String {
104117
let replaced = text
105118
.replace("\u{2} ", "")
@@ -217,15 +230,33 @@ fn region_text(region: &Region, cells: &[TextCell]) -> String {
217230
let mut joined = String::new();
218231
let mut prev: Option<&&TextCell> = None;
219232
for c in &inside {
233+
let t = c.text.trim();
220234
if let Some(p) = prev {
221235
let same_band = ((p.t / band).round() as i64) == ((c.t / band).round() as i64);
222236
let h = (c.b - c.t).abs().max((p.b - p.t).abs()).max(1.0);
223237
let gap = if rtl { p.l - c.r } else { c.l - p.r };
224-
if dp || !same_band || gap > h * 0.25 {
238+
// Dehyphenate a wrapped word: a line ending in a hyphen/dash followed
239+
// by a lowercase continuation joins without the dash or a space
240+
// (`platforms—` + `reflects` → `platformsreflects`). The dash is still
241+
// raw here (clean_text normalizes em/en dashes later), so match them all.
242+
let ends_dash = matches!(
243+
joined.chars().last(),
244+
Some('-' | '\u{2010}' | '\u{2013}' | '\u{2014}')
245+
);
246+
let dehyph = dp
247+
&& ends_dash
248+
&& joined
249+
.chars()
250+
.nth_back(1)
251+
.is_some_and(|c| c.is_alphabetic())
252+
&& t.chars().next().is_some_and(|c| c.is_lowercase());
253+
if dehyph {
254+
joined.pop();
255+
} else if dp || !same_band || gap > h * 0.25 {
225256
joined.push(' ');
226257
}
227258
}
228-
joined.push_str(c.text.trim());
259+
joined.push_str(t);
229260
prev = Some(c);
230261
}
231262
clean_text(&joined)
@@ -455,17 +486,31 @@ pub fn assemble_page(
455486
// `##` (it never emits a top-level `#` for PDFs), so match that.
456487
"title" | "section_header" => doc.push(Node::Heading { level: 2, text }),
457488
// docling drops the rendered bullet glyph; the Markdown serializer
458-
// adds its own `- ` marker.
459-
"list_item" => doc.push(Node::ListItem {
460-
ordered: false,
461-
number: 0,
462-
first_in_list: false,
463-
text: text
489+
// adds its own `- ` marker. An item whose text opens with an `N.`
490+
// enumeration marker is an ordered item (rendered `N. text`).
491+
"list_item" => {
492+
let stripped = text
464493
.trim_start_matches(['•', '◦', '▪', '·', '*', '-'])
465494
.trim_start()
466-
.to_string(),
467-
level: 0,
468-
}),
495+
.to_string();
496+
if let Some((number, rest)) = parse_ordered_marker(&stripped) {
497+
doc.push(Node::ListItem {
498+
ordered: true,
499+
number,
500+
first_in_list: false,
501+
text: rest,
502+
level: 0,
503+
});
504+
} else {
505+
doc.push(Node::ListItem {
506+
ordered: false,
507+
number: 0,
508+
first_in_list: false,
509+
text: stripped,
510+
level: 0,
511+
});
512+
}
513+
}
469514
// TableFormer structure (cells + spans, text matched from word cells)
470515
// when available; otherwise geometric grid reconstruction; finally a
471516
// single cell.

0 commit comments

Comments
 (0)