Skip to content

Commit ea78a2e

Browse files
artizclaude
andcommitted
feat(pdf): normalize U+2212 minus + merge hyphen-ending paragraph fragments
- Map U+2212 (minus sign) to ASCII `-` in clean_text alongside the en/em dashes (author-ID `[0000 −0002]` → `[0000 -0002]`). - merge_continuations now also joins a fragment ending in a wrap hyphen/dash with a lowercase continuation (`vocab-` + `ulary` → `vocab- ulary`), matching docling. 2305v1 44→40; the 4 exact PDFs unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent e262439 commit ea78a2e

1 file changed

Lines changed: 6 additions & 6 deletions

File tree

crates/fleischwolf-pdf/src/assemble.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ fn clean_text(text: &str) -> String {
157157
.replace(['\u{2}', '\u{ad}'], "") // any stray wrap hyphens not at a join
158158
.replace(['\u{2018}', '\u{2019}'], "'") // ‘ ’ → '
159159
.replace(['\u{201c}', '\u{201d}'], "\"") // “ ” → "
160-
.replace(['\u{2013}', '\u{2014}'], "-") // – — → -
160+
.replace(['\u{2013}', '\u{2014}', '\u{2212}'], "-") // – — → -
161161
.replace('\u{2026}', "..."); // … → ...
162162
let out = if crate::pdfium_backend::use_dp_lines() {
163163
// The docling-parse sanitizer already placed the correct spacing (e.g.
@@ -613,11 +613,11 @@ pub(crate) fn merge_continuations(nodes: &mut Vec<Node>) {
613613
while i + 1 < nodes.len() {
614614
let merged = match (&nodes[i], &nodes[i + 1]) {
615615
(Node::Paragraph { text: a }, Node::Paragraph { text: b }) => {
616-
let a_open = a
617-
.trim_end()
618-
.chars()
619-
.next_back()
620-
.is_some_and(|c| c.is_alphabetic());
616+
// Open if the fragment ends mid-word (a letter) or with a wrap
617+
// hyphen/dash — docling joins `vocab-` + `ulary` → `vocab- ulary`.
618+
let a_open = a.trim_end().chars().next_back().is_some_and(|c| {
619+
c.is_alphabetic() || matches!(c, '-' | '\u{2010}' | '\u{2013}' | '\u{2014}')
620+
});
621621
let b_cont = b
622622
.trim_start()
623623
.chars()

0 commit comments

Comments
 (0)