|
| 1 | +//! Port of docling-parse's line-cell sanitizer |
| 2 | +//! (`src/parse/page_item_sanitators/cells.h` → `create_line_cells` / |
| 3 | +//! `contract_cells_into_lines_v1`). It merges per-glyph char cells into line |
| 4 | +//! cells via a 3-pass contraction — left-to-right, right-to-left, then |
| 5 | +//! left-to-right with reverse — using corner-distance adjacency and inserting at |
| 6 | +//! most one space per merge. This reproduces docling-parse's inter-word spacing |
| 7 | +//! (justified double spaces, the space before a `:`, and RTL ordering) that the |
| 8 | +//! ad-hoc `lines_from_glyphs` reconstruction can't. |
| 9 | +//! |
| 10 | +//! Geometry uses native PDF coordinates (y increases upward); each cell carries |
| 11 | +//! its four transformed corners r0=bottom-left, r1=bottom-right, r2=top-right, |
| 12 | +//! r3=top-left, exactly like `page_cell.h`. |
| 13 | +
|
| 14 | +use crate::pdfium_backend::{Glyph, TextCell}; |
| 15 | + |
| 16 | +// config.h: the factors that actually bind for line cells. |
| 17 | +const MERGE: f64 = 1.0; // line_space_width_factor_for_merge (adjacency gate) |
| 18 | +const MERGE_WITH_SPACE: f64 = 0.33; // line_space_width_factor_for_merge_with_space |
| 19 | +const H_TOL: f64 = 1.0; // horizontal_cell_tolerance (ligature eps_d1 relaxation) |
| 20 | + |
| 21 | +#[derive(Clone)] |
| 22 | +struct Cell { |
| 23 | + text: String, |
| 24 | + rx0: f64, |
| 25 | + ry0: f64, // bottom-left |
| 26 | + rx1: f64, |
| 27 | + ry1: f64, // bottom-right |
| 28 | + rx2: f64, |
| 29 | + ry2: f64, // top-right |
| 30 | + rx3: f64, |
| 31 | + ry3: f64, // top-left |
| 32 | + ltr: bool, |
| 33 | + active: bool, |
| 34 | + lig_carry: bool, // last_merged_cell_was_ligature |
| 35 | + font: u64, // hash of the PDF font name+flags (for enforce_same_font) |
| 36 | +} |
| 37 | + |
| 38 | +impl Cell { |
| 39 | + /// Length of the bottom edge (baseline advance) — `page_cell.h::length`. |
| 40 | + fn length(&self) -> f64 { |
| 41 | + ((self.rx1 - self.rx0).powi(2) + (self.ry1 - self.ry0).powi(2)).sqrt() |
| 42 | + } |
| 43 | + |
| 44 | + /// Running mean glyph advance over the whole accumulated cell. |
| 45 | + fn avg_char_width(&self) -> f64 { |
| 46 | + let n = self.text.chars().count(); |
| 47 | + if n > 0 { |
| 48 | + self.length() / n as f64 |
| 49 | + } else { |
| 50 | + 0.0 |
| 51 | + } |
| 52 | + } |
| 53 | + |
| 54 | + /// Distance from this cell's bottom-right corner to `other`'s bottom-left. |
| 55 | + fn gap(&self, other: &Cell) -> f64 { |
| 56 | + ((self.rx1 - other.rx0).powi(2) + (self.ry1 - other.ry0).powi(2)).sqrt() |
| 57 | + } |
| 58 | + |
| 59 | + /// `is_adjacent_to`: both the bottom-corner gap (`< eps0`) and the top-corner |
| 60 | + /// gap (`< eps1`) must be small. The vertical component keeps different |
| 61 | + /// baselines/lines from merging. |
| 62 | + fn adjacent(&self, other: &Cell, eps0: f64, eps1: f64) -> bool { |
| 63 | + let d0 = self.gap(other); |
| 64 | + let d1 = ((self.rx2 - other.rx3).powi(2) + (self.ry2 - other.ry3).powi(2)).sqrt(); |
| 65 | + d0 < eps0 && d1 < eps1 |
| 66 | + } |
| 67 | + |
| 68 | + /// Punctuation/space cells are bidi-neutral bridges. |
| 69 | + fn same_orientation(&self, other: &Cell) -> bool { |
| 70 | + self.ltr == other.ltr || is_punct_or_space(&self.text) || is_punct_or_space(&other.text) |
| 71 | + } |
| 72 | + |
| 73 | + /// `merge_with`: absorb `other` (which lies to this cell's right). Insert at |
| 74 | + /// most one separator space when the gap exceeds `delta`. RTL prepends. |
| 75 | + fn merge_with(&mut self, other: &Cell, delta: f64) { |
| 76 | + let d0 = self.gap(other); |
| 77 | + if !self.ltr || !other.ltr { |
| 78 | + if delta < d0 { |
| 79 | + self.text.insert(0, ' '); |
| 80 | + } |
| 81 | + self.text = format!("{}{}", other.text, self.text); |
| 82 | + self.ltr = false; |
| 83 | + } else { |
| 84 | + if delta < d0 { |
| 85 | + self.text.push(' '); |
| 86 | + } |
| 87 | + self.text.push_str(&other.text); |
| 88 | + self.ltr = true; |
| 89 | + } |
| 90 | + // Extend the right edge to `other`. |
| 91 | + self.rx1 = other.rx1; |
| 92 | + self.ry1 = other.ry1; |
| 93 | + self.rx2 = other.rx2; |
| 94 | + self.ry2 = other.ry2; |
| 95 | + } |
| 96 | +} |
| 97 | + |
| 98 | +/// `applicable_for_merge`: both active, same font (ligatures bridge fonts), and |
| 99 | +/// same reading orientation. |
| 100 | +fn applicable(a: &Cell, b: &Cell) -> bool { |
| 101 | + if !a.active || !b.active { |
| 102 | + return false; |
| 103 | + } |
| 104 | + if a.font != b.font && !is_ligature(&a.text) && !is_ligature(&b.text) { |
| 105 | + return false; |
| 106 | + } |
| 107 | + a.same_orientation(b) |
| 108 | +} |
| 109 | + |
| 110 | +/// Left-to-right pass: `i` ascending accumulates cells to its right. |
| 111 | +fn pass_ltr(cells: &mut [Cell], allow_reverse: bool) { |
| 112 | + for i in 0..cells.len() { |
| 113 | + if !cells[i].active { |
| 114 | + continue; |
| 115 | + } |
| 116 | + let mut j = i + 1; |
| 117 | + while j < cells.len() { |
| 118 | + if !applicable(&cells[i], &cells[j]) { |
| 119 | + break; |
| 120 | + } |
| 121 | + let i_lig = is_ligature(&cells[i].text) || cells[i].lig_carry; |
| 122 | + let j_lig = is_ligature(&cells[j].text) || cells[j].lig_carry; |
| 123 | + let d0 = cells[i].avg_char_width() * MERGE; |
| 124 | + let d1 = cells[i].avg_char_width() * MERGE_WITH_SPACE; |
| 125 | + let adj_d1 = d0 + if i_lig || j_lig { H_TOL } else { 0.0 }; |
| 126 | + if cells[i].adjacent(&cells[j], d0, adj_d1) { |
| 127 | + let other = cells[j].clone(); |
| 128 | + cells[i].merge_with(&other, d1); |
| 129 | + cells[i].lig_carry = is_ligature(&other.text); |
| 130 | + cells[j].active = false; |
| 131 | + j += 1; // i keeps absorbing the next cell to its right |
| 132 | + } else if allow_reverse && cells[j].adjacent(&cells[i], d0, adj_d1) { |
| 133 | + let other = cells[i].clone(); |
| 134 | + cells[j].merge_with(&other, d1); |
| 135 | + cells[j].lig_carry = is_ligature(&other.text); |
| 136 | + cells[i].active = false; |
| 137 | + break; // i is consumed |
| 138 | + } else { |
| 139 | + break; |
| 140 | + } |
| 141 | + } |
| 142 | + } |
| 143 | +} |
| 144 | + |
| 145 | +/// Right-to-left pass: `i` descending; its immediate left neighbour `i-1` |
| 146 | +/// absorbs it (then the outer loop continues leftward through the absorber). |
| 147 | +fn pass_rtl(cells: &mut [Cell]) { |
| 148 | + let n = cells.len(); |
| 149 | + for k in 0..n { |
| 150 | + let i = n - 1 - k; |
| 151 | + if !cells[i].active || i == 0 { |
| 152 | + continue; |
| 153 | + } |
| 154 | + let j = i - 1; |
| 155 | + if !applicable(&cells[i], &cells[j]) { |
| 156 | + continue; |
| 157 | + } |
| 158 | + let i_lig = is_ligature(&cells[i].text) || cells[i].lig_carry; |
| 159 | + let j_lig = is_ligature(&cells[j].text) || cells[j].lig_carry; |
| 160 | + let d0 = cells[i].avg_char_width() * MERGE; |
| 161 | + let d1 = cells[i].avg_char_width() * MERGE_WITH_SPACE; |
| 162 | + let adj_d1 = d0 + if i_lig || j_lig { H_TOL } else { 0.0 }; |
| 163 | + if cells[j].adjacent(&cells[i], d0, adj_d1) { |
| 164 | + let other = cells[i].clone(); |
| 165 | + cells[j].merge_with(&other, d1); |
| 166 | + cells[j].lig_carry = is_ligature(&other.text); |
| 167 | + cells[i].active = false; |
| 168 | + } |
| 169 | + } |
| 170 | +} |
| 171 | + |
| 172 | +fn contract(cells: &mut Vec<Cell>) { |
| 173 | + pass_ltr(cells, false); |
| 174 | + cells.retain(|c| c.active); |
| 175 | + pass_rtl(cells); |
| 176 | + cells.retain(|c| c.active); |
| 177 | + pass_ltr(cells, true); |
| 178 | + cells.retain(|c| c.active); |
| 179 | +} |
| 180 | + |
| 181 | +/// Build line cells from a page's glyph stream via the docling-parse contraction. |
| 182 | +pub(crate) fn line_cells(glyphs: &[Glyph], page_h: f32) -> Vec<TextCell> { |
| 183 | + let mut cells: Vec<Cell> = glyphs |
| 184 | + .iter() |
| 185 | + .filter_map(|g| { |
| 186 | + // Use the loose box (uniform font ascent/descent + advance) so adjacent |
| 187 | + // glyphs share a top edge, matching docling-parse's `compute_rect`. |
| 188 | + if !g.ll.is_finite() { |
| 189 | + return None; // a space pdfium gave no box for; can't place it |
| 190 | + } |
| 191 | + let text = g.ch.to_string(); |
| 192 | + let ltr = !is_right_to_left(&text); |
| 193 | + Some(Cell { |
| 194 | + text, |
| 195 | + rx0: g.ll as f64, |
| 196 | + ry0: g.lb as f64, |
| 197 | + rx1: g.lr as f64, |
| 198 | + ry1: g.lb as f64, |
| 199 | + rx2: g.lr as f64, |
| 200 | + ry2: g.lt as f64, |
| 201 | + rx3: g.ll as f64, |
| 202 | + ry3: g.lt as f64, |
| 203 | + ltr, |
| 204 | + active: true, |
| 205 | + lig_carry: false, |
| 206 | + font: g.font, |
| 207 | + }) |
| 208 | + }) |
| 209 | + .collect(); |
| 210 | + contract(&mut cells); |
| 211 | + cells |
| 212 | + .into_iter() |
| 213 | + .map(|c| { |
| 214 | + let l = c.rx0.min(c.rx1).min(c.rx2).min(c.rx3) as f32; |
| 215 | + let r = c.rx0.max(c.rx1).max(c.rx2).max(c.rx3) as f32; |
| 216 | + let top = c.ry0.max(c.ry1).max(c.ry2).max(c.ry3) as f32; |
| 217 | + let bot = c.ry0.min(c.ry1).min(c.ry2).min(c.ry3) as f32; |
| 218 | + TextCell { |
| 219 | + text: c.text, |
| 220 | + l, |
| 221 | + t: page_h - top, |
| 222 | + r, |
| 223 | + b: page_h - bot, |
| 224 | + } |
| 225 | + }) |
| 226 | + .collect() |
| 227 | +} |
| 228 | + |
| 229 | +fn is_rtl_char(c: char) -> bool { |
| 230 | + let ch = c as u32; |
| 231 | + (0x0600..=0x06FF).contains(&ch) |
| 232 | + || (0x0750..=0x077F).contains(&ch) |
| 233 | + || (0x08A0..=0x08FF).contains(&ch) |
| 234 | + || (0xFB50..=0xFDFF).contains(&ch) |
| 235 | + || (0xFE70..=0xFEFF).contains(&ch) |
| 236 | + || (0x0590..=0x05FF).contains(&ch) |
| 237 | + || (0xFB1D..=0xFB4F).contains(&ch) |
| 238 | + || (0x0700..=0x074F).contains(&ch) |
| 239 | + || (0x0780..=0x07BF).contains(&ch) |
| 240 | + || (0x07C0..=0x07FF).contains(&ch) |
| 241 | +} |
| 242 | + |
| 243 | +/// All codepoints are RTL-script (matches `string.h::is_right_to_left`). |
| 244 | +fn is_right_to_left(s: &str) -> bool { |
| 245 | + !s.is_empty() && s.chars().all(is_rtl_char) |
| 246 | +} |
| 247 | + |
| 248 | +/// A single-codepoint punctuation/space cell (matches `string.h`). |
| 249 | +fn is_punct_or_space(s: &str) -> bool { |
| 250 | + let mut chars = s.chars(); |
| 251 | + let (Some(c), None) = (chars.next(), chars.next()) else { |
| 252 | + return false; |
| 253 | + }; |
| 254 | + if matches!( |
| 255 | + c, |
| 256 | + ' ' | '\t' |
| 257 | + | '\n' |
| 258 | + | '\r' |
| 259 | + | '\u{0c}' |
| 260 | + | '\u{0b}' |
| 261 | + | '.' |
| 262 | + | ',' |
| 263 | + | ';' |
| 264 | + | ':' |
| 265 | + | '!' |
| 266 | + | '?' |
| 267 | + | '(' |
| 268 | + | ')' |
| 269 | + | '[' |
| 270 | + | ']' |
| 271 | + | '{' |
| 272 | + | '}' |
| 273 | + | '\'' |
| 274 | + | '"' |
| 275 | + | '`' |
| 276 | + | '\u{2018}' |
| 277 | + | '\u{2019}' |
| 278 | + | '\u{201c}' |
| 279 | + | '\u{201d}' |
| 280 | + | '-' |
| 281 | + | '\u{2013}' |
| 282 | + | '\u{2014}' |
| 283 | + | '_' |
| 284 | + | '/' |
| 285 | + | '\\' |
| 286 | + | '|' |
| 287 | + | '@' |
| 288 | + | '#' |
| 289 | + | '%' |
| 290 | + | '&' |
| 291 | + | '*' |
| 292 | + | '+' |
| 293 | + | '=' |
| 294 | + | '<' |
| 295 | + | '>' |
| 296 | + ) { |
| 297 | + return true; |
| 298 | + } |
| 299 | + let ch = c as u32; |
| 300 | + (0x2000..=0x206F).contains(&ch) |
| 301 | + || (0x3000..=0x303F).contains(&ch) |
| 302 | + || (0xFE50..=0xFE6F).contains(&ch) |
| 303 | + || (0xFF00..=0xFF0F).contains(&ch) |
| 304 | + || (0xFF1A..=0xFF1F).contains(&ch) |
| 305 | + || (0xFF3B..=0xFF5E).contains(&ch) |
| 306 | +} |
| 307 | + |
| 308 | +/// Ligature glyph or its ASCII spelling (matches `string.h::is_ligature`). |
| 309 | +fn is_ligature(s: &str) -> bool { |
| 310 | + matches!(s, "ff" | "fi" | "fl" | "ffi" | "ffl") |
| 311 | + || s.chars().any(|c| (0xFB00..=0xFB06).contains(&(c as u32))) |
| 312 | +} |
0 commit comments