Skip to content

Commit 4c67b2a

Browse files
committed
fix(pdf): narrow duplicate-code fix to a code-only nested-box dedup
The previous approach (dropping a box that contains a same-label kept box in resolve()) was too broad: it discarded legitimate container regions across many fixtures, whose cells then leaked back out as orphan text (PDF snapshot conformance fell to 69/91). Revert that resolve() change. Replace it with `dedup_nested_code`, a post-resolve pass restricted to `code` regions: when one code box is mostly inside a strictly larger code box, drop the inner and keep the larger. Keeping the larger box (rather than dropping it) means every cell stays covered, so nothing leaks as orphan text; restricting to `code` leaves all other region kinds — and the 22 non-code fixtures — untouched. Unit tests cover the nested-code collapse (larger box kept) and that distinct / differently-typed nested regions are preserved. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01H3AUFjF71HP2YY9cD32wx8
1 parent 42e506d commit 4c67b2a

1 file changed

Lines changed: 61 additions & 31 deletions

File tree

crates/fleischwolf-pdf/src/assemble.rs

Lines changed: 61 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -33,24 +33,51 @@ pub fn resolve(mut regions: Vec<Region>) -> Vec<Region> {
3333
let covered = kept.iter().any(|k| {
3434
let i = inter(&r, k.l, k.t, k.r, k.b);
3535
let ka = area(k.l, k.t, k.r, k.b).max(1.0);
36-
// Drop r if most of it sits inside an already-kept box, or the two
37-
// strongly overlap (IoU). Also drop r when it *contains* a kept box of
38-
// the same label: the detector sometimes emits a taller near-duplicate
39-
// of a block (e.g. a code box that also swallows the `XML` language
40-
// label) with a slightly lower score than the tight box, so the tight
41-
// one is kept first and the container — not "mostly inside" anything —
42-
// would otherwise survive and emit the block a second time. Restricting
43-
// the containment drop to the same label keeps genuinely nested,
44-
// differently-typed regions (a text box inside a table) intact.
45-
i / ra > 0.7 || i / (ra + ka - i) > 0.5 || (r.label == k.label && i / ka > 0.7)
36+
// drop if most of r is inside k, or they strongly mutually overlap
37+
i / ra > 0.7 || i / (ra + ka - i) > 0.5
4638
});
4739
if !covered {
4840
kept.push(r);
4941
}
5042
}
43+
dedup_nested_code(&mut kept);
5144
kept
5245
}
5346

47+
/// Collapse `code` regions where one is nested inside another, keeping the larger.
48+
///
49+
/// RT-DETR sometimes emits a tight code box *and* a wider near-duplicate that also
50+
/// captures the block's language label (`XML`, `C#`, …). When the tight box scores
51+
/// higher it is kept first, and the wider container — not "mostly inside" the tight
52+
/// box — survives [`resolve`]'s greedy pass, so the block is emitted twice. Keeping
53+
/// the **larger** box (rather than dropping it) collapses the pair without leaking
54+
/// the container's extra cells back out as orphan text, since the larger box still
55+
/// covers every cell. Restricted to `code` so genuinely distinct nested regions of
56+
/// other kinds are untouched.
57+
fn dedup_nested_code(kept: &mut Vec<Region>) {
58+
let mut drop = vec![false; kept.len()];
59+
for i in 0..kept.len() {
60+
if kept[i].label != "code" {
61+
continue;
62+
}
63+
let ai = area(kept[i].l, kept[i].t, kept[i].r, kept[i].b).max(1.0);
64+
for j in 0..kept.len() {
65+
if i == j || drop[j] || kept[j].label != "code" {
66+
continue;
67+
}
68+
let aj = area(kept[j].l, kept[j].t, kept[j].r, kept[j].b).max(1.0);
69+
// Drop i when it is mostly inside a strictly larger code box j.
70+
let overlap = inter(&kept[i], kept[j].l, kept[j].t, kept[j].r, kept[j].b);
71+
if aj > ai && overlap / ai > 0.7 {
72+
drop[i] = true;
73+
break;
74+
}
75+
}
76+
}
77+
let mut keep = drop.iter();
78+
kept.retain(|_| !*keep.next().unwrap());
79+
}
80+
5481
/// Append `text` regions for cells the layout left uncovered ("orphan cells"),
5582
/// the way docling's `LayoutPostprocessor` does (`create_orphan_clusters`): any
5683
/// non-empty cell that no kept region covers (>50% of the cell's area) becomes a
@@ -1071,30 +1098,33 @@ mod tests {
10711098
}
10721099

10731100
#[test]
1074-
fn resolve_collapses_same_label_nested_duplicates() {
1075-
// The detector emitted three overlapping `code` boxes for one block: a
1076-
// tight high-score box and two taller lower-score near-duplicates that
1077-
// contain it. All must collapse to one so the block isn't emitted twice.
1078-
let regions = vec![
1079-
region("code", 0.70, 78.0, 252.0, 300.0, 346.0), // tall container
1080-
region("code", 0.95, 78.0, 292.0, 300.0, 330.0), // tight, highest score
1081-
region("code", 0.80, 78.0, 260.0, 300.0, 332.0), // medium container
1082-
];
1083-
let kept = super::resolve(regions);
1084-
assert_eq!(kept.len(), 1, "nested same-label code boxes must collapse");
1085-
assert!((kept[0].score - 0.95).abs() < 1e-6, "the tight box wins");
1101+
fn resolve_collapses_nested_code_keeping_the_larger_box() {
1102+
// The detector emitted a tight high-score `code` box and a taller
1103+
// lower-score near-duplicate that contains it (the wider one also captures
1104+
// the language label). They must collapse to one — and it must be the
1105+
// *larger* box, so every cell stays covered and nothing leaks out as orphan
1106+
// text.
1107+
let tight = region("code", 0.95, 78.0, 292.0, 300.0, 330.0);
1108+
let wide = region("code", 0.66, 63.0, 260.0, 320.0, 346.0);
1109+
let kept = super::resolve(vec![tight, wide]);
1110+
assert_eq!(kept.len(), 1, "nested code boxes must collapse to one");
1111+
assert!(
1112+
kept[0].l == 63.0 && kept[0].b == 346.0,
1113+
"the larger containing box is kept"
1114+
);
10861115
}
10871116

10881117
#[test]
1089-
fn resolve_keeps_differently_typed_nested_regions() {
1090-
// A text box fully inside a lower-score table must NOT drop the table:
1091-
// the containment drop is same-label only.
1092-
let regions = vec![
1093-
region("text", 0.95, 90.0, 210.0, 200.0, 230.0), // small, high score
1094-
region("table", 0.60, 80.0, 200.0, 400.0, 500.0), // big table around it
1095-
];
1096-
let kept = super::resolve(regions);
1097-
assert_eq!(kept.len(), 2, "table containing a text box is preserved");
1118+
fn resolve_keeps_distinct_and_differently_typed_regions() {
1119+
// A text box fully inside a lower-score table must NOT be collapsed (the
1120+
// code dedup is code-only), and two separate code blocks stay separate.
1121+
let text = region("text", 0.95, 90.0, 210.0, 200.0, 230.0);
1122+
let table = region("table", 0.60, 80.0, 200.0, 400.0, 500.0);
1123+
assert_eq!(super::resolve(vec![text, table]).len(), 2);
1124+
1125+
let code_a = region("code", 0.9, 78.0, 100.0, 300.0, 140.0);
1126+
let code_b = region("code", 0.9, 78.0, 300.0, 300.0, 360.0); // far below, no overlap
1127+
assert_eq!(super::resolve(vec![code_a, code_b]).len(), 2);
10981128
}
10991129

11001130
#[test]

0 commit comments

Comments
 (0)