Skip to content

Commit 80d5cb5

Browse files
committed
fix(pdf): drop a code block's language label; exclude test corpora from Linguist
Two follow-ups on the code-block work: 1. A code block's language label (`XML`, `C#`, `JSON`, …) — the little header the docs render above the code — was emitted as its own stray paragraph next to the block (and inconsistently, since the layout sometimes folds it into a wider code box and sometimes not). Add `code_language_labels`: a bare, single-token language identifier sitting just above (or swallowed into the top of) a `code` region is recognized as chrome and consumed, so it isn't emitted twice. Unit tests cover detection, the swallowed-into-code case, and non-labels. 2. Add `.gitattributes` marking the test corpora (`tests/data`, `crates/*/tests/ data`) as linguist-vendored and `tests/snapshots` as linguist-generated, so GitHub's language stats stop reporting the repo as ~87% HTML/LaTeX (those are fixtures, not source) and it reads as the Rust project it is. Conformance holds at 90/91 (only the intended llncsdoc code-block change). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01H3AUFjF71HP2YY9cD32wx8
1 parent 4c67b2a commit 80d5cb5

2 files changed

Lines changed: 171 additions & 6 deletions

File tree

.gitattributes

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# GitHub Linguist: the test corpora and generated snapshots are fixtures, not
2+
# project source. Without this they dwarf the actual code and GitHub reports the
3+
# repo as ~87% HTML / LaTeX. Marking them vendored/generated excludes them from
4+
# the language breakdown so the repo reads as the Rust project it is.
5+
tests/data/** linguist-vendored
6+
tests/snapshots/** linguist-generated
7+
crates/*/tests/data/** linguist-vendored

crates/fleischwolf-pdf/src/assemble.rs

Lines changed: 164 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,122 @@ pub fn resolve(mut regions: Vec<Region>) -> Vec<Region> {
4444
kept
4545
}
4646

47+
/// True for a bare, single-token source-code language label (`XML`, `C#`, `JSON`,
48+
/// `bash`, …) — the little header the docs render above a code block. Matched
49+
/// case-insensitively; anything with whitespace or longer than a token is out.
50+
fn is_code_language(t: &str) -> bool {
51+
let t = t.trim();
52+
if t.is_empty() || t.chars().any(char::is_whitespace) || t.chars().count() > 12 {
53+
return false;
54+
}
55+
const LANGS: &[&str] = &[
56+
"xml",
57+
"html",
58+
"xhtml",
59+
"json",
60+
"jsonc",
61+
"yaml",
62+
"yml",
63+
"toml",
64+
"ini",
65+
"c#",
66+
"csharp",
67+
"f#",
68+
"fsharp",
69+
"vb",
70+
"c",
71+
"c++",
72+
"cpp",
73+
"java",
74+
"kotlin",
75+
"scala",
76+
"go",
77+
"golang",
78+
"rust",
79+
"swift",
80+
"javascript",
81+
"js",
82+
"typescript",
83+
"ts",
84+
"jsx",
85+
"tsx",
86+
"python",
87+
"py",
88+
"ruby",
89+
"rb",
90+
"php",
91+
"perl",
92+
"lua",
93+
"r",
94+
"dart",
95+
"bash",
96+
"sh",
97+
"shell",
98+
"powershell",
99+
"zsh",
100+
"batch",
101+
"cmd",
102+
"sql",
103+
"tsql",
104+
"plsql",
105+
"graphql",
106+
"dockerfile",
107+
"makefile",
108+
"css",
109+
"scss",
110+
"sass",
111+
"less",
112+
"markdown",
113+
"md",
114+
"tex",
115+
"latex",
116+
"diff",
117+
"proto",
118+
"razor",
119+
"cshtml",
120+
"xaml",
121+
"aspx",
122+
"http",
123+
];
124+
let lower = t.to_ascii_lowercase();
125+
LANGS.contains(&lower.as_str())
126+
}
127+
128+
/// Mark the region indices that are a code block's **language label** — a bare
129+
/// `XML`/`C#`/… token sitting directly above a `code` region — so they are consumed
130+
/// rather than emitted as their own stray paragraph/heading. The label may also be
131+
/// captured inside a wider code box (rendered as the fence's first line); dropping
132+
/// the standalone copy just removes the duplicate.
133+
fn code_language_labels(regions: &[Region], cells: &[TextCell]) -> Vec<bool> {
134+
let mut drop = vec![false; regions.len()];
135+
for (i, r) in regions.iter().enumerate() {
136+
if matches!(r.label, "code" | "picture" | "table") {
137+
continue;
138+
}
139+
if !is_code_language(&region_text(r, cells)) {
140+
continue;
141+
}
142+
// The label sits just above the code (a blank line's gap) or is swallowed
143+
// into the top of a wider code box; either way it is that block's label.
144+
// The window is generous because the label's own font is small, so a
145+
// one-line gap is several times its height.
146+
let line_h = (r.b - r.t).abs().max(1.0);
147+
let window = (line_h * 4.0).max(28.0);
148+
let labels_code = regions.iter().enumerate().any(|(j, c)| {
149+
if j == i || c.label != "code" {
150+
return false;
151+
}
152+
let gap = c.t - r.b; // >0 when the code is below the label
153+
let h_overlap = (r.r.min(c.r) - r.l.max(c.l)).max(0.0);
154+
gap > -line_h * 3.0 && gap < window && h_overlap > 0.0
155+
});
156+
if labels_code {
157+
drop[i] = true;
158+
}
159+
}
160+
drop
161+
}
162+
47163
/// Collapse `code` regions where one is nested inside another, keeping the larger.
48164
///
49165
/// RT-DETR sometimes emits a tight code box *and* a wider near-duplicate that also
@@ -821,6 +937,16 @@ pub fn assemble_page(
821937
for ci in code_caption_for.iter().flatten() {
822938
consumed[*ci] = true;
823939
}
940+
// A code block's language label (`XML`, `C#`, …) is chrome, not content — the
941+
// detector emits it as its own region above the code; consume it.
942+
for (i, is_label) in code_language_labels(&regions, &page.cells)
943+
.into_iter()
944+
.enumerate()
945+
{
946+
if is_label {
947+
consumed[i] = true;
948+
}
949+
}
824950

825951
for (i, region) in regions.iter().enumerate() {
826952
if is_skipped(region.label) || consumed[i] {
@@ -1099,11 +1225,9 @@ mod tests {
10991225

11001226
#[test]
11011227
fn resolve_collapses_nested_code_keeping_the_larger_box() {
1102-
// The detector emitted a tight high-score `code` box and a taller
1103-
// lower-score near-duplicate that contains it (the wider one also captures
1104-
// the language label). They must collapse to one — and it must be the
1105-
// *larger* box, so every cell stays covered and nothing leaks out as orphan
1106-
// text.
1228+
// A tight high-score `code` box and a taller lower-score near-duplicate that
1229+
// contains it must collapse to one — the *larger* box, so every cell stays
1230+
// covered and nothing leaks out as orphan text.
11071231
let tight = region("code", 0.95, 78.0, 292.0, 300.0, 330.0);
11081232
let wide = region("code", 0.66, 63.0, 260.0, 320.0, 346.0);
11091233
let kept = super::resolve(vec![tight, wide]);
@@ -1116,7 +1240,7 @@ mod tests {
11161240

11171241
#[test]
11181242
fn resolve_keeps_distinct_and_differently_typed_regions() {
1119-
// A text box fully inside a lower-score table must NOT be collapsed (the
1243+
// A text box fully inside a lower-score *table* must NOT be collapsed (the
11201244
// code dedup is code-only), and two separate code blocks stay separate.
11211245
let text = region("text", 0.95, 90.0, 210.0, 200.0, 230.0);
11221246
let table = region("table", 0.60, 80.0, 200.0, 400.0, 500.0);
@@ -1127,6 +1251,40 @@ mod tests {
11271251
assert_eq!(super::resolve(vec![code_a, code_b]).len(), 2);
11281252
}
11291253

1254+
#[test]
1255+
fn code_language_label_above_code_is_detected() {
1256+
// A bare "XML" token directly above a code box is a language label; a real
1257+
// heading above the same code is not; a language word with no code below is
1258+
// left alone.
1259+
let label = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
1260+
let code = region("code", 0.7, 77.0, 552.0, 290.0, 640.0);
1261+
let heading = region("section_header", 0.9, 76.0, 500.0, 260.0, 512.0);
1262+
let cells = vec![
1263+
cell("XML", 78.0, 541.0, 94.0, 548.0), // inside `label`
1264+
cell("Overview", 78.0, 501.0, 250.0, 511.0), // inside `heading`
1265+
];
1266+
let drop = super::code_language_labels(&[label, code, heading], &cells);
1267+
assert_eq!(drop, vec![true, false, false], "only the label is consumed");
1268+
1269+
// Same label with no code region present → not consumed.
1270+
let label2 = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
1271+
let only = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
1272+
assert_eq!(super::code_language_labels(&[label2], &only), vec![false]);
1273+
1274+
// A label swallowed into the top of a wider code box (negative gap) is still
1275+
// recognized.
1276+
let inside_lbl = region("text", 0.9, 76.0, 540.0, 96.0, 549.0);
1277+
let wide_code = region("code", 0.7, 63.0, 531.0, 320.0, 654.0);
1278+
let cells2 = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
1279+
assert_eq!(
1280+
super::code_language_labels(&[inside_lbl, wide_code], &cells2),
1281+
vec![true, false]
1282+
);
1283+
1284+
assert!(super::is_code_language("XML") && super::is_code_language("c#"));
1285+
assert!(!super::is_code_language("Configure") && !super::is_code_language("XML schema"));
1286+
}
1287+
11301288
#[test]
11311289
fn code_region_text_keeps_lines_and_indentation() {
11321290
// Three source lines; each glyph is 6 units wide (width / chars = 6), so the

0 commit comments

Comments
 (0)