From f76e085c99b479f743603d3d1f7e9a7a79960bc5 Mon Sep 17 00:00:00 2001 From: jonwiggins Date: Sun, 19 Jul 2026 22:22:32 -0600 Subject: [PATCH] feat: group stacked redaction leaks into one recoverable block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redaction tools draw one black box per line, so a poorly-redacted paragraph surfaced as a dozen separate findings with its recovered text split across all of them. checkPage now merges leak findings that stack into a single block (same check, overlapping columns, vertical gap up to ~1.25 line-heights so double-spaced filings like the Manafort response merge correctly). The merged finding spans the whole block and carries the recovered text joined in reading order, so a hidden passage reads back in full and the overlay frames the whole thing. Also updates the real-world regression set: - add the 2005 Calipari/Sgrena US Army report (a textbook black-box-over- live-text failure; grade F, hundreds of recovered names/details); - reclassify the TSA SOP fixture from failure to control — the fetchable cryptome copy is a reveal edition (once-hidden text shown openly with a red outline, nothing actually covered), so grade A is correct and it now guards that exact false positive; - rebaseline Manafort/JPMorgan leak counts for the per-passage merge. Adds tests/merge-findings.test.ts and gitignores scan working dirs and downloaded releases so they can never be committed. --- .gitignore | 7 ++ README.md | 11 ++- examples/real-world/README.md | 34 +++++---- examples/real-world/manifest.json | 37 +++++++--- src/lib/analyzer/checks.ts | 112 ++++++++++++++++++++++++++++- tests/merge-findings.test.ts | 115 ++++++++++++++++++++++++++++++ 6 files changed, 289 insertions(+), 27 deletions(-) create mode 100644 tests/merge-findings.test.ts diff --git a/.gitignore b/.gitignore index f6f1df4..bb748ea 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,10 @@ yarn-error.log* # real-world example PDFs are downloaded on demand, never committed # (only examples/real-world/{manifest.json,fetch.mjs,README.md} are tracked) /examples/real-world/*.pdf + +# batch-scan working dirs and any downloaded document releases. These routinely +# contain the very content a failed redaction exposed, so they never get +# committed — see scripts/unredact-scan/scan-release.mjs. +/scans/ +/election-integrity/ +/dni-scan/ diff --git a/README.md b/README.md index 80b2883..48ede13 100644 --- a/README.md +++ b/README.md @@ -118,12 +118,17 @@ redaction failures** and runs them through the shipping analyzer. The current corpus and a results table live in [`examples/real-world/`](./examples/real-world/README.md): +Leak counts are per **passage**: consecutive per-line redaction boxes are merged +into one block finding, so a hidden paragraph is a single recoverable leak (with +the full text), not one leak per line. + | Document | Year | Our grade | | --- | --- | --- | -| TSA Screening Management SOP | 2009 | **F** (superficial ×25 + metadata + bookmark + word-size) | -| Manafort response (*US v. Manafort*) | 2019 | **F** (superficial ×32, incl. the p.5 polling-data passage) | -| USVI v. JPMorgan, Exhibit 1 (DoJ Epstein release) | 2022 | **F** (superficial ×71, incl. p.41, + metadata) | +| Calipari / Sgrena AR 15-6 report (US Army) | 2005 | **F** (superficial; hundreds of names and details recovered from under black boxes) | +| Manafort response (*US v. Manafort*) | 2019 | **F** (superficial; the p.5 polling-data passage recovered as one block) | +| USVI v. JPMorgan, Exhibit 1 (DoJ Epstein release) | 2022 | **F** (superficial across many pages, incl. p.41) | | EU–AstraZeneca contract *(true-negative control)* | 2021 | **A** (correctly clean) | +| TSA Screening Management SOP *(true-negative control)* | 2009 | **A** (this cryptome copy is a reveal edition — once-hidden text is shown openly, outlined in red; nothing is actually covered) | These PDFs are **never committed**: they contain the very content their failed redactions exposed, and storing improperly-disclosed third-party data in the diff --git a/examples/real-world/README.md b/examples/real-world/README.md index 2842292..9996d98 100644 --- a/examples/real-world/README.md +++ b/examples/real-world/README.md @@ -1,7 +1,7 @@ # Real-world redaction failures A private regression set of **famous, publicly-documented** redaction failures -(plus one true-negative control), run through the shipping analyzer to see what +(plus true-negative controls), run through the shipping analyzer to see what it catches. This is the real-world counterpart to the synthetic fixtures in [`../`](../README.md). @@ -20,25 +20,28 @@ it catches. This is the real-world counterpart to the synthetic fixtures in Results below are from the actual `extract → checks → grade` pipeline (via `tests/real-examples.test.ts`), not a reimplementation. "Recovered" = characters -lifted out from under black boxes. +lifted out from under black boxes. Leak counts are **per passage**: consecutive +per-line redaction boxes are merged into one block finding, so a hidden paragraph +counts once (and reads back in full), not once per line. | Document | Year | The documented failure | Our grade | Checks fired | Recovered | Detected? | | --- | --- | --- | --- | --- | --- | --- | -| **TSA Screening Management SOP** | 2009 | 93-page manual posted to FedBizOpps with black boxes over live text; copy-paste revealed screening thresholds | **F** | superficial-mark ×25, word-size | ~15,500 chars across 14+ pages | ✅ | -| **Manafort response** (*US v. Manafort*) | 2019 | Defense drew black boxes over the Kilimnik / 2016-campaign-polling-data passage (p.5); reporters copy-pasted it | **F** | superficial-mark ×32 (incl. p.5) | ~2,300 chars | ✅ | -| **USVI v. JPMorgan, Exhibit 1** (DoJ Epstein release) | 2022 | Failed redactions the PDF Association flagged on p.41; marks recoverable across the exhibit | **F** | superficial-mark ×71 (incl. p.41) | ~2,600 chars | ✅ | +| **Calipari / Sgrena AR 15-6 report** (US Army) | 2005 | Army investigation into the Baghdad checkpoint shooting posted with black boxes over classified text that stayed selectable; recovered within hours | **F** | superficial-mark across 12+ pages | hundreds of names/details | ✅ | +| **Manafort response** (*US v. Manafort*) | 2019 | Defense drew black boxes over the Kilimnik / 2016-campaign-polling-data passage (p.5); reporters copy-pasted it | **F** | superficial-mark (p.5 passage as one block) | ~1,975 chars over 4 pages | ✅ | +| **USVI v. JPMorgan, Exhibit 1** (DoJ Epstein release) | 2022 | Failed redactions the PDF Association flagged on p.41; marks recoverable across the exhibit | **F** | superficial-mark across 11 pages (incl. p.41) | ~2,400 chars | ✅ | | **EU–AstraZeneca APA** *(control)* | 2021 | The infamous copy leaked redacted section titles via PDF bookmarks — but it was online for only hours and was never archived | **A** | none | — | ✅ correctly clean | +| **TSA Screening Management SOP** *(control)* | 2009 | This cryptome copy is a *reveal* edition: the once-hidden text is printed openly with a red outline around each passage — nothing is actually covered | **A** | none | — | ✅ correctly clean | -Three famous failures, all graded **F** with text recovered from under the -marks; one famous case kept as a true-negative. +Three famous failures, all graded **F** with text recovered from under the marks; +two famous cases kept as true-negatives. ### Notes on the cases -- **TSA SOP** is the textbook government redaction failure (Schneier, Wired, The - Register, 2009). One real document exercises both of our content checks at - once: superficial marks and a width-only `word-size` leak. (The original also - leaked Title metadata and a PDF bookmark; those document-level checks are - currently disabled.) +- **Calipari / Sgrena report** is the textbook *government* redaction failure and + the oldest case here (2005): the U.S. Army posted its AR 15-6 investigation with + black boxes over classified passages, but the text stayed selectable and Italian + press published the recovered content within hours. Hundreds of inline + single-name/phrase redactions, so it exercises `superficial-mark` at volume. - **Manafort** is the textbook *court-filing* failure — the famous January 2019 pleading whose blacked-out text journalists recovered by copy-paste. We grab the forensically-confirmed copy (DocumentCloud `5677512`); the version on the @@ -54,6 +57,13 @@ marks; one famous case kept as a true-negative. is the **corrected reissue**. We keep it because a real, famous, *properly* redacted government contract is a strong false-positive guard — and our analyzer correctly grades it **A**. +- **TSA SOP** is the textbook government redaction failure (Schneier, Wired, The + Register, 2009) — but the fetchable cryptome copy is a *reveal* edition: the + once-hidden passages are printed as normal, fully-visible text, each wrapped in + a red **outline** box (a clip path plus a red stroke, no dark fill). Nothing is + covered, so **A** is correct. We keep it as a false-positive guard: the analyzer + must not flag visible text just because a drawn box surrounds it — the exact + case the clip-path handling in `extract.ts` exists to get right. Sources for each document (and the SHA-256 we pin) are in [`manifest.json`](./manifest.json). diff --git a/examples/real-world/manifest.json b/examples/real-world/manifest.json index 4dcccef..014a5a8 100644 --- a/examples/real-world/manifest.json +++ b/examples/real-world/manifest.json @@ -3,19 +3,19 @@ "files": [ { "file": "tsa-screening-sop-2008.pdf", - "kind": "failure", + "kind": "control", "title": "TSA Screening Management Standard Operating Procedures (rev. 3, May 2008)", "case": "Transportation Security Administration; posted to FedBizOpps.gov, 2009", "year": 2009, "source": "https://cryptome.org/2013/01/tsa-screening-mgmt-sop.pdf", "reportedBy": "Schneier on Security; The Register; Wired — the textbook 'black box over live text' government leak", - "failure": "93-page manual where redacted passages were covered with black rectangles but never removed; copy-paste reveals screening thresholds and procedures.", + "failure": "TRUE-NEGATIVE CONTROL. The famous 2009 leak was black boxes over live text. This cryptome copy, however, is a REVEAL edition: the once-hidden passages are printed as normal, fully-visible black text with a red OUTLINE rectangle drawn around each (a clip path + red stroke, no dark fill). Nothing is hidden here, so an A is correct — the passage is openly readable. Kept as a regression guard: the analyzer must NOT flag visible text merely because a drawn box surrounds it (the exact false positive the clip-path/endPath handling in extract.ts prevents). It is NOT a hidden-text fixture; genuine 'black box over live text' cases are covered by manafort and jpmorgan.", "sha256": "9ca6e9fcf134593bc1259d358776ec69a3605c68a00ab40e92e3929539a21331", "expect": { - "grade": "F", - "checks": ["superficial-mark"], - "minLeaks": 20, - "leakOnPages": [9, 38, 44] + "grade": "A", + "checks": [], + "minLeaks": 0, + "leakOnPages": [] } }, { @@ -26,15 +26,32 @@ "year": 2019, "source": "https://assets.documentcloud.org/documents/5677512/Manafort-20190108-Dc.pdf", "reportedBy": "The Guardian; Vice Motherboard — reporters copy-pasted the blacked-out passage", - "failure": "Defense drew black boxes over the passage describing Manafort sharing 2016 campaign polling data with Konstantin Kilimnik (page 5); the text underneath was never removed.", + "failure": "Defense drew black boxes over the passage describing Manafort sharing 2016 campaign polling data with Konstantin Kilimnik (page 5); the text underneath was never removed. The per-line boxes are merged into whole-paragraph blocks, so the leak count reflects passages (a handful), not lines (~two dozen).", "sha256": "7e21c26404618d30b768caf55dfeff5e7b31c651595c55d41a91f54c8852ece0", "expect": { "grade": "F", "checks": ["superficial-mark"], - "minLeaks": 18, + "minLeaks": 3, "leakOnPages": [5] } }, + { + "file": "calipari-army-report-2005.pdf", + "kind": "failure", + "title": "AR 15-6 Investigation — 4 March 2005 Baghdad checkpoint shooting (Calipari / Sgrena)", + "case": "U.S. Army / Multi-National Force–Iraq; released 2005-04-30", + "year": 2005, + "source": "http://web.archive.org/web/20250217140752id_/https://win.senzabavaglio.info/documenti/calipari/classified.pdf", + "reportedBy": "Corriere della Sera & La Repubblica (1-2 May 2005); a textbook 'how not to redact' case — see Wikisource 'Calipari Report'", + "failure": "The Army investigation into the checkpoint shooting of Italian intelligence officer Nicola Calipari was posted with black boxes drawn over classified passages, but the text underneath was never removed — names (e.g. 'Andrea Carpani'), unit details, and rules of engagement were selectable and copy-pasteable within hours. Hundreds of inline single-name/phrase redactions, so leaks stay per-box (they don't merge into paragraph blocks).", + "sha256": "dccf6b57d0a5b1f94c488b7888712324995c190d20827b9a24b5ac9b3beb5135", + "expect": { + "grade": "F", + "checks": ["superficial-mark"], + "minLeaks": 100, + "leakOnPages": [1, 5] + } + }, { "file": "jpmorgan-complaint-exhibit-1.pdf", "kind": "failure", @@ -44,12 +61,12 @@ "source": "https://storage.courtlistener.com/recap/gov.uscourts.nysd.591653/gov.uscourts.nysd.591653.1.1.pdf", "canonical": "https://www.justice.gov/multimedia/Court Records/Government of the United States Virgin Islands v. JPMorgan Chase Bank, N.A., No. 122-cv-10904 (S.D.N.Y. 2022)/001-01.pdf", "reportedBy": "PDF Association — 'A case study in PDF forensics: The Epstein PDFs' (flags defective redaction on page 41)", - "failure": "DoJ-released court exhibit with failed redactions; the PDF Association flagged page 41, and copy-paste recovers text under marks on many pages.", + "failure": "DoJ-released court exhibit with failed redactions; the PDF Association flagged page 41, and copy-paste recovers text under marks on many pages. Per-line boxes are merged into paragraph blocks, so the leak count is per-passage.", "sha256": "9ff93e1d34ee0fbb8690225da6606ebd806d5f638d7d4d1dc7b26009e7cf48f5", "expect": { "grade": "F", "checks": ["superficial-mark"], - "minLeaks": 50, + "minLeaks": 25, "leakOnPages": [41] } }, diff --git a/src/lib/analyzer/checks.ts b/src/lib/analyzer/checks.ts index 4cca244..bc47a4c 100644 --- a/src/lib/analyzer/checks.ts +++ b/src/lib/analyzer/checks.ts @@ -5,7 +5,14 @@ import { MAX_WORD_SIZE_CHARS, MIN_COVERAGE, } from './constants' -import {coverage, height, intersection, sameLine, width} from './geometry' +import { + coverage, + height, + intersection, + sameLine, + union, + width, +} from './geometry' /** * pdf.js merges adjacent glyphs into long line-runs, so a redaction box usually @@ -233,6 +240,107 @@ function isVisibleLabelRun( return covered / width(clip) >= LABEL_RUN_COVERAGE } +/** + * A poorly-redacted paragraph is almost never one big rectangle — redaction + * tools draw one black box PER LINE, so the per-cover loop above emits one leak + * finding per line. Left as-is, a single hidden paragraph becomes a dozen cards + * and its recovered text is split across all of them, so a reader can't read the + * passage back in one piece (this is exactly what the Manafort filing does). + * + * Merge leak findings that stack into one text block: same check, columns that + * overlap horizontally, and a vertical gap no larger than about a line. The + * merged finding spans the whole block (so the "show in document" overlay frames + * the passage) and carries the recovered text joined top-to-bottom, left-to- + * right. Non-leak findings (word-size length hints, un-applied marks) are each + * their own thing and pass through untouched. + */ +export function mergeAdjacentLeaks(findings: Finding[]): Finding[] { + const order = new Map() + findings.forEach((f, i) => order.set(f, i)) + + const leaks = findings.filter((f) => f.severity === 'leak' && f.bbox) + if (leaks.length < 2) return findings + + type Group = {items: Finding[]; bbox: BBox} + const groups: Group[] = [] + + // Two boxes belong to the same block when their columns overlap and the lower + // one starts within about one line-pitch of the block's current bottom. The + // horizontal test is against the NARROWER box so a short last line still + // attaches to a wide block. The vertical gap may be slightly negative (boxes + // that touch or overlap); the ceiling of ~1.25 line-heights admits the next + // line of DOUBLE-spaced text (where consecutive line-boxes sit a full box- + // height apart, as in the Manafort filing) while still excluding a real + // paragraph break, whose blank line puts the next box ~2 pitches down. + const continues = (block: BBox, box: BBox): boolean => { + const overlapX = Math.min(block[2], box[2]) - Math.max(block[0], box[0]) + const minW = Math.min(width(block), width(box)) + if (minW <= 0 || overlapX < minW * 0.35) return false + const lineH = Math.min(height(block), height(box)) || height(box) + const gap = box[1] - block[3] + return gap <= lineH * 1.25 + } + + // Top-to-bottom, then left-to-right — so each group accumulates in reading + // order and its recovered text reads back in the right sequence. + const sorted = [...leaks].sort( + (a, b) => a.bbox![1] - b.bbox![1] || a.bbox![0] - b.bbox![0] + ) + for (const f of sorted) { + const box = f.bbox as BBox + let best: Group | null = null + for (const g of groups) { + if (g.items[0]!.check !== f.check) continue + if (!continues(g.bbox, box)) continue + // Prefer the closest open group (largest bottom edge above this box). + if (!best || g.bbox[3] > best.bbox[3]) best = g + } + if (best) { + best.items.push(f) + best.bbox = union(best.bbox, box) + } else { + groups.push({items: [f], bbox: [...box] as BBox}) + } + } + + const merged: Array<{finding: Finding; order: number}> = [] + for (const g of groups) { + const rank = Math.min(...g.items.map((it) => order.get(it) ?? 0)) + const first = g.items[0]! + if (g.items.length === 1) { + merged.push({finding: first, order: rank}) + continue + } + const text = g.items + .map((it) => it.recovered?.text?.trim() ?? '') + .filter(Boolean) + .join(' ') + .replace(/\s+/g, ' ') + .trim() + const runCount = g.items.reduce( + (n, it) => n + (it.recovered?.runCount ?? 0), + 0 + ) + merged.push({ + finding: { + ...first, + bbox: g.bbox, + confidence: Math.max(...g.items.map((it) => it.confidence)), + recovered: text ? {text, runCount} : first.recovered, + }, + order: rank, + }) + } + + // Re-attach the untouched non-leak findings and restore document order. + for (const f of findings) { + if (f.severity === 'leak' && f.bbox) continue + merged.push({finding: f, order: order.get(f) ?? 0}) + } + merged.sort((a, b) => a.order - b.order) + return merged.map((m) => m.finding) +} + /** Run the per-page covers/text checks (1, 2, 3, 5). */ export function checkPage(extract: PageExtract, deadline?: number): Finding[] { const findings: Finding[] = [] @@ -340,5 +448,5 @@ export function checkPage(extract: PageExtract, deadline?: number): Finding[] { } } - return findings + return mergeAdjacentLeaks(findings) } diff --git a/tests/merge-findings.test.ts b/tests/merge-findings.test.ts new file mode 100644 index 0000000..87e0448 --- /dev/null +++ b/tests/merge-findings.test.ts @@ -0,0 +1,115 @@ +import {describe, it, expect} from 'vitest' +import {mergeAdjacentLeaks} from '@/lib/analyzer/checks' +import type {BBox, CheckId, Finding, Severity} from '@/lib/analyzer/types' + +/** + * Unit coverage for the block-merge that collapses per-line redaction findings + * into one readable passage. The real-world Manafort case exercises this end to + * end, but that PDF is fetched on demand and skipped in CI, so these synthetic + * findings pin the geometry (double-spacing, columns, paragraph gaps) directly. + */ + +let seq = 0 +function leak( + bbox: BBox, + text: string, + check: CheckId = 'superficial-mark' +): Finding { + return { + id: `f-${seq++}`, + check, + page: 1, + bbox, + severity: 'leak', + confidence: 0.8, + coverKind: 'path-fill', + title: 'Text is still under the box', + detail: '', + recovered: {text, runCount: 1}, + } +} + +function other(bbox: BBox, severity: Severity = 'warning'): Finding { + return { + id: `w-${seq++}`, + check: 'word-size', + page: 1, + bbox, + severity, + confidence: 0.6, + coverKind: 'path-fill', + title: 'This redaction leaks its length', + detail: '', + } +} + +describe('mergeAdjacentLeaks', () => { + it('merges consecutive double-spaced lines into one passage', () => { + // 14pt-tall boxes stacked a full box-height apart (double-spaced), same + // left column — the Manafort shape. Expect a single merged block. + const merged = mergeAdjacentLeaks([ + leak([72, 265, 540, 279], 'the first line of the hidden'), + leak([72, 293, 540, 307], 'passage continues onto the second'), + leak([72, 320, 302, 334], 'and ends here.'), + ]) + expect(merged).toHaveLength(1) + expect(merged[0]!.recovered?.text).toBe( + 'the first line of the hidden passage continues onto the second and ends here.' + ) + // Bbox spans the whole block so the overlay frames the passage. + expect(merged[0]!.bbox).toEqual([72, 265, 540, 334]) + }) + + it('keeps a first-line indent attached to the block', () => { + const merged = mergeAdjacentLeaks([ + leak([208, 265, 540, 279], 'indented opening'), + leak([72, 293, 540, 307], 'flush continuation'), + ]) + expect(merged).toHaveLength(1) + expect(merged[0]!.recovered?.text).toBe( + 'indented opening flush continuation' + ) + }) + + it('does not merge across a real paragraph break (~2 line gap)', () => { + const merged = mergeAdjacentLeaks([ + leak([72, 265, 540, 279], 'end of paragraph one'), + leak([72, 320, 540, 334], 'start of paragraph two'), + ]) + expect(merged).toHaveLength(2) + }) + + it('does not merge two columns on the same line', () => { + const merged = mergeAdjacentLeaks([ + leak([72, 265, 250, 279], 'left column'), + leak([320, 265, 500, 279], 'right column'), + ]) + expect(merged).toHaveLength(2) + }) + + it('does not merge findings from different checks', () => { + const merged = mergeAdjacentLeaks([ + leak([72, 265, 540, 279], 'vector fill line', 'superficial-mark'), + leak([72, 293, 540, 307], 'ocr line', 'ocr-layer-leak'), + ]) + expect(merged).toHaveLength(2) + }) + + it('leaves non-leak findings untouched and preserves order', () => { + const input = [ + leak([72, 265, 540, 279], 'hidden line one'), + leak([72, 293, 540, 307], 'hidden line two'), + other([72, 400, 200, 414]), + ] + const merged = mergeAdjacentLeaks(input) + expect(merged).toHaveLength(2) + expect(merged[0]!.severity).toBe('leak') + expect(merged[0]!.recovered?.text).toBe('hidden line one hidden line two') + expect(merged[1]!.severity).toBe('warning') + }) + + it('is a no-op for a single finding', () => { + const input = [leak([72, 265, 540, 279], 'lonely leak')] + expect(mergeAdjacentLeaks(input)).toEqual(input) + }) +})