From a08567681beced0b01d523caedb8233530acf84e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20G=C3=A4dicke?= Date: Thu, 14 May 2026 21:17:34 +0200 Subject: [PATCH] feat(report): add text extraction health check --- README.md | 14 +- .../page_006/quality_report.md | 13 ++ .../page_012/quality_report.md | 13 ++ src/pdf_quality_report/checks.py | 70 +++++++++- src/pdf_quality_report/interpret.py | 55 ++++++++ src/pdf_quality_report/report.py | 3 + tests/test_interpret.py | 34 +++++ tests/test_quality_report.py | 122 +++++++++++++++++- 8 files changed, 318 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e07bb2f..0cb5103 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,19 @@ The report answers three practical questions: 2. Can each block be traced back to a source page and bounding box? 3. Does the page contain layout noise that a human should review before using the output? -The five current checks are: +The six current checks are: 1. Required field coverage: are core fields such as `id`, `type`, `page_number`, `bbox`, and `reading_order_index` present? 2. Provenance completeness: can each block be traced back to Docling metadata and the source PDF? 3. BBox sanity: are bounding boxes present, finite, and ordered correctly? 4. Content vs noise ratio: how much looks like document content compared with headers, footers, images, or other secondary blocks? 5. Text usefulness: are text blocks empty, suspiciously short, or duplicated? +6. Text extraction health: is extracted text available in the normalized blocks for review, without judging whether the text is correct? + +Text usefulness flags local text-quality issues such as empty body blocks, very short text, or repeated fragments. +Text extraction health summarizes whether the document contains enough extracted text overall for review. + +Warning-only checks can move the report to `REVIEW`; the report CLI still exits non-zero only for hard failures. It also lists diagnostic noise/layout signals such as table-marker artifacts, headers/footers, and ambiguous image blocks. These signals are for review only; they do not add hard failures or warnings by themselves. @@ -139,9 +145,9 @@ The tests use small synthetic Docling-style JSON fixtures, not the committed MDP ## Example Report Output Excerpt -The excerpt below comes from page 12 of the included MDPI research-paper example. A block is one extracted -page element, such as text, a header, an image, or a caption. Later check sections and block-level warning -details are not shown. +The simplified excerpt below comes from page 12 of the included MDPI research-paper example. A block is one +extracted page element, such as text, a header, an image, or a caption. Later check sections and block-level +warning details are not shown. ```markdown ## Summary diff --git a/examples/mdpi_pdf_elements/page_006/quality_report.md b/examples/mdpi_pdf_elements/page_006/quality_report.md index ada90a2..d6f86ba 100644 --- a/examples/mdpi_pdf_elements/page_006/quality_report.md +++ b/examples/mdpi_pdf_elements/page_006/quality_report.md @@ -33,6 +33,7 @@ Each section below is a separate check. - BBox Sanity checks whether bounding boxes look structurally valid. - Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise. - Text Usefulness flags very short or repeated text fragments. +- Text Extraction Health checks extracted-text availability, not extracted-text correctness. ## Required Field Coverage PASS @@ -64,6 +65,18 @@ PASS text usefulness checks passed +## Text Extraction Health +PASS + +extracted text availability checks passed + +- text_bearing_blocks=8 +- non_empty_text_blocks=8 +- empty_text_blocks=0 +- total_text_chars=5040 +- image_blocks=0 +- empty_text_block_ratio=0.000 + ## Recommended Review Actions Because the decision is REVIEW, inspect layout/noise signals before using this output for Markdown export, RAG ingestion preparation, or manual extraction. If the listed items are expected chart labels, headers, footers, or figure labels, keep, exclude, or describe them according to the downstream use case. If they point to missing or incorrect content, adjust extraction before reuse. diff --git a/examples/mdpi_pdf_elements/page_012/quality_report.md b/examples/mdpi_pdf_elements/page_012/quality_report.md index 5e82fca..8a1e72c 100644 --- a/examples/mdpi_pdf_elements/page_012/quality_report.md +++ b/examples/mdpi_pdf_elements/page_012/quality_report.md @@ -44,6 +44,7 @@ Each section below is a separate check. - BBox Sanity checks whether bounding boxes look structurally valid. - Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise. - Text Usefulness flags very short or repeated text fragments. +- Text Extraction Health checks extracted-text availability, not extracted-text correctness. ## Required Field Coverage PASS @@ -104,6 +105,18 @@ WARN - repeated text value '125' appears in block IDs: p12-texts-613, p12-texts-618 - repeated text value '325' appears in block IDs: p12-texts-617, p12-texts-621 +## Text Extraction Health +PASS + +extracted text availability checks passed + +- text_bearing_blocks=81 +- non_empty_text_blocks=81 +- empty_text_blocks=0 +- total_text_chars=4482 +- image_blocks=1 +- empty_text_block_ratio=0.000 + ## Recommended Review Actions Because the decision is REVIEW, inspect short/repeated text fragments and layout/noise signals before using this output for Markdown export, RAG ingestion preparation, or manual extraction. If the listed items are expected chart labels, headers, footers, or figure labels, keep, exclude, or describe them according to the downstream use case. If they point to missing or incorrect content, adjust extraction before reuse. diff --git a/src/pdf_quality_report/checks.py b/src/pdf_quality_report/checks.py index 4e79224..9e594fd 100644 --- a/src/pdf_quality_report/checks.py +++ b/src/pdf_quality_report/checks.py @@ -16,6 +16,9 @@ BBOX_Y1_INDEX = 3 BBOX_TOLERANCE = 0.0001 SHORT_TEXT_THRESHOLD = 3 +LOW_TEXT_CHAR_THRESHOLD = 40 +IMAGE_LOW_TEXT_CHAR_THRESHOLD = 120 +EMPTY_TEXT_BLOCK_RATIO_THRESHOLD = 0.5 TEXT_BODY_TYPES = {"heading", "paragraph", "list_item_text", "caption", "footnote", "table"} TEXT_BEARING_TYPES = TEXT_BODY_TYPES | {"header", "footer"} @@ -39,7 +42,7 @@ def run_quality_checks(uif_document: dict[str, Any]) -> QualityReport: - """Run the five v0 checks on a normalized block document.""" + """Run the v0 checks on a normalized block document.""" blocks = _blocks(uif_document) results = [ check_required_field_coverage(uif_document), @@ -47,6 +50,7 @@ def run_quality_checks(uif_document: dict[str, Any]) -> QualityReport: check_bbox_sanity(blocks), check_content_vs_noise_ratio(blocks), check_text_usefulness(blocks), + check_text_extraction_health(blocks), ] return QualityReport( total_blocks=len(blocks), @@ -219,6 +223,70 @@ def check_text_usefulness(blocks: list[dict[str, Any]]) -> CheckResult: return CheckResult("Text Usefulness", status, summary, [*failures, *warnings]) +def check_text_extraction_health(blocks: list[dict[str, Any]]) -> CheckResult: + """Check extracted-text availability without judging text correctness.""" + total_blocks = len(blocks) + text_bearing_blocks = 0 + non_empty_text_blocks = 0 + total_text_chars = 0 + image_blocks = 0 + + for block in blocks: + block_type = str(block.get("type", "unknown")) + if block_type == "image": + image_blocks += 1 + if block_type not in TEXT_BEARING_TYPES: + continue + text_bearing_blocks += 1 + normalized_text = _normalize_text(_content_text(block)) + if normalized_text: + non_empty_text_blocks += 1 + total_text_chars += len(normalized_text) + + empty_text_blocks = text_bearing_blocks - non_empty_text_blocks + empty_text_block_ratio = empty_text_blocks / text_bearing_blocks if text_bearing_blocks else 0.0 + details = [ + f"text_bearing_blocks={text_bearing_blocks}", + f"non_empty_text_blocks={non_empty_text_blocks}", + f"empty_text_blocks={empty_text_blocks}", + f"total_text_chars={total_text_chars}", + f"image_blocks={image_blocks}", + f"empty_text_block_ratio={empty_text_block_ratio:.3f}", + ] + + warnings: list[str] = [] + if total_blocks == 0: + warnings.append("no_blocks: document has no blocks to evaluate") + if total_blocks > 0 and text_bearing_blocks == 0: + warnings.append("no_text_bearing_blocks: document has blocks but no text-bearing blocks") + if text_bearing_blocks > 0 and non_empty_text_blocks == 0: + warnings.append("no_non_empty_text_blocks: text-bearing blocks contain no extracted text") + if total_blocks > 0 and total_text_chars < LOW_TEXT_CHAR_THRESHOLD: + warnings.append( + f"low_text_coverage: total_text_chars={total_text_chars} below threshold={LOW_TEXT_CHAR_THRESHOLD}" + ) + if image_blocks > 0 and total_text_chars < IMAGE_LOW_TEXT_CHAR_THRESHOLD: + warnings.append( + "image_low_text_coverage: document contains image blocks while extracted text remains very limited " + f"(total_text_chars={total_text_chars} below threshold={IMAGE_LOW_TEXT_CHAR_THRESHOLD})" + ) + if text_bearing_blocks > 0 and empty_text_block_ratio > EMPTY_TEXT_BLOCK_RATIO_THRESHOLD: + warnings.append( + "high_empty_text_block_ratio: " + f"empty_text_block_ratio={empty_text_block_ratio:.3f} " + f"above threshold={EMPTY_TEXT_BLOCK_RATIO_THRESHOLD:.3f}" + ) + + if warnings: + summary = ( + "document has no blocks to evaluate" + if total_blocks == 0 + else f"{len(warnings)} text extraction health warning(s)" + ) + return CheckResult("Text Extraction Health", "WARN", summary, [*details, *warnings]) + return CheckResult("Text Extraction Health", "PASS", "extracted text availability checks passed", details) + + def _blocks(uif_document: dict[str, Any]) -> list[dict[str, Any]]: raw_blocks = uif_document.get("blocks", []) return [block for block in raw_blocks if isinstance(block, dict)] if isinstance(raw_blocks, list) else [] diff --git a/src/pdf_quality_report/interpret.py b/src/pdf_quality_report/interpret.py index e5588bc..0edf0d2 100644 --- a/src/pdf_quality_report/interpret.py +++ b/src/pdf_quality_report/interpret.py @@ -15,6 +15,7 @@ _DUPLICATE_RE = re.compile(r"^duplicate normalized text across blocks: (?P.+)$") _REPEATED_VALUE_RE = re.compile(r"^repeated text value (?P.+) appears in block IDs: (?P.+)$") _DETAIL_COUNT_RE = re.compile(r"^(?P[a-z_]+)=(?P\d+)$") +_DETAIL_FLOAT_RE = re.compile(r"^(?P[a-z_]+)=(?P\d+(?:\.\d+)?)$") _SIGNAL_ID_RE = re.compile(r"^(?P[^:]+):") _SIGNAL_DETAIL_RE = re.compile(r"^(?P[^:]+): type=(?P[^,]+), text=(?P.+)$") _NUMERIC_LIKE_RE = re.compile(r"^[+-]?(?:\d+(?:\.\d*)?|\.\d+)$") @@ -40,6 +41,8 @@ def interpret_quality_report(report: QualityReport) -> list[str]: def _interpret_result(result: CheckResult, signals: NoiseLayoutSignals) -> list[str]: if result.name == "Text Usefulness": return _interpret_text_usefulness(result) + if result.name == "Text Extraction Health": + return _interpret_text_extraction_health(result) if result.name == "Content vs Noise Ratio": return _interpret_content_vs_noise(result, signals) if result.name == "BBox Sanity": @@ -117,6 +120,49 @@ def _interpret_text_usefulness(result: CheckResult) -> list[str]: return bullets +def _interpret_text_extraction_health(result: CheckResult) -> list[str]: + counts = _detail_counts(result.details) + ratios = _detail_floats(result.details) + text_bearing_blocks = counts.get("text_bearing_blocks") + non_empty_text_blocks = counts.get("non_empty_text_blocks") + empty_text_blocks = counts.get("empty_text_blocks") + total_text_chars = counts.get("total_text_chars") + image_blocks = counts.get("image_blocks") + empty_text_block_ratio = ratios.get("empty_text_block_ratio") + + if None in (text_bearing_blocks, non_empty_text_blocks, empty_text_blocks, total_text_chars, image_blocks): + return [ + f"{result.name} is {result.status}: {result.summary}. " + "This check evaluates whether extracted text is available for review; it does not judge text correctness." + ] + + ratio_text = ( + f"{empty_text_block_ratio:.3f}" + if empty_text_block_ratio is not None + else "not available" + ) + bullets = [ + f"{result.name} is {result.status}: This report found " + f"{non_empty_text_blocks} non-empty text-bearing {_plural(non_empty_text_blocks, 'block')} " + f"out of {text_bearing_blocks}, with {total_text_chars} extracted text characters." + ] + bullets.append( + "This check evaluates extracted-text availability for review; it does not judge text correctness, " + "OCR behavior, or parser accuracy." + ) + if image_blocks: + bullets.append( + f"The normalized output includes {image_blocks} image {_plural(image_blocks, 'block')}; " + "image presence is only a review signal when extracted text remains limited." + ) + if empty_text_blocks: + bullets.append( + f"{empty_text_blocks} text-bearing {_plural(empty_text_blocks, 'block')} " + f"had no extracted text. The empty text-block ratio is {ratio_text}." + ) + return bullets + + def _interpret_generic_result(result: CheckResult) -> str: examples = _detail_examples(result.details) suffix = f" Examples: {examples}." if examples else "" @@ -179,6 +225,15 @@ def _detail_counts(details: list[str]) -> dict[str, int]: return counts +def _detail_floats(details: list[str]) -> dict[str, float]: + values: dict[str, float] = {} + for detail in details: + match = _DETAIL_FLOAT_RE.match(detail) + if match: + values[match.group("name")] = float(match.group("value")) + return values + + def _leading_count(text: str) -> int | None: first_token = text.split(maxsplit=1)[0] if text else "" return int(first_token) if first_token.isdigit() else None diff --git a/src/pdf_quality_report/report.py b/src/pdf_quality_report/report.py index fb1eba9..22406b1 100644 --- a/src/pdf_quality_report/report.py +++ b/src/pdf_quality_report/report.py @@ -19,6 +19,7 @@ "BBox Sanity checks whether bounding boxes look structurally valid.", "Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise.", "Text Usefulness flags very short or repeated text fragments.", + "Text Extraction Health checks extracted-text availability, not extracted-text correctness.", ) @@ -220,6 +221,8 @@ def _review_targets(report: QualityReport) -> str: targets: list[str] = [] if "Text Usefulness" in warning_names: targets.append("short/repeated text fragments") + if "Text Extraction Health" in warning_names: + targets.append("extracted-text availability") if "Content vs Noise Ratio" in warning_names or any( ( report.noise_layout_signals.table_marker_artifacts, diff --git a/tests/test_interpret.py b/tests/test_interpret.py index 7853784..f6f85fb 100644 --- a/tests/test_interpret.py +++ b/tests/test_interpret.py @@ -121,6 +121,40 @@ def test_text_usefulness_duplicate_falls_back_to_ids_when_value_is_not_available assert any("p12-texts-700 and p12-texts-701" in bullet for bullet in bullets) +def test_text_extraction_health_warn_gets_conservative_explanation() -> None: + report = _report( + [ + CheckResult( + "Text Extraction Health", + "WARN", + "3 text extraction health warning(s)", + [ + "text_bearing_blocks=1", + "non_empty_text_blocks=0", + "empty_text_blocks=1", + "total_text_chars=0", + "image_blocks=1", + "empty_text_block_ratio=1.000", + "no_non_empty_text_blocks: text-bearing blocks contain no extracted text", + "low_text_coverage: total_text_chars=0 below threshold=40", + "image_low_text_coverage: document contains image blocks while extracted text remains very limited " + "(total_text_chars=0 below threshold=120)", + ], + ) + ] + ) + + bullets = interpret_quality_report(report) + + assert any("Text Extraction Health is WARN" in bullet for bullet in bullets) + assert any("0 non-empty text-bearing blocks out of 1" in bullet for bullet in bullets) + assert any("does not judge text correctness" in bullet for bullet in bullets) + assert any("image presence is only a review signal" in bullet for bullet in bullets) + assert not any("failed" in bullet.lower() for bullet in bullets) + assert not any("supports OCR" in bullet for bullet in bullets) + assert report.decision == "REVIEW" + + def test_bbox_failure_gets_failure_explanation_without_decision_change() -> None: report = _report( [ diff --git a/tests/test_quality_report.py b/tests/test_quality_report.py index 92ef29b..7553088 100644 --- a/tests/test_quality_report.py +++ b/tests/test_quality_report.py @@ -15,7 +15,13 @@ def _valid_block(block_id: str, block_type: str = "paragraph") -> dict: "bbox": [1.0, 2.0, 3.0, 4.0], "relationships": [], "reading_order_index": 0, - "content": {"text": "Useful text", "spans": []}, + "content": { + "text": ( + "Useful extracted text with enough characters for deterministic health checks. " + "This sentence keeps ordinary valid fixtures above the image-text warning threshold." + ), + "spans": [], + }, "provenance": { "parser_backend": "docling", "docling_schema": "DoclingDocument", @@ -50,6 +56,7 @@ def test_quality_checks_pass_hard_checks_and_warn_for_noise() -> None: assert statuses["Provenance Completeness"] == "PASS" assert statuses["BBox Sanity"] == "PASS" assert statuses["Content vs Noise Ratio"] == "WARN" + assert statuses["Text Extraction Health"] == "PASS" assert report.hard_failures == 0 assert report.warnings == 1 assert report.decision == "REVIEW" @@ -113,6 +120,101 @@ def test_noise_layout_signals_capture_table_furniture_and_images() -> None: assert report.hard_failures == 0 +def test_text_extraction_health_passes_when_text_is_available() -> None: + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [_valid_block("p6-texts-48")]}) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + assert health.status == "PASS" + assert health.summary == "extracted text availability checks passed" + assert "text_bearing_blocks=1" in health.details + assert "non_empty_text_blocks=1" in health.details + assert "empty_text_blocks=0" in health.details + assert "image_blocks=0" in health.details + assert "empty_text_block_ratio=0.000" in health.details + + +def test_text_extraction_health_warns_without_hard_failure() -> None: + image = _valid_block("p6-pictures-4", "image") + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [image]}) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + assert health.status == "WARN" + assert "text_bearing_blocks=0" in health.details + assert "non_empty_text_blocks=0" in health.details + assert "empty_text_blocks=0" in health.details + assert "total_text_chars=0" in health.details + assert "image_blocks=1" in health.details + assert "empty_text_block_ratio=0.000" in health.details + assert "no_text_bearing_blocks: document has blocks but no text-bearing blocks" in health.details + assert "low_text_coverage: total_text_chars=0 below threshold=40" in health.details + assert any( + "document contains image blocks while extracted text remains very limited" in detail + for detail in health.details + ) + assert health.status != "FAIL" + assert report.hard_failures == 0 + assert report.decision == "REVIEW" + + +def test_text_extraction_health_warns_when_text_bearing_blocks_are_empty_without_body_failures() -> None: + header = _valid_block("p6-texts-45", "header") + header["content"]["text"] = "" + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [header]}) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + text_usefulness = next(result for result in report.results if result.name == "Text Usefulness") + assert health.status == "WARN" + assert text_usefulness.status == "PASS" + assert "text_bearing_blocks=1" in health.details + assert "non_empty_text_blocks=0" in health.details + assert "no_non_empty_text_blocks: text-bearing blocks contain no extracted text" in health.details + assert health.status != "FAIL" + assert report.hard_failures == 0 + assert report.decision == "REVIEW" + + +def test_text_extraction_health_warns_for_empty_text_ratio() -> None: + empty_body = _valid_block("p6-texts-49") + empty_body["content"]["text"] = "" + filled_body = _valid_block("p6-texts-50") + filled_body["content"]["text"] = ( + "Enough extracted text to avoid the low-text threshold for this specific ratio case." + ) + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [empty_body, filled_body]}) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + assert health.status == "PASS" + assert "empty_text_block_ratio=0.500" in health.details + + second_empty_body = _valid_block("p6-texts-51") + second_empty_body["content"]["text"] = "" + report = run_quality_checks( + {"metadata": {"source": "test"}, "blocks": [empty_body, second_empty_body, filled_body]} + ) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + text_usefulness = next(result for result in report.results if result.name == "Text Usefulness") + assert health.status == "WARN" + assert text_usefulness.status == "FAIL" + assert "empty_text_block_ratio=0.667" in health.details + assert any("high_empty_text_block_ratio" in detail for detail in health.details) + assert health.status != "FAIL" + assert report.hard_failures == 1 + + +def test_text_extraction_health_warns_when_document_has_no_blocks() -> None: + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": []}) + + health = next(result for result in report.results if result.name == "Text Extraction Health") + assert health.status == "WARN" + assert health.summary == "document has no blocks to evaluate" + assert "no_blocks: document has no blocks to evaluate" in health.details + assert report.hard_failures == 0 + + def test_markdown_report_contains_required_sections() -> None: table_marker = _valid_block("p12-texts-109") table_marker["content"]["text"] = "+" @@ -135,6 +237,8 @@ def test_markdown_report_contains_required_sections() -> None: assert "table_marker_artifacts: 1" in markdown assert "## Required Field Coverage" in markdown assert "## Text Usefulness" in markdown + assert "## Text Extraction Health" in markdown + assert "Text Extraction Health checks extracted-text availability, not extracted-text correctness." in markdown def test_cli_writes_quality_report(tmp_path: Path) -> None: @@ -149,3 +253,19 @@ def test_cli_writes_quality_report(tmp_path: Path) -> None: assert "hard_failures: 0" in report_text assert "decision: GO" in report_text assert "This does not prove complete or semantically correct extraction" in report_text + + +def test_cli_exits_zero_when_only_text_extraction_health_warns(tmp_path: Path) -> None: + uif_path = tmp_path / "uif.json" + report_path = tmp_path / "quality_report.md" + block = _valid_block("p6-unknown-1", "unknown") + block["content"]["text"] = "" + uif_path.write_text(json.dumps({"metadata": {}, "blocks": [block]}), encoding="utf-8") + + assert main(["--input", str(uif_path), "--output", str(report_path)]) == 0 + + report_text = report_path.read_text(encoding="utf-8") + assert "hard_failures: 0" in report_text + assert "decision: REVIEW" in report_text + assert "## Text Extraction Health" in report_text + assert "no_text_bearing_blocks: document has blocks but no text-bearing blocks" in report_text