diff --git a/README.md b/README.md index 23d12a2..ef4cd94 100644 --- a/README.md +++ b/README.md @@ -16,23 +16,28 @@ Technical note: the traceable output is plain JSON. It is inspired by UIF, an in ## What The Report Checks -The report answers three practical questions: +The report answers practical questions such as: 1. Does every extracted block have the basic fields a pipeline needs? 2. Can each block be traced back to a source page and bounding box? 3. Does the page contain layout noise that a human should review before using the output? -The six current checks are: +The seven current checks are: 1. Required field coverage: are core fields such as `id`, `type`, `page_number`, `bbox`, and `reading_order_index` present? -2. Provenance completeness: can each block be traced back to Docling metadata and the source PDF? -3. BBox sanity: are bounding boxes present, finite, and ordered correctly? -4. Content vs noise ratio: how much looks like document content compared with headers, footers, images, or other secondary blocks? -5. Text usefulness: are text blocks empty, suspiciously short, or duplicated? -6. Text extraction health: is extracted text available in the normalized blocks for review, without judging whether the text is correct? +2. Table output structure signals: do table-labeled normalized blocks contain visible table-output structure signals? +3. Provenance completeness: can each block be traced back to Docling metadata and the source PDF? +4. BBox sanity: are bounding boxes present, finite, and ordered correctly? +5. Content vs noise ratio: how much looks like document content compared with headers, footers, images, or other secondary blocks? +6. Text usefulness: are text blocks empty, suspiciously short, or duplicated? +7. Text extraction health: is extracted text available in the normalized blocks for review, without judging whether the text is correct? Text usefulness flags local text-quality issues such as empty body blocks, very short text, or repeated fragments. Text extraction health summarizes whether the document contains enough extracted text overall for review. +The table output structure signal check flags table-labeled normalized blocks that lack obvious structured table-output +signals; it does not validate table reconstruction correctness against the PDF. +Structured table-output signals include normalized grid fields such as `data_grid` or `rows`, optional `cells` / `html`, +or delimiter-like `content.text`. Warning-only checks can move the report to `REVIEW`; the report CLI still exits non-zero only for hard failures. diff --git a/examples/mdpi_pdf_elements/page_006/quality_report.md b/examples/mdpi_pdf_elements/page_006/quality_report.md index d6f86ba..3b25920 100644 --- a/examples/mdpi_pdf_elements/page_006/quality_report.md +++ b/examples/mdpi_pdf_elements/page_006/quality_report.md @@ -29,6 +29,7 @@ Details: Each section below is a separate check. - Required Field Coverage checks whether required JSON fields are present. +- Table Output Structure Signals checks whether table-labeled normalized blocks contain visible table-output structure signals. - Provenance Completeness checks whether source, page, and bounding-box metadata is present. - BBox Sanity checks whether bounding boxes look structurally valid. - Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise. @@ -40,6 +41,20 @@ PASS all required fields present +## Table Output Structure Signals +PASS + +no table-labeled blocks found + +- table_blocks=0 +- structured_grid_blocks=0 +- alternate_structure_signal_blocks=0 +- text_structure_signal_blocks=0 +- plain_text_only_blocks=0 +- empty_or_short_table_text_blocks=0 +- inconsistent_grid_blocks=0 +- table_blocks=0; no table-labeled blocks found + ## Provenance Completeness PASS diff --git a/examples/mdpi_pdf_elements/page_012/quality_report.md b/examples/mdpi_pdf_elements/page_012/quality_report.md index 8a1e72c..644e9b7 100644 --- a/examples/mdpi_pdf_elements/page_012/quality_report.md +++ b/examples/mdpi_pdf_elements/page_012/quality_report.md @@ -40,6 +40,7 @@ Details: Each section below is a separate check. - Required Field Coverage checks whether required JSON fields are present. +- Table Output Structure Signals checks whether table-labeled normalized blocks contain visible table-output structure signals. - Provenance Completeness checks whether source, page, and bounding-box metadata is present. - BBox Sanity checks whether bounding boxes look structurally valid. - Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise. @@ -51,6 +52,20 @@ PASS all required fields present +## Table Output Structure Signals +PASS + +no table-labeled blocks found + +- table_blocks=0 +- structured_grid_blocks=0 +- alternate_structure_signal_blocks=0 +- text_structure_signal_blocks=0 +- plain_text_only_blocks=0 +- empty_or_short_table_text_blocks=0 +- inconsistent_grid_blocks=0 +- table_blocks=0; no table-labeled blocks found + ## Provenance Completeness PASS diff --git a/src/pdf_quality_report/checks.py b/src/pdf_quality_report/checks.py index 9e594fd..0d1f99c 100644 --- a/src/pdf_quality_report/checks.py +++ b/src/pdf_quality_report/checks.py @@ -19,6 +19,7 @@ LOW_TEXT_CHAR_THRESHOLD = 40 IMAGE_LOW_TEXT_CHAR_THRESHOLD = 120 EMPTY_TEXT_BLOCK_RATIO_THRESHOLD = 0.5 +SHORT_TABLE_TEXT_THRESHOLD = 20 TEXT_BODY_TYPES = {"heading", "paragraph", "list_item_text", "caption", "footnote", "table"} TEXT_BEARING_TYPES = TEXT_BODY_TYPES | {"header", "footer"} @@ -46,6 +47,7 @@ def run_quality_checks(uif_document: dict[str, Any]) -> QualityReport: blocks = _blocks(uif_document) results = [ check_required_field_coverage(uif_document), + check_table_output_structure(blocks), check_provenance_completeness(blocks), check_bbox_sanity(blocks), check_content_vs_noise_ratio(blocks), @@ -140,6 +142,90 @@ def check_provenance_completeness(blocks: list[dict[str, Any]]) -> CheckResult: return CheckResult("Provenance Completeness", status, summary, [*failures, *warnings]) +def check_table_output_structure(blocks: list[dict[str, Any]]) -> CheckResult: + """Check visible table-output structure signals without judging table correctness.""" + table_blocks = [ + (index, block) + for index, block in enumerate(blocks) + if str(block.get("type", "unknown")) == "table" + ] + counts = Counter[str]() + warnings: list[str] = [] + + for index, block in table_blocks: + block_id = _block_id(block, index) + text = _content_text(block) + normalized_text = _normalize_text(text) + grid = _table_grid_signal(block) + has_grid_signal = _has_non_empty_grid(grid) + has_alternate_structured_signal = _has_alternate_table_structure_signal(block) + has_text_structure_signal = _has_table_text_structure_signal(text) + + if has_grid_signal: + counts["structured_grid_blocks"] += 1 + if has_alternate_structured_signal: + counts["alternate_structure_signal_blocks"] += 1 + if has_text_structure_signal: + counts["text_structure_signal_blocks"] += 1 + + row_widths = _row_widths(grid) + if len(set(row_widths)) > 1: + counts["inconsistent_grid_blocks"] += 1 + warnings.append( + f"{block_id}: table data_grid has inconsistent row widths: {_format_int_values(row_widths)}" + ) + + has_any_structure_signal = has_grid_signal or has_alternate_structured_signal or has_text_structure_signal + if not normalized_text and not has_any_structure_signal: + counts["empty_or_short_table_text_blocks"] += 1 + warnings.append(f"{block_id}: table block has no structured table signal and no markdown text") + elif not has_any_structure_signal and len(normalized_text) <= SHORT_TABLE_TEXT_THRESHOLD: + counts["empty_or_short_table_text_blocks"] += 1 + warnings.append( + f"{block_id}: table block has very short table text and no obvious row/column structure signal " + f"(chars={len(normalized_text)})" + ) + elif not has_any_structure_signal: + counts["plain_text_only_blocks"] += 1 + warnings.append( + f"{block_id}: table block has plain text only and no obvious row/column structure signal " + f"(chars={len(normalized_text)})" + ) + elif has_grid_signal and not normalized_text: + counts["empty_or_short_table_text_blocks"] += 1 + warnings.append(f"{block_id}: table block has structured grid signal but empty markdown content.text") + + details = [ + f"table_blocks={len(table_blocks)}", + f"structured_grid_blocks={counts['structured_grid_blocks']}", + f"alternate_structure_signal_blocks={counts['alternate_structure_signal_blocks']}", + f"text_structure_signal_blocks={counts['text_structure_signal_blocks']}", + f"plain_text_only_blocks={counts['plain_text_only_blocks']}", + f"empty_or_short_table_text_blocks={counts['empty_or_short_table_text_blocks']}", + f"inconsistent_grid_blocks={counts['inconsistent_grid_blocks']}", + ] + if not table_blocks: + return CheckResult( + "Table Output Structure Signals", + "PASS", + "no table-labeled blocks found", + [*details, "table_blocks=0; no table-labeled blocks found"], + ) + if warnings: + return CheckResult( + "Table Output Structure Signals", + "WARN", + f"{len(warnings)} table output structure warning(s)", + [*details, *warnings], + ) + return CheckResult( + "Table Output Structure Signals", + "PASS", + "table-labeled blocks include visible structure signals", + details, + ) + + def check_bbox_sanity(blocks: list[dict[str, Any]]) -> CheckResult: """Check bbox shape, ordering, units, origin, and raw Docling consistency.""" details: list[str] = [] @@ -366,6 +452,78 @@ def _normalize_text(text: str) -> str: return re.sub(r"\s+", " ", text).strip() +def _table_grid_signal(block: dict[str, Any]) -> list[list[str]]: + for value in ( + block.get("data_grid"), + _dict_or_empty(block.get("content")).get("data_grid"), + _dict_or_empty(block.get("content")).get("rows"), + block.get("rows"), + ): + grid = _coerce_grid(value) + if grid: + return grid + return [] + + +def _coerce_grid(value: Any) -> list[list[str]]: + if not isinstance(value, list): + return [] + rows: list[list[str]] = [] + for row in value: + if isinstance(row, list): + rows.append([_cell_text(cell) for cell in row]) + return rows + + +def _cell_text(value: Any) -> str: + if isinstance(value, dict): + text = value.get("text") + return text if isinstance(text, str) else "" + return str(value) if value is not None else "" + + +def _has_non_empty_grid(grid: list[list[str]]) -> bool: + return any(cell.strip() for row in grid for cell in row) + + +def _has_alternate_table_structure_signal(block: dict[str, Any]) -> bool: + content = _dict_or_empty(block.get("content")) + return ( + _has_non_empty_cells(block.get("cells")) + or _has_non_empty_cells(content.get("cells")) + or _has_non_empty_html(block.get("html")) + or _has_non_empty_html(content.get("html")) + ) + + +def _has_non_empty_cells(value: Any) -> bool: + if not isinstance(value, list): + return False + return any(_cell_text(cell).strip() for cell in value) + + +def _has_non_empty_html(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _has_table_text_structure_signal(text: str) -> bool: + if "\t" in text or "|" in text: + return True + lines = [line.strip() for line in text.splitlines() if line.strip()] + if len(lines) < 2: + return False + column_counts = [len(re.split(r"\s{2,}", line)) for line in lines] + return min(column_counts) > 1 and len(set(column_counts)) == 1 + + +def _row_widths(grid: list[list[str]]) -> list[int]: + return [len(row) for row in grid if row] + + +def _format_int_values(values: list[int]) -> str: + return ", ".join(str(value) for value in sorted(set(values))) + + def _block_id(block: dict[str, Any], index: int) -> str: raw_id = block.get("id") return raw_id if isinstance(raw_id, str) and raw_id else f"block[{index}]" diff --git a/src/pdf_quality_report/interpret.py b/src/pdf_quality_report/interpret.py index 0edf0d2..bce5e19 100644 --- a/src/pdf_quality_report/interpret.py +++ b/src/pdf_quality_report/interpret.py @@ -43,6 +43,8 @@ def _interpret_result(result: CheckResult, signals: NoiseLayoutSignals) -> list[ return _interpret_text_usefulness(result) if result.name == "Text Extraction Health": return _interpret_text_extraction_health(result) + if result.name == "Table Output Structure Signals": + return _interpret_table_output_structure(result) if result.name == "Content vs Noise Ratio": return _interpret_content_vs_noise(result, signals) if result.name == "BBox Sanity": @@ -163,6 +165,52 @@ def _interpret_text_extraction_health(result: CheckResult) -> list[str]: return bullets +def _interpret_table_output_structure(result: CheckResult) -> list[str]: + counts = _detail_counts(result.details) + table_blocks = counts.get("table_blocks") + plain_text_only_blocks = counts.get("plain_text_only_blocks") + empty_or_short_blocks = counts.get("empty_or_short_table_text_blocks") + inconsistent_grid_blocks = counts.get("inconsistent_grid_blocks") + + if table_blocks is None: + return [ + f"{result.name} is {result.status}: {result.summary}. " + "This check reviews table-output structure signals in normalized blocks; it does not judge table " + "correctness." + ] + + issue_counts = [ + count + for count in (plain_text_only_blocks, empty_or_short_blocks, inconsistent_grid_blocks) + if count is not None and count > 0 + ] + bullets = [ + _table_output_structure_count_bullet(result, table_blocks) + ] + if issue_counts: + bullets.append( + "Some table-labeled blocks lack obvious structured table-output signals or have inconsistent row-width " + "signals in the normalized parser output." + ) + bullets.append( + "This check does not validate table reconstruction correctness against the PDF, infer PDF ground truth, " + "or judge parser accuracy." + ) + examples = _warning_detail_examples(result.details) + if examples: + bullets.append(f"Examples: {examples}.") + return bullets + + +def _table_output_structure_count_bullet(result: CheckResult, table_blocks: int) -> str: + if table_blocks == 0: + return f"{result.name} is {result.status}: There are no table-labeled normalized blocks to review." + return ( + f"{result.name} is {result.status}: This report found {table_blocks} table-labeled normalized " + f"{_plural(table_blocks, 'block')}." + ) + + def _interpret_generic_result(result: CheckResult) -> str: examples = _detail_examples(result.details) suffix = f" Examples: {examples}." if examples else "" @@ -196,6 +244,11 @@ def _interpret_noise_layout_signals(signals: NoiseLayoutSignals) -> list[str]: return bullets +def _warning_detail_examples(details: list[str], limit: int = 2) -> str: + warning_details = [detail for detail in details if ": " in detail] + return "; ".join(warning_details[:limit]) + + def _detail_examples(details: list[str], limit: int = 2) -> str: return "; ".join(details[:limit]) diff --git a/src/pdf_quality_report/report.py b/src/pdf_quality_report/report.py index 22406b1..cac693d 100644 --- a/src/pdf_quality_report/report.py +++ b/src/pdf_quality_report/report.py @@ -15,6 +15,8 @@ CHECK_RESULT_EXPLANATIONS = ( "Each section below is a separate check.", "Required Field Coverage checks whether required JSON fields are present.", + "Table Output Structure Signals checks whether table-labeled normalized blocks contain visible table-output " + "structure signals.", "Provenance Completeness checks whether source, page, and bounding-box metadata is present.", "BBox Sanity checks whether bounding boxes look structurally valid.", "Content vs Noise Ratio checks how much extracted content looks like main text versus layout/noise.", diff --git a/tests/test_interpret.py b/tests/test_interpret.py index f6f85fb..2bc688f 100644 --- a/tests/test_interpret.py +++ b/tests/test_interpret.py @@ -155,6 +155,40 @@ def test_text_extraction_health_warn_gets_conservative_explanation() -> None: assert report.decision == "REVIEW" +def test_table_output_structure_warn_gets_conservative_explanation() -> None: + report = _report( + [ + CheckResult( + "Table Output Structure Signals", + "WARN", + "1 table output structure warning(s)", + [ + "table_blocks=1", + "structured_grid_blocks=0", + "alternate_structure_signal_blocks=0", + "text_structure_signal_blocks=0", + "plain_text_only_blocks=1", + "empty_or_short_table_text_blocks=0", + "inconsistent_grid_blocks=0", + "p6-tables-1: table block has plain text only and no obvious row/column structure signal " + "(chars=72)", + ], + ) + ] + ) + + bullets = interpret_quality_report(report) + + assert any("Table Output Structure Signals is WARN" in bullet for bullet in bullets) + assert any("1 table-labeled normalized block" in bullet for bullet in bullets) + assert any("lack obvious structured table-output signals" in bullet for bullet in bullets) + assert any("does not validate table reconstruction correctness against the PDF" in bullet for bullet in bullets) + assert any("p6-tables-1" in bullet for bullet in bullets) + assert not any("failed" in bullet.lower() for bullet in bullets) + assert not any("accuracy" in bullet.lower() and "does not" not in bullet.lower() for bullet in bullets) + assert report.decision == "REVIEW" + + def test_bbox_failure_gets_failure_explanation_without_decision_change() -> None: report = _report( [ diff --git a/tests/test_quality_report.py b/tests/test_quality_report.py index 7553088..a57af0c 100644 --- a/tests/test_quality_report.py +++ b/tests/test_quality_report.py @@ -43,6 +43,18 @@ def _valid_block(block_id: str, block_type: str = "paragraph") -> dict: return block +def _table_block( + block_id: str, + *, + text: str = "| A | B |\n| --- | --- |\n| 1 | 2 |", + data_grid: list[list[str]] | None = None, +) -> dict: + block = _valid_block(block_id, "table") + block["content"]["text"] = text + block["data_grid"] = data_grid if data_grid is not None else [["A", "B"], ["1", "2"]] + return block + + def test_quality_checks_pass_hard_checks_and_warn_for_noise() -> None: body = _valid_block("p6-texts-48") header = _valid_block("p6-texts-45", "header") @@ -53,6 +65,7 @@ def test_quality_checks_pass_hard_checks_and_warn_for_noise() -> None: statuses = {result.name: result.status for result in report.results} assert statuses["Required Field Coverage"] == "PASS" + assert statuses["Table Output Structure Signals"] == "PASS" assert statuses["Provenance Completeness"] == "PASS" assert statuses["BBox Sanity"] == "PASS" assert statuses["Content vs Noise Ratio"] == "WARN" @@ -62,6 +75,91 @@ def test_quality_checks_pass_hard_checks_and_warn_for_noise() -> None: assert report.decision == "REVIEW" +def test_table_output_structure_passes_without_table_blocks() -> None: + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [_valid_block("p6-texts-48")]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "PASS" + assert table_check.summary == "no table-labeled blocks found" + assert "table_blocks=0" in table_check.details + assert "table_blocks=0; no table-labeled blocks found" in table_check.details + + +def test_table_output_structure_passes_for_structured_data_grid() -> None: + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [_table_block("p6-tables-1")]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "PASS" + assert table_check.summary == "table-labeled blocks include visible structure signals" + assert "table_blocks=1" in table_check.details + assert "structured_grid_blocks=1" in table_check.details + + +def test_table_output_structure_accepts_optional_rows_signal() -> None: + table = _table_block("p6-tables-1", text="A B C", data_grid=[]) + table["content"]["rows"] = [["A", "B"], ["1", "2"]] + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [table]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "PASS" + assert "structured_grid_blocks=1" in table_check.details + + +def test_table_output_structure_warns_for_plain_text_only_without_hard_failure() -> None: + table = _table_block( + "p6-tables-1", + text="This table-labeled block appears as one plain paragraph without obvious columns.", + data_grid=[], + ) + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [table]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "WARN" + assert "plain_text_only_blocks=1" in table_check.details + assert any("plain text only and no obvious row/column structure signal" in detail for detail in table_check.details) + assert report.hard_failures == 0 + assert report.decision == "REVIEW" + + +def test_table_output_structure_warns_for_inconsistent_grid_widths() -> None: + table = _table_block("p6-tables-1", data_grid=[["A", "B"], ["1"]]) + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [table]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "WARN" + assert "inconsistent_grid_blocks=1" in table_check.details + assert "p6-tables-1: table data_grid has inconsistent row widths: 1, 2" in table_check.details + assert report.hard_failures == 0 + + +def test_table_output_structure_passes_for_text_structure_signal() -> None: + table = _table_block("p6-tables-1", text="A\tB\n1\t2", data_grid=[]) + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [table]}) + + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert table_check.status == "PASS" + assert "text_structure_signal_blocks=1" in table_check.details + + +def test_table_output_structure_does_not_replace_required_field_failures() -> None: + table = _table_block("p6-tables-1") + del table["data_grid"] + + report = run_quality_checks({"metadata": {"source": "test"}, "blocks": [table]}) + + required = next(result for result in report.results if result.name == "Required Field Coverage") + table_check = next(result for result in report.results if result.name == "Table Output Structure Signals") + assert required.status == "FAIL" + assert "p6-tables-1: table missing data_grid" in required.details + assert table_check.status == "PASS" + assert table_check.status != "FAIL" + assert report.hard_failures == 1 + + def test_quality_checks_fail_missing_required_field() -> None: block = _valid_block("p6-texts-48") del block["bbox"] @@ -236,8 +334,10 @@ def test_markdown_report_contains_required_sections() -> None: assert markdown.index("## Noise / Layout Signals") < markdown.index("## Check Results") assert "table_marker_artifacts: 1" in markdown assert "## Required Field Coverage" in markdown + assert "## Table Output Structure Signals" in markdown assert "## Text Usefulness" in markdown assert "## Text Extraction Health" in markdown + assert "Table Output Structure Signals checks whether table-labeled normalized blocks contain visible" in markdown assert "Text Extraction Health checks extracted-text availability, not extracted-text correctness." in markdown @@ -269,3 +369,22 @@ def test_cli_exits_zero_when_only_text_extraction_health_warns(tmp_path: Path) - assert "decision: REVIEW" in report_text assert "## Text Extraction Health" in report_text assert "no_text_bearing_blocks: document has blocks but no text-bearing blocks" in report_text + + +def test_cli_exits_zero_when_only_table_output_structure_warns(tmp_path: Path) -> None: + uif_path = tmp_path / "uif.json" + report_path = tmp_path / "quality_report.md" + table = _table_block( + "p6-tables-1", + text="This table-labeled block appears as one plain paragraph without obvious columns.", + data_grid=[], + ) + uif_path.write_text(json.dumps({"metadata": {}, "blocks": [table]}), encoding="utf-8") + + assert main(["--input", str(uif_path), "--output", str(report_path)]) == 0 + + report_text = report_path.read_text(encoding="utf-8") + assert "hard_failures: 0" in report_text + assert "decision: REVIEW" in report_text + assert "## Table Output Structure Signals" in report_text + assert "plain text only and no obvious row/column structure signal" in report_text