test_pdf.py

import re
from pprint import pprint

from fitz import Document

from remarks.output.ObsidianMarkdownFile import merge_highlights
from remarks.output.PdfFile import extract_annot
from tests.notebook_fixtures import *
from tests.pdf_test_support import assert_page_renders_without_warnings, assert_warning_exists

r"""
 _____  _____  ______
|  __ \|  __ \|  ____|
| |__) | |  | | |__
|  ___/| |  | |  __|
| |    | |__| | |
|_|    |_____/|_|
"""
@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_valid_pdf(notebook: NotebookMetadata, remarks_document: Document):
    assert remarks_document.is_pdf


@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_correct_output_page_count(notebook: NotebookMetadata, remarks_document: Document):
    assert remarks_document.page_count == notebook.pdf_pages


@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_warnings_match_specification(notebook: NotebookMetadata, remarks_document: Document):
    for page in notebook.pages:
        if page.warnings:
            for warning in page.warnings:
                assert_warning_exists(remarks_document, page.pdf_document_index, warning)
        else:
            # If no warnings specified for this page, verify page is clean
            assert_page_renders_without_warnings(remarks_document, page.pdf_document_index)


@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_smart_highlights(notebook: NotebookMetadata, remarks_document: Document):
    for page_metadata in notebook.pages:
        if page_metadata.raw_highlights:
            if scrybble_warning_typed_text_highlighting_not_supported in page_metadata.warnings:
                continue
            document_page = remarks_document[page_metadata.pdf_document_index]
            words_on_page = document_page.get_textpage().extractWORDS()
            annots = list(document_page.annots())

            # sort by reading-order
            annots.sort(key=lambda a: (a.rect.y0, a.rect.x0))
            assert len(annots) == len(page_metadata.raw_highlights)
            for i, annotation in enumerate(annots):
                text = extract_annot(annotation, words_on_page)
                assert text == page_metadata.raw_highlights[i]
                # TODO: We should implement the colour check as well, once that is ready.


def demarkdown(markdown_text: str):
    """Takes in a Markdown string, and gets rid of all Markdown, essentially returning pure plaintext"""

    # Process the Markdown text to remove formatting
    text = markdown_text

    # Remove headers (# Header)
    text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)

    # Remove bold/italic formatting
    text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', text)  # Bold
    text = re.sub(r'([*_])(.*?)\1', r'\2', text)  # Italic

    # Remove code blocks and inline code
    text = re.sub(r'```[\s\S]*?```', '', text)  # Code blocks
    text = re.sub(r'`([^`]+)`', r'\1', text)  # Inline code

    # Remove blockquotes
    text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE)

    # Remove horizontal rules
    text = re.sub(r'^\s*[-*_]{3,}\s*$', '', text, flags=re.MULTILINE)

    # Remove links - [text](url) -> text
    text = re.sub(r'\[([^]]+)]\([^)]+\)', r'\1', text)

    # Remove image syntax - ![alt](url) -> alt
    text = re.sub(r'!\[([^]]+)]\([^)]+\)', r'\1', text)

    # Remove HTML tags
    text = re.sub(r'<[^>]+>', '', text)

    # Remove ordered/unordered list markers
    text = re.sub(r'^\s*[*\-+]\s+', '', text, flags=re.MULTILINE)  # Unordered lists
    text = re.sub(r'^\s*\d+\.\s+', '', text, flags=re.MULTILINE)  # Ordered lists

    # Clean up extra whitespace
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = text.strip()

    return text



@pytest.mark.pdf
@pytest.mark.unfinished_feature
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_typed_text_is_readable(notebook: NotebookMetadata, remarks_document: Document):
    pages_with_typed_text = list(filter(lambda x: x.typed_text is not None, notebook.pages))

    if pages_with_typed_text:
        for page in pages_with_typed_text:
            document_page = remarks_document[page.pdf_document_index]
            plaintext = demarkdown(page.typed_text)
            for line in document_page.get_textpage().extractText().splitlines():
                assert line in plaintext


r"""
 _____   ____ _______    _______ _____ ____  _   _
|  __ \ / __ \__   __|/\|__   __|_   _/ __ \| \ | |
| |__) | |  | | | |  /  \  | |    | || |  | |  \| |
|  _  /| |  | | | | / /\ \ | |    | || |  | | . ` |
| | \ \| |__| | | |/ ____ \| |   _| || |__| | |\  |
|_|  \_\\____/  |_/_/    \_\_|  |_____\____/|_| \_|
"""


@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_output_pdf_has_no_rotation_metadata(notebook: NotebookMetadata, remarks_document: Document):
    """Verify that all pages in output PDFs have rotation=0.

    After processing by remarks, all rotation should be "baked in" to the page content,
    meaning the PDF rotation metadata should always be 0.
    """
    for page_idx in range(remarks_document.page_count):
        page = remarks_document[page_idx]
        assert page.rotation == 0, f"Page {page_idx} has rotation {page.rotation}, expected 0"


@pytest.mark.pdf
@pytest.mark.parametrize("notebook", all_notebooks, indirect=True)
def test_output_pdf_has_correct_dimensions(notebook: NotebookMetadata, remarks_document: Document):
    """Verify that output PDFs have correctly transformed dimensions.

    For documents with a source PDF:
    - 90/270 degree rotations should swap width and height
    - 0/180 degree rotations should preserve dimensions

    This test ensures remarks correctly handles PDF rotation metadata across all input documents.
    """
    source_pdf = notebook.get_source_pdf()
    if source_pdf is None:
        # Notebook-type documents don't have a source PDF, skip dimension check
        return

    for page_idx in range(min(source_pdf.page_count, remarks_document.page_count)):
        source_page = source_pdf[page_idx]
        output_page = remarks_document[page_idx]

        source_rotation = source_page.rotation
        source_mediabox = source_page.mediabox

        # Get the "natural" dimensions from mediabox (before rotation is applied)
        source_width = source_mediabox.width
        source_height = source_mediabox.height

        output_width = output_page.rect.width
        output_height = output_page.rect.height

        # For 90/270 rotations, dimensions should be swapped
        # For 0/180 rotations, dimensions should match
        if source_rotation in [90, 270]:
            assert abs(output_width - source_height) < 1, \
                f"Page {page_idx}: for {source_rotation}° rotation, output width ({output_width}) should match source height ({source_height})"
            assert abs(output_height - source_width) < 1, \
                f"Page {page_idx}: for {source_rotation}° rotation, output height ({output_height}) should match source width ({source_width})"
        else:
            assert abs(output_width - source_width) < 1, \
                f"Page {page_idx}: for {source_rotation}° rotation, output width ({output_width}) should match source width ({source_width})"
            assert abs(output_height - source_height) < 1, \
                f"Page {page_idx}: for {source_rotation}° rotation, output height ({output_height}) should match source height ({source_height})"