diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 066788d23..089a9a03e 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Annotated, Any, Optional, Union -from pydantic import AnyUrl, BaseModel, Field, PositiveInt +from pydantic import AnyUrl, BaseModel, Field, PositiveInt, model_validator from tabulate import _column_type, tabulate from typing_extensions import override @@ -144,6 +144,14 @@ class OrigListItemMarkerMode(str, Enum): AUTO = "auto" +class FurnitureMode(str, Enum): + """Display mode for document furniture (headers/footers).""" + + NONE = "none" + ALL = "all" + DISTINCT = "distinct" + + class MarkdownParams(CommonParams): """Markdown-specific serialization parameters.""" @@ -183,6 +191,17 @@ class MarkdownParams(CommonParams): ) ), ] = False + furniture_mode: FurnitureMode = Field( + default=FurnitureMode.NONE, + description="Control whether headers and footers are serialized (none, all, or distinct).", + ) + + @model_validator(mode="after") + def _adjust_layers_for_furniture(self) -> "MarkdownParams": + """Automatically include FURNITURE layer if mode is not NONE.""" + if self.furniture_mode != FurnitureMode.NONE: + self.layers.add(ContentLayer.FURNITURE) + return self class MarkdownTextSerializer(BaseModel, BaseTextSerializer): @@ -958,6 +977,30 @@ def serialize( **kwargs: Any, ) -> SerializationResult: """Serialize a given node.""" + + if item is not None and getattr(item, "content_layer", None) == ContentLayer.FURNITURE: + # 'none' mode: skip completely + if self.params.furniture_mode == FurnitureMode.NONE: + return create_ser_result() + + # 'distinct' mode: deduplicate based on text + if self.params.furniture_mode == FurnitureMode.DISTINCT: + # Initialize the tracking set on the serializer instance if it doesn't exist + if not hasattr(self, "_seen_furniture"): + self._seen_furniture: set[str] = set() + + # Extract text for comparison (fallback to ref if no text exists) + item_text = getattr(item, "text", None) + if not item_text: + item_text = item.self_ref + + # If we've seen this exact header/footer text before, skip it + if item_text in self._seen_furniture: + return create_ser_result() + + # Otherwise, mark it as seen and proceed + self._seen_furniture.add(item_text) + return super().serialize( item=item, list_level=list_level, diff --git a/test/test_serialization.py b/test/test_serialization.py index b6bd4b1db..a58c7fd7a 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -16,6 +16,7 @@ HTMLTableSerializer, ) from docling_core.transforms.serializer.markdown import ( + FurnitureMode, MarkdownDocSerializer, MarkdownParams, MarkdownTableSerializer, @@ -29,6 +30,7 @@ from docling_core.types.doc.document import ( BaseMeta, CharSpan, + ContentLayer, DescriptionAnnotation, EntitiesMetaField, EntityMention, @@ -612,6 +614,44 @@ def test_md_traverse_pictures(): assert "" in result_with_traverse +def test_md_furniture_modes(): + """Test that furniture_mode correctly filters and deduplicates headers/footers.""" + doc = DoclingDocument(name="Furniture Test") + + # 1. Add standard body text + doc.add_text(label=DocItemLabel.PARAGRAPH, text="Main body paragraph text.") + + # 2. Add headers/footers and tag them as FURNITURE layer items + h1 = doc.add_text(label=DocItemLabel.TEXT, text="Document Header Title") + h1.content_layer = ContentLayer.FURNITURE + + h2 = doc.add_text(label=DocItemLabel.TEXT, text="Document Header Title") # Duplicate text + h2.content_layer = ContentLayer.FURNITURE + + h3 = doc.add_text(label=DocItemLabel.TEXT, text="Different Footer Notice") + h3.content_layer = ContentLayer.FURNITURE + + # Test Mode 1: NONE (Default) -> Should completely drop furniture items + ser_none = MarkdownDocSerializer(doc=doc, params=MarkdownParams(furniture_mode=FurnitureMode.NONE)) + text_none = ser_none.serialize().text + assert "Main body paragraph text." in text_none + assert "Document Header Title" not in text_none + + # Test Mode 2: ALL -> Should render every furniture item, including duplicates + ser_all = MarkdownDocSerializer(doc=doc, params=MarkdownParams(furniture_mode=FurnitureMode.ALL)) + text_all = ser_all.serialize().text + assert "Main body paragraph text." in text_all + assert text_all.count("Document Header Title") == 2 + assert "Different Footer Notice" in text_all + + # Test Mode 3: DISTINCT -> Should render unique text items exactly once + ser_distinct = MarkdownDocSerializer(doc=doc, params=MarkdownParams(furniture_mode=FurnitureMode.DISTINCT)) + text_distinct = ser_distinct.serialize().text + assert "Main body paragraph text." in text_distinct + assert text_distinct.count("Document Header Title") == 1 + assert "Different Footer Notice" in text_distinct + + # =============================== # HTML tests # ===============================