diff --git a/docling_core/types/doc/__init__.py b/docling_core/types/doc/__init__.py index 4625f7402..380ef1fa3 100644 --- a/docling_core/types/doc/__init__.py +++ b/docling_core/types/doc/__init__.py @@ -21,6 +21,10 @@ BaseMeta, BasePrediction, CodeMetaField, + DataPointDirection, + DataPointMention, + DataPointPrecision, + DataPointScale, DescriptionMetaField, EntitiesMetaField, EntityMention, diff --git a/docling_core/types/doc/common/meta.py b/docling_core/types/doc/common/meta.py index fd7be4238..2dabaabae 100644 --- a/docling_core/types/doc/common/meta.py +++ b/docling_core/types/doc/common/meta.py @@ -1,7 +1,7 @@ """Metadata models attached to document nodes.""" from enum import Enum -from typing import Annotated, Any, Final, Optional +from typing import Annotated, Any, Final, Literal, Optional, Union from pydantic import ( BaseModel, @@ -10,6 +10,7 @@ Field, FieldSerializationInfo, field_serializer, + field_validator, model_validator, ) from typing_extensions import Self @@ -159,10 +160,159 @@ class EntityMention(BasePrediction): ] = None +DataPointPrecision = Literal["exact", "approximate", "lower_bound", "upper_bound", "range_low", "range_high"] +"""Valid values for DataPointMention.precision.""" + +DataPointDirection = Literal["increase", "decrease", "neutral"] +"""Valid values for DataPointMention.direction.""" + +DataPointScale = Literal[ + "hundred", + "thousand", + "k", + "million", + "m", + "billion", + "b", + "bn", + "trillion", + "t", + "quadrillion", +] +"""Valid values for DataPointMention.scale. Each value maps to a known numeric multiplier.""" + + +class DataPointMention(EntityMention): + """A quantitative data point mentioned in text, extending EntityMention with a numeric breakdown. + + Inherited fields: text (normalised form), orig (source text), label (category), + charspan, confidence, created_by. + """ + + value: Annotated[ + float, + Field( + description="Numeric value as written, before scale application. E.g. 4.0 for '$4B'.", + examples=[4.0, 30.0, 3.0], + ), + ] + + unit: Annotated[ + Optional[str], + Field( + description="Dimensional unit, e.g. 'USD', '°C', '%'. ISO 4217 recommended for currencies.", + examples=["USD", "°C", "%"], + ), + ] = None + + scale: Annotated[ + Optional[DataPointScale], + Field( + description="Magnitude multiplier from the source text, e.g. 'billion', 'million', 'k'.", + examples=["billion", "million", "k"], + ), + ] = None + + normalized_value: Annotated[ + Optional[float], + Field( + description="value * scale_factor in base units, e.g. 4e9 for '$4B'. Derived convenience field.", + examples=[4_000_000_000.0, 30.0], + ), + ] = None + + range_end: Annotated[ + Optional[float], + Field( + description=( + "Upper bound when the data point expresses a range, e.g. 20.0 for 'between 10 and 20°C'. " + "Use with precision='range_low'; value holds the lower bound." + ), + examples=[20.0], + ), + ] = None + + display_dp: Annotated[ + Optional[int], + Field( + ge=0, + description=( + "Decimal places as written in the source: 0 for '3%', 1 for '3.0%', 2 for '3.00%'. " + "None when not determinable." + ), + examples=[0, 1, 2], + ), + ] = None + + precision: Annotated[ + Optional[DataPointPrecision], + Field( + description=( + "Author's epistemic qualification: 'exact', 'approximate', " + "'lower_bound', 'upper_bound', 'range_low', 'range_high'." + ), + examples=["approximate", "lower_bound"], + ), + ] = None + + direction: Annotated[ + Optional[DataPointDirection], + Field( + description=( + "Direction of change for delta/growth values: 'increase', 'decrease', 'neutral'. " + "E.g. 'revenue grew 3%' → 'increase'." + ), + examples=["increase", "decrease"], + ), + ] = None + + @field_validator("range_end", mode="after") + @classmethod + def _validate_range_end(cls, v: Optional[float], info: Any) -> Optional[float]: + if v is not None and info.data.get("value") is None: + raise ValueError("range_end requires value to be set") + return v + + @property + def scale_factor(self) -> Optional[float]: + """Numeric multiplier for scale, or None if absent or unrecognised.""" + if self.scale is None: + return None + _SCALE_MAP: dict[str, float] = { + "hundred": 1e2, + "thousand": 1e3, + "k": 1e3, + "million": 1e6, + "m": 1e6, + "billion": 1e9, + "b": 1e9, + "bn": 1e9, + "trillion": 1e12, + "t": 1e12, + "quadrillion": 1e15, + } + return _SCALE_MAP.get(self.scale) + + def compute_normalized_value(self) -> Optional[float]: + """Return value * scale_factor without caching. + + Returns None when scale is present but unrecognised. + Use this to derive or refresh normalized_value after construction. + """ + factor = self.scale_factor + if factor is None and self.scale is not None: + return None + return self.value * (factor if factor is not None else 1.0) + + class EntitiesMetaField(_ExtraAllowingModel): - """Container for extracted entity mentions.""" + """Container for extracted entity mentions. + + The mentions list accepts both plain EntityMention objects and the richer + DataPointMention subclass; use isinstance(m, DataPointMention) to filter. + """ - mentions: Annotated[list[EntityMention], Field(min_length=1)] + mentions: Annotated[list[Union[DataPointMention, EntityMention]], Field(min_length=1)] class KeywordsMetaField(_ExtraAllowingModel): diff --git a/docs/DoclingDocument.json b/docs/DoclingDocument.json index 74f4d839d..3182d6670 100644 --- a/docs/DoclingDocument.json +++ b/docs/DoclingDocument.json @@ -590,6 +590,266 @@ "title": "CoordOrigin", "type": "string" }, + "DataPointMention": { + "additionalProperties": true, + "description": "A quantitative data point mentioned in text, extending EntityMention with a numeric breakdown.\n\nInherited fields: text (normalised form), orig (source text), label (category),\ncharspan, confidence, created_by.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "description": "Normalized text of the entity mention.", + "title": "Text", + "type": "string" + }, + "orig": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact source text extracted from the original charspan, analogous to TextItem.orig. This may differ from 'text' when the mention has been normalized.", + "title": "Orig" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Entity type or category.", + "title": "Label" + }, + "charspan": { + "anyOf": [ + { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Character span (0-indexed) of the entity mention in the source text.", + "title": "Charspan" + }, + "value": { + "description": "Numeric value as written, before scale application. E.g. 4.0 for '$4B'.", + "examples": [ + 4.0, + 30.0, + 3.0 + ], + "title": "Value", + "type": "number" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dimensional unit, e.g. 'USD', '°C', '%'. ISO 4217 recommended for currencies.", + "examples": [ + "USD", + "°C", + "%" + ], + "title": "Unit" + }, + "scale": { + "anyOf": [ + { + "enum": [ + "hundred", + "thousand", + "k", + "million", + "m", + "billion", + "b", + "bn", + "trillion", + "t", + "quadrillion" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Magnitude multiplier from the source text, e.g. 'billion', 'million', 'k'.", + "examples": [ + "billion", + "million", + "k" + ], + "title": "Scale" + }, + "normalized_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "value * scale_factor in base units, e.g. 4e9 for '$4B'. Derived convenience field.", + "examples": [ + 4000000000.0, + 30.0 + ], + "title": "Normalized Value" + }, + "range_end": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Upper bound when the data point expresses a range, e.g. 20.0 for 'between 10 and 20°C'. Use with precision='range_low'; value holds the lower bound.", + "examples": [ + 20.0 + ], + "title": "Range End" + }, + "display_dp": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Decimal places as written in the source: 0 for '3%', 1 for '3.0%', 2 for '3.00%'. None when not determinable.", + "examples": [ + 0, + 1, + 2 + ], + "title": "Display Dp" + }, + "precision": { + "anyOf": [ + { + "enum": [ + "exact", + "approximate", + "lower_bound", + "upper_bound", + "range_low", + "range_high" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Author's epistemic qualification: 'exact', 'approximate', 'lower_bound', 'upper_bound', 'range_low', 'range_high'.", + "examples": [ + "approximate", + "lower_bound" + ], + "title": "Precision" + }, + "direction": { + "anyOf": [ + { + "enum": [ + "increase", + "decrease", + "neutral" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Direction of change for delta/growth values: 'increase', 'decrease', 'neutral'. E.g. 'revenue grew 3%' → 'increase'.", + "examples": [ + "increase", + "decrease" + ], + "title": "Direction" + } + }, + "required": [ + "text", + "value" + ], + "title": "DataPointMention", + "type": "object" + }, "DescriptionAnnotation": { "description": "DescriptionAnnotation.", "properties": { @@ -707,11 +967,18 @@ }, "EntitiesMetaField": { "additionalProperties": true, - "description": "Container for extracted entity mentions.", + "description": "Container for extracted entity mentions.\n\nThe mentions list accepts both plain EntityMention objects and the richer\nDataPointMention subclass; use isinstance(m, DataPointMention) to filter.", "properties": { "mentions": { "items": { - "$ref": "#/$defs/EntityMention" + "anyOf": [ + { + "$ref": "#/$defs/DataPointMention" + }, + { + "$ref": "#/$defs/EntityMention" + } + ] }, "minItems": 1, "title": "Mentions", diff --git a/test/data/doc/meta/entity_mentions.dt b/test/data/doc/meta/entity_mentions.dt new file mode 100644 index 000000000..38100bb1f --- /dev/null +++ b/test/data/doc/meta/entity_mentions.dt @@ -0,0 +1,4 @@ +IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year. +The population of Switzerland in 2026 is approximately 9 million people. +Operating margins improved to between 18% and 21% during the period. + \ No newline at end of file diff --git a/test/data/doc/meta/entity_mentions.html b/test/data/doc/meta/entity_mentions.html new file mode 100644 index 000000000..078d8aea6 --- /dev/null +++ b/test/data/doc/meta/entity_mentions.html @@ -0,0 +1,184 @@ + + + + +entity-mentions-example + + + + +
+

IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year.

+
Meta
entities: IBM (ORG, [0,3]), Q3 2025 (DATE, [34,41]), 16.3 billion USD (REVENUE, [24,30]), 9 percent (GROWTH_RATE, [46,48])
+

The population of Switzerland in 2026 is approximately 9 million people.

+
Meta
entities: Switzerland (LOC, [18,29]), 2026 (DATE, [33,37]), 9 million (POPULATION, [55,64])
+

Operating margins improved to between 18% and 21% during the period.

+
Meta
entities: 18 to 21 percent (OPERATING_MARGIN, [37,40])
+
+ + \ No newline at end of file diff --git a/test/data/doc/meta/entity_mentions.json b/test/data/doc/meta/entity_mentions.json new file mode 100644 index 000000000..126e65379 --- /dev/null +++ b/test/data/doc/meta/entity_mentions.json @@ -0,0 +1,180 @@ +{ + "schema_name": "DoclingDocument", + "version": "1.10.0", + "name": "entity-mentions-example", + "furniture": { + "self_ref": "#/furniture", + "children": [], + "content_layer": "furniture", + "name": "_root_", + "label": "unspecified" + }, + "body": { + "self_ref": "#/body", + "children": [ + { + "$ref": "#/texts/0" + }, + { + "$ref": "#/texts/1" + }, + { + "$ref": "#/texts/2" + } + ], + "content_layer": "body", + "name": "_root_", + "label": "unspecified" + }, + "groups": [], + "texts": [ + { + "self_ref": "#/texts/0", + "parent": { + "$ref": "#/body" + }, + "children": [], + "content_layer": "body", + "meta": { + "entities": { + "mentions": [ + { + "text": "IBM", + "label": "ORG", + "charspan": [ + 0, + 3 + ] + }, + { + "text": "Q3 2025", + "label": "DATE", + "charspan": [ + 34, + 41 + ] + }, + { + "text": "16.3 billion USD", + "orig": "$16.3B", + "label": "REVENUE", + "charspan": [ + 24, + 30 + ], + "value": 16.3, + "unit": "USD", + "scale": "billion", + "normalized_value": 16300000000.0, + "display_dp": 1, + "precision": "exact" + }, + { + "text": "9 percent", + "orig": "9%", + "label": "GROWTH_RATE", + "charspan": [ + 46, + 48 + ], + "value": 9.0, + "unit": "%", + "display_dp": 0, + "precision": "exact", + "direction": "increase" + } + ] + } + }, + "label": "text", + "prov": [], + "orig": "IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year.", + "text": "IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year." + }, + { + "self_ref": "#/texts/1", + "parent": { + "$ref": "#/body" + }, + "children": [], + "content_layer": "body", + "meta": { + "entities": { + "mentions": [ + { + "text": "Switzerland", + "label": "LOC", + "charspan": [ + 18, + 29 + ] + }, + { + "text": "2026", + "label": "DATE", + "charspan": [ + 33, + 37 + ] + }, + { + "text": "9 million", + "label": "POPULATION", + "charspan": [ + 55, + 64 + ], + "value": 9.0, + "scale": "million", + "normalized_value": 9000000.0, + "display_dp": 0, + "precision": "approximate" + } + ] + } + }, + "label": "text", + "prov": [], + "orig": "The population of Switzerland in 2026 is approximately 9 million people.", + "text": "The population of Switzerland in 2026 is approximately 9 million people." + }, + { + "self_ref": "#/texts/2", + "parent": { + "$ref": "#/body" + }, + "children": [], + "content_layer": "body", + "meta": { + "entities": { + "mentions": [ + { + "text": "18 to 21 percent", + "orig": "18%", + "label": "OPERATING_MARGIN", + "charspan": [ + 37, + 40 + ], + "value": 18.0, + "unit": "%", + "range_end": 21.0, + "display_dp": 0, + "precision": "range_low", + "direction": "increase" + } + ] + } + }, + "label": "text", + "prov": [], + "orig": "Operating margins improved to between 18% and 21% during the period.", + "text": "Operating margins improved to between 18% and 21% during the period." + } + ], + "pictures": [], + "tables": [], + "key_value_items": [], + "form_items": [], + "pages": {} +} \ No newline at end of file diff --git a/test/data/doc/meta/entity_mentions.md b/test/data/doc/meta/entity_mentions.md new file mode 100644 index 000000000..a50646676 --- /dev/null +++ b/test/data/doc/meta/entity_mentions.md @@ -0,0 +1,11 @@ +IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year. + +[Entities] mentions=[EntityMention(confidence=None, created_by=None, text='IBM', orig=None, label='ORG', charspan=(0, 3)), EntityMention(confidence=None, created_by=None, text='Q3 2025', orig=None, label='DATE', charspan=(34, 41)), DataPointMention(confidence=None, created_by=None, text='16.3 billion USD', orig='$16.3B', label='REVENUE', charspan=(24, 30), value=16.3, unit='USD', scale='billion', normalized_value=16300000000.0, range_end=None, display_dp=1, precision='exact', direction=None), DataPointMention(confidence=None, created_by=None, text='9 percent', orig='9%', label='GROWTH_RATE', charspan=(46, 48), value=9.0, unit='%', scale=None, normalized_value=None, range_end=None, display_dp=0, precision='exact', direction='increase')] + +The population of Switzerland in 2026 is approximately 9 million people. + +[Entities] mentions=[EntityMention(confidence=None, created_by=None, text='Switzerland', orig=None, label='LOC', charspan=(18, 29)), EntityMention(confidence=None, created_by=None, text='2026', orig=None, label='DATE', charspan=(33, 37)), DataPointMention(confidence=None, created_by=None, text='9 million', orig=None, label='POPULATION', charspan=(55, 64), value=9.0, unit=None, scale='million', normalized_value=9000000.0, range_end=None, display_dp=0, precision='approximate', direction=None)] + +Operating margins improved to between 18% and 21% during the period. + +[Entities] mentions=[DataPointMention(confidence=None, created_by=None, text='18 to 21 percent', orig='18%', label='OPERATING_MARGIN', charspan=(37, 40), value=18.0, unit='%', scale=None, normalized_value=None, range_end=21.0, display_dp=0, precision='range_low', direction='increase')] \ No newline at end of file diff --git a/test/test_metadata.py b/test/test_metadata.py index 45abf8a3f..7870694fd 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -15,6 +15,7 @@ ) from docling_core.types.doc import ( BaseMeta, + DataPointMention, DocItem, DocItemLabel, DoclingDocument, @@ -379,3 +380,269 @@ def test_keywords_topics_required_values() -> None: TopicsMetaField(values=34) with pytest.raises(ValidationError, match="valid string"): TopicsMetaField(values=[34]) + + +# --------------------------------------------------------------------------- +# Entity mentions tests +# --------------------------------------------------------------------------- + +_META_GT_DIR = Path("test/data/doc/meta") + + +@pytest.fixture(scope="module") +def doc_with_entity_mentions() -> DoclingDocument: + """A document with three paragraphs annotated with mixed entity mentions. + + Paragraph 1 — financial statement: + "IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year." + Mentions: ORG, DATE, REVENUE (DataPointMention), GROWTH_RATE (DataPointMention) + + Paragraph 2 — population fact: + "The population of Switzerland in 2026 is approximately 9 million people." + Mentions: LOC, DATE, HEADCOUNT (DataPointMention) + + Paragraph 3 — range data point: + "Operating margins improved to between 18% and 21% during the period." + Mentions: OPERATING_MARGIN (DataPointMention, range) + """ + doc = DoclingDocument(name="entity-mentions-example") + + # ── Paragraph 1 ────────────────────────────────────────────────────────── + p1 = doc.add_text( + label=DocItemLabel.TEXT, + text="IBM reported revenue of $16.3B in Q3 2025, up 9% year-over-year.", + ) + p1.meta = BaseMeta( + entities=EntitiesMetaField( + mentions=[ + EntityMention(text="IBM", label="ORG", charspan=(0, 3)), + EntityMention(text="Q3 2025", label="DATE", charspan=(34, 41)), + DataPointMention( + text="16.3 billion USD", + orig="$16.3B", + label="REVENUE", + charspan=(24, 30), + value=16.3, + unit="USD", + scale="billion", + normalized_value=16_300_000_000.0, + display_dp=1, + precision="exact", + ), + DataPointMention( + text="9 percent", + orig="9%", + label="GROWTH_RATE", + charspan=(46, 48), + value=9.0, + unit="%", + display_dp=0, + precision="exact", + direction="increase", + ), + ] + ) + ) + + # ── Paragraph 2 ────────────────────────────────────────────────────────── + p2 = doc.add_text( + label=DocItemLabel.TEXT, + text="The population of Switzerland in 2026 is approximately 9 million people.", + ) + p2.meta = BaseMeta( + entities=EntitiesMetaField( + mentions=[ + EntityMention(text="Switzerland", label="LOC", charspan=(18, 29)), + EntityMention(text="2026", label="DATE", charspan=(33, 37)), + DataPointMention( + text="9 million", + label="POPULATION", + charspan=(55, 64), + value=9.0, + scale="million", + normalized_value=9_000_000.0, + display_dp=0, + precision="approximate", + ), + ] + ) + ) + + # ── Paragraph 3 ────────────────────────────────────────────────────────── + p3 = doc.add_text( + label=DocItemLabel.TEXT, + text="Operating margins improved to between 18% and 21% during the period.", + ) + p3.meta = BaseMeta( + entities=EntitiesMetaField( + mentions=[ + DataPointMention( + text="18 to 21 percent", + orig="18%", + label="OPERATING_MARGIN", + charspan=(37, 40), + value=18.0, + unit="%", + display_dp=0, + precision="range_low", + range_end=21.0, + direction="increase", + ), + ] + ) + ) + + return doc + + +def test_data_point_roundtrip_and_html_rendering( + doc_with_entity_mentions: DoclingDocument, +) -> None: + doc = doc_with_entity_mentions + + roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json")) + meta_p1 = roundtrip.texts[0].meta + assert meta_p1 is not None and meta_p1.entities is not None + assert meta_p1.has_content() + + mentions = meta_p1.entities.mentions + # plain EntityMention (ORG) has no value field — falls back to base type in Union + org = mentions[0] + assert isinstance(org, EntityMention) + assert not isinstance(org, DataPointMention) + + # DataPointMention (REVENUE) — exact, scaled value + revenue = mentions[2] + assert isinstance(revenue, DataPointMention) + assert revenue.orig == "$16.3B" + assert revenue.value == 16.3 + assert revenue.unit == "USD" + assert revenue.scale == "billion" + assert revenue.normalized_value == 16_300_000_000.0 + assert revenue.display_dp == 1 + assert revenue.precision == "exact" + assert revenue.scale_factor == 1e9 + assert revenue.compute_normalized_value() == revenue.normalized_value + + html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text + assert 'data-meta-name="entities"' in html + assert "IBM (ORG, [0,3])" in html + assert "16.3 billion USD (REVENUE, [24,30])" in html + + +def test_data_point_range_and_direction( + doc_with_entity_mentions: DoclingDocument, +) -> None: + doc = doc_with_entity_mentions + + roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json")) + + # GROWTH_RATE from paragraph 1 — direction + exact precision + meta_p1 = roundtrip.texts[0].meta + assert meta_p1 is not None and meta_p1.entities is not None + growth = meta_p1.entities.mentions[3] + assert isinstance(growth, DataPointMention) + assert growth.value == 9.0 + assert growth.unit == "%" + assert growth.display_dp == 0 + assert growth.precision == "exact" + assert growth.direction == "increase" + + # OPERATING_MARGIN from paragraph 3 — range_low + range_end + meta_p3 = roundtrip.texts[2].meta + assert meta_p3 is not None and meta_p3.entities is not None + margin = meta_p3.entities.mentions[0] + assert isinstance(margin, DataPointMention) + assert margin.value == 18.0 + assert margin.range_end == 21.0 + assert margin.precision == "range_low" + assert margin.display_dp == 0 + + +def test_data_point_numeric_helpers() -> None: + # scale_factor maps canonical scale strings to their multipliers + assert DataPointMention(text="4B", value=4.0, scale="billion").scale_factor == 1e9 + assert DataPointMention(text="5k", value=5.0, scale="k").scale_factor == 1e3 + assert DataPointMention(text="x", value=0.0).scale_factor is None + + # compute_normalized_value applies the scale factor on demand + assert DataPointMention(text="4B", value=4.0, scale="billion").compute_normalized_value() == 4_000_000_000.0 + assert DataPointMention(text="30", value=30.0).compute_normalized_value() == 30.0 + # display_dp is independent of the numeric value: "3%" and "3.0%" share value=3.0 + assert DataPointMention(text="3%", value=3.0, display_dp=0).display_dp == 0 + assert DataPointMention(text="3.0%", value=3.0, display_dp=1).display_dp == 1 + + +def test_data_point_range_end_requires_value() -> None: + with pytest.raises(ValidationError, match="range_end requires value"): + DataPointMention.model_validate({"text": "x", "value": None, "range_end": 20.0}) + + +def test_entity_mentions_json_roundtrip( + doc_with_entity_mentions: DoclingDocument, +) -> None: + """Ground truth: JSON serialization and structural round-trip.""" + import json + + doc = doc_with_entity_mentions + gt_path = _META_GT_DIR / "entity_mentions.json" + + result = json.dumps(doc.export_to_dict(), indent=2) + assert_or_generate_ground_truth(result, gt_path) + + # Structural round-trip: reload from ground-truth JSON and verify mentions + reloaded = DoclingDocument.model_validate_json(gt_path.read_text()) + + meta_p1 = reloaded.texts[0].meta + assert meta_p1 is not None and meta_p1.entities is not None + mentions_p1 = meta_p1.entities.mentions + assert isinstance(mentions_p1[0], EntityMention) and not isinstance(mentions_p1[0], DataPointMention) + assert isinstance(mentions_p1[2], DataPointMention) + assert mentions_p1[2].value == 16.3 + assert mentions_p1[2].normalized_value == 16_300_000_000.0 + + meta_p2 = reloaded.texts[1].meta + assert meta_p2 is not None and meta_p2.entities is not None + mentions_p2 = meta_p2.entities.mentions + assert isinstance(mentions_p2[2], DataPointMention) + assert mentions_p2[2].precision == "approximate" + + meta_p3 = reloaded.texts[2].meta + assert meta_p3 is not None and meta_p3.entities is not None + mentions_p3 = meta_p3.entities.mentions + assert isinstance(mentions_p3[0], DataPointMention) + assert mentions_p3[0].range_end == 21.0 + assert mentions_p3[0].precision == "range_low" + + +def test_entity_mentions_markdown_serialization( + doc_with_entity_mentions: DoclingDocument, +) -> None: + """Ground truth: Markdown serialization (meta marked).""" + doc = doc_with_entity_mentions + gt_path = _META_GT_DIR / "entity_mentions.md" + + result = MarkdownDocSerializer(doc=doc, params=MarkdownParams(mark_meta=True)).serialize().text + assert_or_generate_ground_truth(result, gt_path) + + +def test_entity_mentions_html_serialization( + doc_with_entity_mentions: DoclingDocument, +) -> None: + """Ground truth: HTML serialization.""" + doc = doc_with_entity_mentions + gt_path = _META_GT_DIR / "entity_mentions.html" + + result = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text + assert_or_generate_ground_truth(result, gt_path) + + +def test_entity_mentions_doctags_serialization( + doc_with_entity_mentions: DoclingDocument, +) -> None: + """Ground truth: DocTags (Doclang) serialization.""" + doc = doc_with_entity_mentions + gt_path = _META_GT_DIR / "entity_mentions.dt" + + result = doc.export_to_doctags(add_location=False) + assert_or_generate_ground_truth(result, gt_path)