Skip to content

Commit 6087c26

Browse files
committed
feat(meta): add DataPointMention for quantitative data points mentioned in text
Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
1 parent a6b6023 commit 6087c26

4 files changed

Lines changed: 561 additions & 4 deletions

File tree

docling_core/types/doc/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
ChartStackedBar,
1414
CodeItem,
1515
ContentLayer,
16+
DataPointDirection,
17+
DataPointMention,
18+
DataPointPrecision,
19+
DataPointScale,
1620
DescriptionAnnotation,
1721
DescriptionMetaField,
1822
DocItem,

docling_core/types/doc/document.py

Lines changed: 153 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1547,10 +1547,161 @@ class EntityMention(BasePrediction):
15471547
] = None
15481548

15491549

1550+
DataPointPrecision = Literal["exact", "approximate", "lower_bound", "upper_bound", "range_low", "range_high"]
1551+
"""Valid values for DataPointMention.precision."""
1552+
1553+
DataPointDirection = Literal["increase", "decrease", "neutral"]
1554+
"""Valid values for DataPointMention.direction."""
1555+
1556+
DataPointScale = Literal[
1557+
"hundred",
1558+
"thousand",
1559+
"k",
1560+
"million",
1561+
"m",
1562+
"billion",
1563+
"b",
1564+
"bn",
1565+
"trillion",
1566+
"t",
1567+
"quadrillion",
1568+
]
1569+
"""Valid values for DataPointMention.scale. Each value maps to a known numeric multiplier."""
1570+
1571+
1572+
class DataPointMention(EntityMention):
1573+
"""A quantitative data point mentioned in text, extending EntityMention with a numeric breakdown.
1574+
1575+
Inherited fields: text (normalised form), orig (source text), label (category),
1576+
charspan, confidence, created_by.
1577+
"""
1578+
1579+
value: Annotated[
1580+
Optional[float],
1581+
Field(
1582+
description="Numeric value as written, before scale application. E.g. 4.0 for '$4B'.",
1583+
examples=[4.0, 30.0, 3.0],
1584+
),
1585+
] = None
1586+
1587+
unit: Annotated[
1588+
Optional[str],
1589+
Field(
1590+
description="Dimensional unit, e.g. 'USD', '°C', '%'. ISO 4217 recommended for currencies.",
1591+
examples=["USD", "°C", "%"],
1592+
),
1593+
] = None
1594+
1595+
scale: Annotated[
1596+
Optional[DataPointScale],
1597+
Field(
1598+
description="Magnitude multiplier from the source text, e.g. 'billion', 'million', 'k'.",
1599+
examples=["billion", "million", "k"],
1600+
),
1601+
] = None
1602+
1603+
normalized_value: Annotated[
1604+
Optional[float],
1605+
Field(
1606+
description="value * scale_factor in base units, e.g. 4e9 for '$4B'. Derived convenience field.",
1607+
examples=[4_000_000_000.0, 30.0],
1608+
),
1609+
] = None
1610+
1611+
range_end: Annotated[
1612+
Optional[float],
1613+
Field(
1614+
description=(
1615+
"Upper bound when the data point expresses a range, e.g. 20.0 for 'between 10 and 20°C'. "
1616+
"Use with precision='range_low'; value holds the lower bound."
1617+
),
1618+
examples=[20.0],
1619+
),
1620+
] = None
1621+
1622+
display_dp: Annotated[
1623+
Optional[int],
1624+
Field(
1625+
ge=0,
1626+
description=(
1627+
"Decimal places as written in the source: 0 for '3%', 1 for '3.0%', 2 for '3.00%'. "
1628+
"None when not determinable."
1629+
),
1630+
examples=[0, 1, 2],
1631+
),
1632+
] = None
1633+
1634+
precision: Annotated[
1635+
Optional[DataPointPrecision],
1636+
Field(
1637+
description=(
1638+
"Author's epistemic qualification: 'exact', 'approximate', "
1639+
"'lower_bound', 'upper_bound', 'range_low', 'range_high'."
1640+
),
1641+
examples=["approximate", "lower_bound"],
1642+
),
1643+
] = None
1644+
1645+
direction: Annotated[
1646+
Optional[DataPointDirection],
1647+
Field(
1648+
description=(
1649+
"Direction of change for delta/growth values: 'increase', 'decrease', 'neutral'. "
1650+
"E.g. 'revenue grew 3%' → 'increase'."
1651+
),
1652+
examples=["increase", "decrease"],
1653+
),
1654+
] = None
1655+
1656+
@field_validator("range_end", mode="after")
1657+
@classmethod
1658+
def _validate_range_end(cls, v: Optional[float], info: Any) -> Optional[float]:
1659+
if v is not None and info.data.get("value") is None:
1660+
raise ValueError("range_end requires value to be set")
1661+
return v
1662+
1663+
@property
1664+
def scale_factor(self) -> Optional[float]:
1665+
"""Numeric multiplier for scale, or None if absent or unrecognised."""
1666+
if self.scale is None:
1667+
return None
1668+
_SCALE_MAP: dict[str, float] = {
1669+
"hundred": 1e2,
1670+
"thousand": 1e3,
1671+
"k": 1e3,
1672+
"million": 1e6,
1673+
"m": 1e6,
1674+
"billion": 1e9,
1675+
"b": 1e9,
1676+
"bn": 1e9,
1677+
"trillion": 1e12,
1678+
"t": 1e12,
1679+
"quadrillion": 1e15,
1680+
}
1681+
return _SCALE_MAP.get(self.scale)
1682+
1683+
def compute_normalized_value(self) -> Optional[float]:
1684+
"""Return value * scale_factor without caching.
1685+
1686+
Returns None when value is absent or scale is present but unrecognised.
1687+
Use this to derive or refresh normalized_value after construction.
1688+
"""
1689+
if self.value is None:
1690+
return None
1691+
factor = self.scale_factor
1692+
if factor is None and self.scale is not None:
1693+
return None
1694+
return self.value * (factor if factor is not None else 1.0)
1695+
1696+
15501697
class EntitiesMetaField(_ExtraAllowingModel):
1551-
"""Container for extracted entity mentions."""
1698+
"""Container for extracted entity mentions.
1699+
1700+
The mentions list accepts both plain EntityMention objects and the richer
1701+
DataPointMention subclass; use isinstance(m, DataPointMention) to filter.
1702+
"""
15521703

1553-
mentions: Annotated[list[EntityMention], Field(min_length=1)]
1704+
mentions: Annotated[list[Union[DataPointMention, EntityMention]], Field(min_length=1)]
15541705

15551706

15561707
def _ensure_unique_list(values: Any) -> Any:

0 commit comments

Comments
 (0)