feat: adapt hierarchical chunker to v2 DoclingDocument

[skip-ci]

Signed-off-by: Panos Vagenas <[email protected]>

Author: vagenas

docling_core/transforms/chunker/__init__.py

@@ -5,11 +5,8 @@
 
 """Define the chunker types."""
 
-from docling_core.transforms.chunker.base import (  # noqa
-    BaseChunker,
-    Chunk,
-    ChunkWithMetadata,
-)
+from docling_core.transforms.chunker.base import BaseChunk, BaseChunker  # noqa
 from docling_core.transforms.chunker.hierarchical_chunker import (  # noqa
+    Chunk,
     HierarchicalChunker,
 )

docling_core/transforms/chunker/base.py

@@ -4,71 +4,35 @@
 #
 
 """Define base classes for chunking."""
-import re
 from abc import ABC, abstractmethod
-from typing import Final, Iterator, Optional
+from typing import Any, Iterator
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel
 
-from docling_core.types import BoundingBox, Document
-from docling_core.types.base import _JSON_POINTER_REGEX
+from docling_core.types.doc import DoclingDocument as DLDocument
 
-# (subset of) JSONPath format, e.g. "$.main-text[84]" (for migration purposes)
-_DEPRECATED_JSON_PATH_PATTERN: Final = re.compile(r"^\$\.([\w-]+)\[(\d+)\]$")
 
+class BaseChunk(BaseModel):
+    """Data model for base chunk."""
 
-def _create_path(pos: int, path_prefix: str = "main-text") -> str:
-    return f"#/{path_prefix}/{pos}"
-
-
-class Chunk(BaseModel):
-    """Data model for Chunk."""
-
-    path: str = Field(pattern=_JSON_POINTER_REGEX)
     text: str
-    heading: Optional[str] = None
-
-    @field_validator("path", mode="before")
-    @classmethod
-    def _json_pointer_from_json_path(cls, path: str):
-        if (match := _DEPRECATED_JSON_PATH_PATTERN.match(path)) is not None:
-            groups = match.groups()
-            if len(groups) == 2 and groups[0] is not None and groups[1] is not None:
-                return _create_path(
-                    pos=int(groups[1]),
-                    path_prefix=groups[0],
-                )
-        return path
-
-
-class ChunkWithMetadata(Chunk):
-    """Data model for Chunk including metadata."""
-
-    page: Optional[int] = None
-    bbox: Optional[BoundingBox] = None
+    meta: Any = None
 
 
 class BaseChunker(BaseModel, ABC):
     """Base class for Chunker."""
 
     @abstractmethod
-    def chunk(self, dl_doc: Document, **kwargs) -> Iterator[Chunk]:
+    def chunk(self, dl_doc: DLDocument, **kwargs) -> Iterator[BaseChunk]:
         """Chunk the provided document.
 
         Args:
-            dl_doc (Document): document to chunk
+            dl_doc (DLDocument): document to chunk
 
         Raises:
             NotImplementedError: in this abstract implementation
 
         Yields:
-            Iterator[Chunk]: iterator over extracted chunks
+            Iterator[BaseChunk]: iterator over extracted chunks
         """
         raise NotImplementedError()
-
-    @classmethod
-    def _create_path(cls, pos: int, path_prefix: str = "main-text") -> str:
-        return _create_path(
-            pos=pos,
-            path_prefix=path_prefix,
-        )