feat: dlgrep tool

.github/scripts/build-packages.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e  # trigger failure on error - do not remove!
+set -x  # display command on output
+
+# Build each package into its own dist subdirectory so the PyPI publish action
+# can upload them independently (a single `dist/` makes the second publish step
+# re-upload the first package's files and fail on `skip-existing: false`).
+
+echo "Building docling-core package..."
+uv build --package docling-core --out-dir dist/docling-core
+
+echo "Building dlgrep package..."
+uv build --package dlgrep --out-dir dist/dlgrep
+
+echo "Build complete."
+echo "docling-core artifacts:"
+ls -lh dist/docling-core/
+echo "dlgrep artifacts:"
+ls -lh dist/dlgrep/

.github/scripts/release.sh

@@ -9,9 +9,14 @@ if [ -z "${TARGET_VERSION}" ]; then
 fi
 CHGLOG_FILE="${CHGLOG_FILE:-CHANGELOG.md}"
 
-# update package version
-uvx --from=toml-cli toml set --toml-path=pyproject.toml project.version "${TARGET_VERSION}"
-uv lock --upgrade-package docling-core
+# update package versions:
+#   - root pyproject.toml               = docling-core
+#   - packages/dlgrep/pyproject.toml    = dlgrep (lockstep, incl. its docling-core== pin)
+uv version "${TARGET_VERSION}" --package docling-core --frozen
+uv version "${TARGET_VERSION}" --package dlgrep --frozen
+uv add "docling-core==${TARGET_VERSION}" --package dlgrep --frozen
+
+uv lock --upgrade-package docling-core --upgrade-package dlgrep
 
 # collect release notes
 REL_NOTES=$(mktemp)
@@ -31,7 +36,7 @@ mv "${TMP_CHGLOG}" "${CHGLOG_FILE}"
 # push changes
 git config --global user.name 'github-actions[bot]'
 git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-git add pyproject.toml uv.lock "${CHGLOG_FILE}"
+git add pyproject.toml packages/dlgrep/pyproject.toml uv.lock "${CHGLOG_FILE}"
 COMMIT_MSG="chore: bump version to ${TARGET_VERSION} [skip ci]"
 git commit -m "${COMMIT_MSG}"
 git push origin main

.github/workflows/cd.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           enable-cache: true
       - name: Install dependencies
-        run: uv sync --only-dev
+        run: uv sync --frozen --only-dev
       - name: Check version of potential release
         id: version_check
         run: |
@@ -55,7 +55,7 @@ jobs:
         with:
           enable-cache: true
       - name: Install dependencies
-        run: uv sync --only-dev
+        run: uv sync --frozen --only-dev
       - name: Run release script
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}

.github/workflows/checks.yml

@@ -38,7 +38,7 @@ jobs:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Install dependencies
-        run: uv sync --frozen --all-extras
+        run: uv sync --frozen --all-extras --all-packages
       - name: Check style and run tests
         run: pre-commit run --all-files
       - name: Upload coverage to Codecov
@@ -61,11 +61,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
       - name: Install dependencies
-        run: uv sync --all-extras
-      - name: Build package
-        run: uv build
-      - name: Check content of wheel
-        run: unzip -l dist/*.whl
+        run: uv sync --frozen --all-extras --all-packages
+      - name: Build packages
+        run: bash .github/scripts/build-packages.sh
+      - name: Check content of wheels
+        run: unzip -l dist/docling-core/*.whl && unzip -l dist/dlgrep/*.whl
       - name: Store the distribution packages
         uses: actions/upload-artifact@v4
         with:
@@ -90,7 +90,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
-      - name: Install package
-        run: uv pip install dist/*.whl
+      # Installing both wheels together, offline for the two local packages, is
+      # what validates the co-release: dlgrep's exact `docling-core==` pin can
+      # only resolve against the docling-core wheel built from this same commit.
+      - name: Install packages
+        run: uv pip install dist/docling-core/*.whl dist/dlgrep/*.whl
       - name: Load the DoclingDocument package
         run: python -c 'from docling_core.types.doc import DoclingDocument'
+      - name: Run the dlgrep CLI
+        run: python -m dlgrep --help

.github/workflows/pypi.yml

@@ -1,4 +1,4 @@
-name: "Build and publish package"
+name: "Build and publish packages"
 
 on:
   release:
@@ -8,16 +8,11 @@ permissions:
   contents: read
 
 jobs:
-  build-and-publish:
+  build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.12']
-    environment:
-      name: pypi
-      url: https://pypi.org/p/docling-core
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
     steps:
       - uses: actions/checkout@v4
       - name: Install uv and set the python version
@@ -26,10 +21,63 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
       - name: Install dependencies
-        run: uv sync --all-extras
-      - name: Build package
-        run: uv build
-      - name: Publish distribution 📦 to PyPI
+        run: uv sync --frozen --all-extras --all-packages
+      - name: Build packages
+        run: bash .github/scripts/build-packages.sh
+      # Last gate before anything is public. Installing both wheels in one
+      # command forces the resolver to satisfy dlgrep's exact `docling-core==`
+      # pin with the core wheel built from this same commit: if the pin named
+      # any other version the two explicit requests conflict and this fails,
+      # rather than shipping a dlgrep wheel nothing can install.
+      - name: Verify the two packages release together
+        run: |
+          uv venv /tmp/relcheck
+          VIRTUAL_ENV=/tmp/relcheck uv pip install dist/docling-core/*.whl dist/dlgrep/*.whl
+          /tmp/relcheck/bin/python -c 'from docling_core.types.doc import DoclingDocument'
+          /tmp/relcheck/bin/dlgrep --version
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-docling-core:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/docling-core
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish docling-core distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          attestations: true
+          packages-dir: dist/docling-core/
+
+  publish-dlgrep:
+    # dlgrep pins docling-core exactly, so publish it second.
+    needs: publish-docling-core
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/dlgrep
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish dlgrep distribution 📦 to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           attestations: true
+          packages-dir: dist/dlgrep/

.pre-commit-config.yaml

@@ -6,18 +6,18 @@ repos:
       - id: ruff-check
         name: "Ruff linter"
         args: [--exit-non-zero-on-fix, --fix, --config=pyproject.toml]
-        files: '^(docling_core|test|docs/examples|examples).*\.(py|ipynb)$'
+        files: '^(docling_core|test|docs/examples|examples|packages/dlgrep).*\.(py|ipynb)$'
         pass_filenames: false
       - id: ruff-format
         name: "Ruff formatter"
         args: [--config=pyproject.toml]
-        files: '^(docling_core|test|docs/examples|examples).*\.(py|ipynb)$'
+        files: '^(docling_core|test|docs/examples|examples|packages/dlgrep).*\.(py|ipynb)$'
         pass_filenames: false
   - repo: local
     hooks:
       - id: mypy
         name: MyPy
-        entry: uv run --no-sync mypy docling_core test
+        entry: uv run --no-sync mypy docling_core test packages/dlgrep/dlgrep packages/dlgrep/tests
         pass_filenames: false
         language: system
         files: '\.py$'
@@ -29,6 +29,12 @@ repos:
         pass_filenames: false
         language: system
         files: '\.py$'
+      - id: pytest-dlgrep
+        name: Pytest (dlgrep)
+        entry: uv run --no-sync pytest packages/dlgrep/tests
+        pass_filenames: false
+        language: system
+        files: '\.py$'
   - repo: local
     hooks:
       - id: docs

docling_core/transforms/deserializer/__init__.py

@@ -1,9 +1,15 @@
 """Define the deserializer types."""
 
 from docling_core.transforms.deserializer.base import BaseDocDeserializer
-from docling_core.transforms.deserializer.doclang import DocLangDocDeserializer
+from docling_core.transforms.deserializer.doclang import (
+    DocLangDocDeserializer,
+    DocLangSourceMap,
+    DocLangSourceTarget,
+)
 
 __all__ = [
     "BaseDocDeserializer",
     "DocLangDocDeserializer",
+    "DocLangSourceMap",
+    "DocLangSourceTarget",
 ]