fix: populate _split_overlap metadata for word/token units in RecursiveDocumentSplitter (#11825)

Co-authored-by: Sebastian Husch Lee <10526848+sjrl@users.noreply.github.com>

Author: Osamaali313

haystack/components/preprocessors/recursive_splitter.py

@@ -403,17 +403,14 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "
 
     def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None:
         prev_doc = new_docs[-1]
-        overlap_length = self._chunk_length(prev_doc.content) - (curr_pos - prev_doc.meta["split_idx_start"])  # type: ignore
+        # curr_pos and split_idx_start are character offsets, so measure the
+        # overlap and range in characters too (not via _chunk_length, which returns a word/token count).
+        prev_doc_length = len(prev_doc.content)  # type: ignore
+        overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"])
         if overlap_length > 0:
             prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)})
             new_doc.meta["_split_overlap"].append(
-                {
-                    "doc_id": prev_doc.id,
-                    "range": (
-                        self._chunk_length(prev_doc.content) - overlap_length,  # type: ignore
-                        self._chunk_length(prev_doc.content),  # type: ignore
-                    ),
-                }
+                {"doc_id": prev_doc.id, "range": (prev_doc_length - overlap_length, prev_doc_length)}
             )
 
     def _run_one(self, doc: Document) -> list[Document]:

releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixed ``RecursiveDocumentSplitter`` not populating the ``_split_overlap``
+    metadata when ``split_unit`` is ``"word"`` or ``"token"`` and ``split_overlap``
+    is greater than 0. The overlap length was computed with a word/token count
+    while the offsets it is compared against are character positions, so it
+    became negative and the overlap metadata was silently dropped. The overlap
+    is now measured in characters, matching the ranges it records.

test/components/preprocessors/test_recursive_splitter.py

@@ -750,6 +750,60 @@ def test_run_split_by_dot_and_overlap_1_word_unit_split_idx_start():
         )
 
 
+def test_word_unit_split_populates_split_overlap_metadata():
+    """
+    _split_overlap ranges must be character offsets into the referenced chunk when
+    split_unit="word" and split_overlap > 0
+    """
+    splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="word")
+    text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four."
+    chunks = splitter.run([Document(content=text)])["documents"]
+    assert len(chunks) == 5
+
+    assert chunks[0].content == "This is sentence one."
+    assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}]  # "one."
+
+    assert chunks[1].content == "one. This is sentence"
+    assert chunks[1].meta["_split_overlap"] == [
+        {"doc_id": chunks[0].id, "range": (17, 21)},  # "one."
+        {"doc_id": chunks[2].id, "range": (0, 8)},  # "sentence"
+    ]
+
+    assert chunks[2].content == "sentence two. This is"
+    assert chunks[2].meta["_split_overlap"] == [
+        {"doc_id": chunks[1].id, "range": (13, 21)},  # "sentence"
+        {"doc_id": chunks[3].id, "range": (0, 2)},  # "is"
+    ]
+
+    assert chunks[3].content == "is sentence three. This"
+    assert chunks[3].meta["_split_overlap"] == [
+        {"doc_id": chunks[2].id, "range": (19, 21)},  # "is"
+        {"doc_id": chunks[4].id, "range": (0, 4)},  # "This"
+    ]
+
+    assert chunks[4].content == "This is sentence four."
+    assert chunks[4].meta["_split_overlap"] == [{"doc_id": chunks[3].id, "range": (19, 23)}]  # "This"
+
+
+@pytest.mark.integration
+def test_token_unit_split_populates_split_overlap_metadata():
+    """
+    _split_overlap ranges must be character offsets into the referenced chunk when
+    split_unit="token" and split_overlap > 0
+    """
+    splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="token")
+    text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four."
+    chunks = splitter.run([Document(content=text)])["documents"]
+    assert len(chunks) == 8
+
+    assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}]
+    assert chunks[1].meta["_split_overlap"] == [
+        {"doc_id": chunks[0].id, "range": (16, 20)},
+        {"doc_id": chunks[2].id, "range": (0, 1)},
+    ]
+    assert chunks[7].meta["_split_overlap"] == [{"doc_id": chunks[6].id, "range": (9, 18)}]
+
+
 def test_run_trigger_dealing_with_remaining_word_larger_than_split_length():
     splitter = RecursiveDocumentSplitter(split_length=3, split_overlap=2, separators=["."], split_unit="word")
     text = """A simple sentence1. A bright sentence2. A clever sentence3"""