fix: correct RecursiveDocumentSplitter split_idx_start for word overlap ending in a separator

haystack/components/preprocessors/recursive_splitter.py

@@ -479,16 +479,35 @@ def _run_one(self, doc: Document) -> list[Document]:
             new_docs.append(new_doc)
             # Advance current_position by chunk length minus overlap.
             # split_overlap is in split_units, not chars, so get the actual
-            # overlap string from _get_overlap() and use its char length.
+            # overlap string from _get_overlap() and measure where it sits in the chunk.
             if self.split_overlap > 0 and split_nr < len(chunks) - 1:
                 overlap_str, _ = self._get_overlap([doc.content for doc in new_docs])  # type: ignore[misc]
-                overlap_char_len = len(overlap_str)
+                overlap_char_len = self._overlap_char_length(chunk, overlap_str)
             else:
                 overlap_char_len = 0
             current_position += len(chunk) - overlap_char_len
 
         return new_docs
 
+    def _overlap_char_length(self, chunk: str, overlap: str) -> int:
+        """
+        Return how many characters at the end of `chunk` are covered by `overlap`.
+
+        In word mode the overlap is rebuilt by joining the last words with single spaces, so its length does not
+        include the whitespace that follows those words in the chunk (for example the trailing separator). The next
+        chunk starts at the first overlapping word, so measure from there to the end of the chunk instead.
+
+        :param chunk: The chunk the overlap was taken from.
+        :param overlap: The overlap string returned by `_get_overlap`.
+        :returns: The number of characters of `chunk`, counted from its end, that the overlap covers.
+        """
+        if self.split_units == "word":
+            n_overlap_words = len(overlap.split())
+            word_starts = [match.start() for match in re.finditer(r"\S+", chunk)]
+            if 0 < n_overlap_words <= len(word_starts):
+                return len(chunk) - word_starts[-n_overlap_words]
+        return len(overlap)
+
     @component.output_types(documents=list[Document])
     def run(self, documents: list[Document]) -> dict[str, list[Document]]:
         """

releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    ``RecursiveDocumentSplitter`` with ``split_unit="word"`` and ``split_overlap > 0`` now reports the correct
+    ``split_idx_start`` and ``_split_overlap`` ranges. The overlap text is rebuilt with single spaces and no trailing
+    whitespace, so the start offset of every chunk after the first was shifted by the length of the previous chunk's
+    trailing separator (for example one character per chunk with the default separators). The offsets are now measured
+    from where the overlapping words start in the previous chunk, so ``document.content[split_idx_start:]`` again
+    begins with the chunk text. Character and token splitting are unchanged.

test/components/preprocessors/test_recursive_splitter.py

@@ -1282,3 +1282,58 @@ def test_fallback_word_unit_no_trailing_whitespace_only_chunk():
     for doc in result:
         assert doc.content is not None
         assert doc.content.strip()
+
+
+def _assert_offsets_and_overlaps_match_text(text: str, docs: list[Document]) -> None:
+    for doc in docs:
+        start = doc.meta["split_idx_start"]
+        assert text[start : start + len(doc.content)] == doc.content
+    # each "_split_overlap" entry stores a range into the document named by its "doc_id"
+    for previous, current in zip(docs, docs[1:], strict=False):
+        (into_current,) = [e["range"] for e in previous.meta["_split_overlap"] if e["doc_id"] == current.id]
+        (into_previous,) = [e["range"] for e in current.meta["_split_overlap"] if e["doc_id"] == previous.id]
+        overlap_text = previous.content[into_previous[0] : into_previous[1]]
+        assert overlap_text == current.content[into_current[0] : into_current[1]]
+        assert text[current.meta["split_idx_start"] :].startswith(overlap_text)
+
+
+def test_run_split_by_word_with_overlap_split_idx_start_when_chunk_ends_with_separator():
+    # the " " separator is kept at the end of each chunk; the overlap must not be measured without it
+    text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four."
+    splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, split_unit="word", separators=[" "])
+    splitter.warm_up()
+
+    docs = splitter.run(documents=[Document(content=text)])["documents"]
+
+    assert [doc.content for doc in docs] == [
+        "This is sentence one. ",
+        "one. This is sentence",
+        "sentence two. This is",
+        "is sentence three. This",
+        "This is sentence four.",
+    ]
+    assert [doc.meta["split_idx_start"] for doc in docs] == [0, 17, 30, 49, 68]
+    _assert_offsets_and_overlaps_match_text(text, docs)
+
+
+def test_run_split_by_word_with_overlap_split_idx_start_with_default_separators():
+    text = " ".join(f"w{i:02d}" for i in range(20))
+    splitter = RecursiveDocumentSplitter(split_length=6, split_overlap=2, split_unit="word")
+    splitter.warm_up()
+
+    docs = splitter.run(documents=[Document(content=text)])["documents"]
+
+    assert len(docs) == 5
+    assert [doc.meta["split_idx_start"] for doc in docs] == [0, 16, 32, 48, 64]
+    _assert_offsets_and_overlaps_match_text(text, docs)
+
+
+def test_run_split_by_char_with_overlap_split_idx_start_unchanged():
+    text = "abcdefghij" * 5
+    splitter = RecursiveDocumentSplitter(split_length=20, split_overlap=5, split_unit="char", separators=["j"])
+    splitter.warm_up()
+
+    docs = splitter.run(documents=[Document(content=text)])["documents"]
+
+    assert len(docs) > 1
+    _assert_offsets_and_overlaps_match_text(text, docs)