From eb5df9562272a882e336a9c2ab840e08dc9f649d Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 15 Sep 2026 08:54:09 +0530 Subject: [PATCH] fix: measure word overlap where it sits in the chunk for RecursiveDocumentSplitter split_idx_start With split_unit="word" the overlap string is rebuilt by joining the last words with single spaces, so it does not include the whitespace that follows those words in the previous chunk. Subtracting its length from the chunk length therefore overshot the next chunk's split_idx_start by the trailing separator, and the _split_overlap ranges inherited the error. Measure the overlap from the first overlapping word to the end of the chunk instead. Character and token modes are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DakLsFtFd23piCzmyiDngh --- .../preprocessors/recursive_splitter.py | 23 +++++++- ...rlap-split-idx-start-bdfbe74c5d60e58a.yaml | 9 +++ .../preprocessors/test_recursive_splitter.py | 55 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 168a0437151..0a7d744f4f1 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -479,16 +479,35 @@ def _run_one(self, doc: Document) -> list[Document]: new_docs.append(new_doc) # Advance current_position by chunk length minus overlap. # split_overlap is in split_units, not chars, so get the actual - # overlap string from _get_overlap() and use its char length. + # overlap string from _get_overlap() and measure where it sits in the chunk. if self.split_overlap > 0 and split_nr < len(chunks) - 1: overlap_str, _ = self._get_overlap([doc.content for doc in new_docs]) # type: ignore[misc] - overlap_char_len = len(overlap_str) + overlap_char_len = self._overlap_char_length(chunk, overlap_str) else: overlap_char_len = 0 current_position += len(chunk) - overlap_char_len return new_docs + def _overlap_char_length(self, chunk: str, overlap: str) -> int: + """ + Return how many characters at the end of `chunk` are covered by `overlap`. + + In word mode the overlap is rebuilt by joining the last words with single spaces, so its length does not + include the whitespace that follows those words in the chunk (for example the trailing separator). The next + chunk starts at the first overlapping word, so measure from there to the end of the chunk instead. + + :param chunk: The chunk the overlap was taken from. + :param overlap: The overlap string returned by `_get_overlap`. + :returns: The number of characters of `chunk`, counted from its end, that the overlap covers. + """ + if self.split_units == "word": + n_overlap_words = len(overlap.split()) + word_starts = [match.start() for match in re.finditer(r"\S+", chunk)] + if 0 < n_overlap_words <= len(word_starts): + return len(chunk) - word_starts[-n_overlap_words] + return len(overlap) + @component.output_types(documents=list[Document]) def run(self, documents: list[Document]) -> dict[str, list[Document]]: """ diff --git a/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml b/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml new file mode 100644 index 00000000000..8d135b70960 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + ``RecursiveDocumentSplitter`` with ``split_unit="word"`` and ``split_overlap > 0`` now reports the correct + ``split_idx_start`` and ``_split_overlap`` ranges. The overlap text is rebuilt with single spaces and no trailing + whitespace, so the start offset of every chunk after the first was shifted by the length of the previous chunk's + trailing separator (for example one character per chunk with the default separators). The offsets are now measured + from where the overlapping words start in the previous chunk, so ``document.content[split_idx_start:]`` again + begins with the chunk text. Character and token splitting are unchanged. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index cf6dbbf0d44..582b98b8ad0 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -1282,3 +1282,58 @@ def test_fallback_word_unit_no_trailing_whitespace_only_chunk(): for doc in result: assert doc.content is not None assert doc.content.strip() + + +def _assert_offsets_and_overlaps_match_text(text: str, docs: list[Document]) -> None: + for doc in docs: + start = doc.meta["split_idx_start"] + assert text[start : start + len(doc.content)] == doc.content + # each "_split_overlap" entry stores a range into the document named by its "doc_id" + for previous, current in zip(docs, docs[1:], strict=False): + (into_current,) = [e["range"] for e in previous.meta["_split_overlap"] if e["doc_id"] == current.id] + (into_previous,) = [e["range"] for e in current.meta["_split_overlap"] if e["doc_id"] == previous.id] + overlap_text = previous.content[into_previous[0] : into_previous[1]] + assert overlap_text == current.content[into_current[0] : into_current[1]] + assert text[current.meta["split_idx_start"] :].startswith(overlap_text) + + +def test_run_split_by_word_with_overlap_split_idx_start_when_chunk_ends_with_separator(): + # the " " separator is kept at the end of each chunk; the overlap must not be measured without it + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, split_unit="word", separators=[" "]) + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert [doc.content for doc in docs] == [ + "This is sentence one. ", + "one. This is sentence", + "sentence two. This is", + "is sentence three. This", + "This is sentence four.", + ] + assert [doc.meta["split_idx_start"] for doc in docs] == [0, 17, 30, 49, 68] + _assert_offsets_and_overlaps_match_text(text, docs) + + +def test_run_split_by_word_with_overlap_split_idx_start_with_default_separators(): + text = " ".join(f"w{i:02d}" for i in range(20)) + splitter = RecursiveDocumentSplitter(split_length=6, split_overlap=2, split_unit="word") + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert len(docs) == 5 + assert [doc.meta["split_idx_start"] for doc in docs] == [0, 16, 32, 48, 64] + _assert_offsets_and_overlaps_match_text(text, docs) + + +def test_run_split_by_char_with_overlap_split_idx_start_unchanged(): + text = "abcdefghij" * 5 + splitter = RecursiveDocumentSplitter(split_length=20, split_overlap=5, split_unit="char", separators=["j"]) + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert len(docs) > 1 + _assert_offsets_and_overlaps_match_text(text, docs)