diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 168a0437151..0a7d744f4f1 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -479,16 +479,35 @@ def _run_one(self, doc: Document) -> list[Document]: new_docs.append(new_doc) # Advance current_position by chunk length minus overlap. # split_overlap is in split_units, not chars, so get the actual - # overlap string from _get_overlap() and use its char length. + # overlap string from _get_overlap() and measure where it sits in the chunk. if self.split_overlap > 0 and split_nr < len(chunks) - 1: overlap_str, _ = self._get_overlap([doc.content for doc in new_docs]) # type: ignore[misc] - overlap_char_len = len(overlap_str) + overlap_char_len = self._overlap_char_length(chunk, overlap_str) else: overlap_char_len = 0 current_position += len(chunk) - overlap_char_len return new_docs + def _overlap_char_length(self, chunk: str, overlap: str) -> int: + """ + Return how many characters at the end of `chunk` are covered by `overlap`. + + In word mode the overlap is rebuilt by joining the last words with single spaces, so its length does not + include the whitespace that follows those words in the chunk (for example the trailing separator). The next + chunk starts at the first overlapping word, so measure from there to the end of the chunk instead. + + :param chunk: The chunk the overlap was taken from. + :param overlap: The overlap string returned by `_get_overlap`. + :returns: The number of characters of `chunk`, counted from its end, that the overlap covers. + """ + if self.split_units == "word": + n_overlap_words = len(overlap.split()) + word_starts = [match.start() for match in re.finditer(r"\S+", chunk)] + if 0 < n_overlap_words <= len(word_starts): + return len(chunk) - word_starts[-n_overlap_words] + return len(overlap) + @component.output_types(documents=list[Document]) def run(self, documents: list[Document]) -> dict[str, list[Document]]: """ diff --git a/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml b/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml new file mode 100644 index 00000000000..8d135b70960 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-word-overlap-split-idx-start-bdfbe74c5d60e58a.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + ``RecursiveDocumentSplitter`` with ``split_unit="word"`` and ``split_overlap > 0`` now reports the correct + ``split_idx_start`` and ``_split_overlap`` ranges. The overlap text is rebuilt with single spaces and no trailing + whitespace, so the start offset of every chunk after the first was shifted by the length of the previous chunk's + trailing separator (for example one character per chunk with the default separators). The offsets are now measured + from where the overlapping words start in the previous chunk, so ``document.content[split_idx_start:]`` again + begins with the chunk text. Character and token splitting are unchanged. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index cf6dbbf0d44..582b98b8ad0 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -1282,3 +1282,58 @@ def test_fallback_word_unit_no_trailing_whitespace_only_chunk(): for doc in result: assert doc.content is not None assert doc.content.strip() + + +def _assert_offsets_and_overlaps_match_text(text: str, docs: list[Document]) -> None: + for doc in docs: + start = doc.meta["split_idx_start"] + assert text[start : start + len(doc.content)] == doc.content + # each "_split_overlap" entry stores a range into the document named by its "doc_id" + for previous, current in zip(docs, docs[1:], strict=False): + (into_current,) = [e["range"] for e in previous.meta["_split_overlap"] if e["doc_id"] == current.id] + (into_previous,) = [e["range"] for e in current.meta["_split_overlap"] if e["doc_id"] == previous.id] + overlap_text = previous.content[into_previous[0] : into_previous[1]] + assert overlap_text == current.content[into_current[0] : into_current[1]] + assert text[current.meta["split_idx_start"] :].startswith(overlap_text) + + +def test_run_split_by_word_with_overlap_split_idx_start_when_chunk_ends_with_separator(): + # the " " separator is kept at the end of each chunk; the overlap must not be measured without it + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, split_unit="word", separators=[" "]) + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert [doc.content for doc in docs] == [ + "This is sentence one. ", + "one. This is sentence", + "sentence two. This is", + "is sentence three. This", + "This is sentence four.", + ] + assert [doc.meta["split_idx_start"] for doc in docs] == [0, 17, 30, 49, 68] + _assert_offsets_and_overlaps_match_text(text, docs) + + +def test_run_split_by_word_with_overlap_split_idx_start_with_default_separators(): + text = " ".join(f"w{i:02d}" for i in range(20)) + splitter = RecursiveDocumentSplitter(split_length=6, split_overlap=2, split_unit="word") + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert len(docs) == 5 + assert [doc.meta["split_idx_start"] for doc in docs] == [0, 16, 32, 48, 64] + _assert_offsets_and_overlaps_match_text(text, docs) + + +def test_run_split_by_char_with_overlap_split_idx_start_unchanged(): + text = "abcdefghij" * 5 + splitter = RecursiveDocumentSplitter(split_length=20, split_overlap=5, split_unit="char", separators=["j"]) + splitter.warm_up() + + docs = splitter.run(documents=[Document(content=text)])["documents"] + + assert len(docs) > 1 + _assert_offsets_and_overlaps_match_text(text, docs)