diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f28d5d11..c1c9e896 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -59,3 +59,16 @@ fixes). Newest entries at the bottom. - A transiently unreadable file keeps its previous hash instead of being classified as removed (which deleted its chunks from the index). - `LEANN_NO_REGISTER=1` env switch skips project-directory registration (for tests/CI). + +## 2026-08-19: verify accepts duplicate content-hash passage ids (issue #5) + +- `leann verify` no longer false-positives on healthy indexes built with + `--id-scheme=content-hash`, where byte-identical chunks legitimately share one passage id + (many-to-one by construction: jsonl keeps one line per chunk, the offset map and IVF + `passage_to_id` are last-wins one-entry-per-unique-id). +- Invariants rewritten to the actual contract: duplicate jsonl ids allowed only with + identical text per id; idx cardinality vs unique ids; IVF `passage_to_id` checked as a + partial inverse (each passage id maps back to one of its FAISS labels, key set equals + the deduped `id_to_passage` value set). +- Same id with different text, broken inversion, or a missing `passage_to_id` entry still + fail verify; tests now include a duplicate-content fixture. diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py index cf5728d3..f37e1ca9 100644 --- a/packages/leann-core/src/leann/cli.py +++ b/packages/leann-core/src/leann/cli.py @@ -2865,19 +2865,29 @@ def verify_command(self, args) -> int: jsonl_path = Path(str(prefix) + ".passages.jsonl") jsonl_ids: list[str] = [] + text_hash_by_id: dict[str, str] = {} + conflicting_ids: set[str] = set() try: with open(jsonl_path, encoding="utf-8") as f: for lineno, line in enumerate(f, 1): try: - pid = json.loads(line)["id"] + record = json.loads(line) + pid = record["id"] except Exception as exc: findings.append(f"passages.jsonl line {lineno} unparseable: {exc}") continue jsonl_ids.append(pid) + text = record.get("text", "") + text_hash = hashlib.sha256(f"{type(text).__name__}:{text}".encode()).hexdigest() + if pid in text_hash_by_id: + if text_hash_by_id[pid] != text_hash: + conflicting_ids.add(pid) + else: + text_hash_by_id[pid] = text_hash except Exception as exc: findings.append(f"passages.jsonl unreadable: {exc}") - if len(set(jsonl_ids)) != len(jsonl_ids): - findings.append("passages.jsonl contains duplicate ids") + for pid in sorted(conflicting_ids, key=str): + findings.append(f"passages.jsonl id {pid!r} appears with different text") offsets: dict[str, int] = {} offsets_ok = False @@ -2893,10 +2903,11 @@ def verify_command(self, args) -> int: findings.append(f"passages.idx unreadable: {exc}") if offsets_ok and (offsets or jsonl_ids): - if len(offsets) != len(jsonl_ids): + unique_jsonl_ids = set(jsonl_ids) + if len(offsets) != len(unique_jsonl_ids): findings.append( f"passages.idx has {len(offsets)} entries but " - f"passages.jsonl has {len(jsonl_ids)} lines" + f"passages.jsonl has {len(unique_jsonl_ids)} unique ids" ) if set(offsets) != set(jsonl_ids): findings.append("passages.idx keys do not match passages.jsonl ids") @@ -2983,11 +2994,15 @@ def _verify_ivf(self, prefix: Path, offsets: Optional[dict[str, int]]) -> list[s findings.append(f"id_to_passage key {key!r} is negative") except ValueError: findings.append(f"id_to_passage key {key!r} is not an integer") - if len(id_to_passage) != len(passage_to_id): - findings.append("id_to_passage and passage_to_id have different sizes") - for key, pid in id_to_passage.items(): - if str(passage_to_id.get(pid)) != key: - findings.append(f"id_to_passage[{key!r}]={pid!r} is not inverted in passage_to_id") + if len(passage_to_id) != len(set(id_to_passage.values())): + findings.append("passage_to_id size does not match unique id_to_passage values") + for pid, fid in passage_to_id.items(): + if id_to_passage.get(str(fid)) != pid: + findings.append( + f"passage_to_id[{pid!r}]={fid!r} does not map back in id_to_passage" + ) + if set(passage_to_id) != set(id_to_passage.values()): + findings.append("passage_to_id keys do not match id_to_passage values") if offsets is not None and set(id_to_passage.values()) != set(offsets): findings.append("id_to_passage values do not match passages.idx keys") numeric_ids = [ diff --git a/tests/test_cli_verify.py b/tests/test_cli_verify.py index 3b4f39f4..65a19798 100644 --- a/tests/test_cli_verify.py +++ b/tests/test_cli_verify.py @@ -3,6 +3,7 @@ import json import pickle from pathlib import Path +from typing import Any import numpy as np from leann.cli import LeannCLI @@ -83,6 +84,50 @@ def _make_ivf_index(tmp_path: Path, passage_ids: list[str]) -> Path: return prefix +def _make_ivf_index_with_entries(tmp_path: Path, entries: list[tuple[Any, Any]]) -> Path: + # Like _make_ivf_index but accepts explicit (pid, text) pairs so a pid may + # repeat (content-hash id scheme). idx and passage_to_id are last-wins. + index_dir = tmp_path / ".leann" / "indexes" / "idx" + index_dir.mkdir(parents=True) + prefix = index_dir / "documents.leann" + + Path(str(prefix) + ".meta.json").write_text( + json.dumps( + { + "backend_name": "ivf", + "embedding_model": "m", + "embedding_mode": "sentence-transformers", + "dimensions": 4, + "backend_kwargs": {}, + } + ), + encoding="utf-8", + ) + + offsets: dict[str, int] = {} + with open(str(prefix) + ".passages.jsonl", "wb") as f: + for pid, text in entries: + offsets[pid] = f.tell() + line = json.dumps({"id": pid, "text": text, "metadata": {}}) + "\n" + f.write(line.encode("utf-8")) + with open(str(prefix) + ".passages.idx", "wb") as f: + pickle.dump(offsets, f) + + id_map = { + "id_to_passage": {str(i): pid for i, (pid, _) in enumerate(entries)}, + "passage_to_id": {pid: i for i, (pid, _) in enumerate(entries)}, + "next_id": len(entries), + } + Path(str(prefix).removesuffix(".leann") + ".ivf_id_map.json").write_text( + json.dumps(id_map), encoding="utf-8" + ) + + _write_faiss_index( + Path(str(prefix).removesuffix(".leann") + ".index"), num_vectors=len(entries) + ) + return prefix + + def _run_verify(index_name: str = "idx") -> int: cli = LeannCLI() args = cli.create_parser().parse_args(["verify", index_name]) @@ -316,3 +361,104 @@ def test_verify_flags_mapped_id_absent_from_index_vectors(tmp_path, monkeypatch, # Assert assert rc == 1 assert "missing 1 of 2 mapped ids" in out + + +def test_verify_passes_on_healthy_index_with_duplicate_content_hash_ids( + tmp_path, monkeypatch, capsys +): + # Arrange: byte-identical chunks legitimately share one content-hash id + _make_ivf_index_with_entries( + tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")] + ) + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + out = capsys.readouterr().out + + # Assert + assert rc == 0 + assert "duplicate" not in out + assert "do not match" not in out + + +def test_verify_fails_on_duplicate_id_with_different_text(tmp_path, monkeypatch, capsys): + # Arrange + _make_ivf_index_with_entries( + tmp_path, [("dup", "text one"), ("dup", "text two"), ("p1", "other text")] + ) + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + out = capsys.readouterr().out + + # Assert + assert rc == 1 + assert out.strip() + + +def test_verify_fails_on_duplicate_id_with_type_coerced_text(tmp_path, monkeypatch, capsys): + # Arrange: same id, texts 1 (int) vs "1" (str) must count as different text + _make_ivf_index_with_entries(tmp_path, [("dup", 1), ("dup", "1")]) + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + out = capsys.readouterr().out + + # Assert + assert rc == 1 + assert "different text" in out + + +def test_verify_reports_conflicts_with_heterogeneous_id_types(tmp_path, monkeypatch, capsys): + # Arrange: conflicting ids of mixed types (int and str) must not crash sorted() + _make_ivf_index_with_entries( + tmp_path, [(1, "text a"), (1, "text b"), ("x", "text c"), ("x", "text d")] + ) + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + out = capsys.readouterr().out + + # Assert + assert rc == 1 + assert "different text" in out + + +def test_verify_fails_when_passage_to_id_label_maps_to_other_pid(tmp_path, monkeypatch): + # Arrange: passage_to_id["p1"] points at a faiss label owned by "dup" + prefix = _make_ivf_index_with_entries( + tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")] + ) + id_map_path = prefix.parent / "documents.ivf_id_map.json" + id_map = json.loads(id_map_path.read_text(encoding="utf-8")) + id_map["passage_to_id"]["p1"] = 0 + id_map_path.write_text(json.dumps(id_map), encoding="utf-8") + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + + # Assert + assert rc != 0 + + +def test_verify_fails_when_pid_missing_from_passage_to_id(tmp_path, monkeypatch): + # Arrange + prefix = _make_ivf_index_with_entries( + tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")] + ) + id_map_path = prefix.parent / "documents.ivf_id_map.json" + id_map = json.loads(id_map_path.read_text(encoding="utf-8")) + del id_map["passage_to_id"]["p1"] + id_map_path.write_text(json.dumps(id_map), encoding="utf-8") + monkeypatch.chdir(tmp_path) + + # Act + rc = _run_verify() + + # Assert + assert rc != 0