fix(verify): allow duplicate content-hash passage ids (#5)

docs/CHANGELOG.md

@@ -59,3 +59,16 @@ fixes). Newest entries at the bottom.
 - A transiently unreadable file keeps its previous hash instead of being classified as
   removed (which deleted its chunks from the index).
 - `LEANN_NO_REGISTER=1` env switch skips project-directory registration (for tests/CI).
+
+## 2026-08-19: verify accepts duplicate content-hash passage ids (issue #5)
+
+- `leann verify` no longer false-positives on healthy indexes built with
+  `--id-scheme=content-hash`, where byte-identical chunks legitimately share one passage id
+  (many-to-one by construction: jsonl keeps one line per chunk, the offset map and IVF
+  `passage_to_id` are last-wins one-entry-per-unique-id).
+- Invariants rewritten to the actual contract: duplicate jsonl ids allowed only with
+  identical text per id; idx cardinality vs unique ids; IVF `passage_to_id` checked as a
+  partial inverse (each passage id maps back to one of its FAISS labels, key set equals
+  the deduped `id_to_passage` value set).
+- Same id with different text, broken inversion, or a missing `passage_to_id` entry still
+  fail verify; tests now include a duplicate-content fixture.

packages/leann-core/src/leann/cli.py

@@ -2865,19 +2865,29 @@ def verify_command(self, args) -> int:
 
         jsonl_path = Path(str(prefix) + ".passages.jsonl")
         jsonl_ids: list[str] = []
+        text_hash_by_id: dict[str, str] = {}
+        conflicting_ids: set[str] = set()
         try:
             with open(jsonl_path, encoding="utf-8") as f:
                 for lineno, line in enumerate(f, 1):
                     try:
-                        pid = json.loads(line)["id"]
+                        record = json.loads(line)
+                        pid = record["id"]
                     except Exception as exc:
                         findings.append(f"passages.jsonl line {lineno} unparseable: {exc}")
                         continue
                     jsonl_ids.append(pid)
+                    text = record.get("text", "")
+                    text_hash = hashlib.sha256(f"{type(text).__name__}:{text}".encode()).hexdigest()
+                    if pid in text_hash_by_id:
+                        if text_hash_by_id[pid] != text_hash:
+                            conflicting_ids.add(pid)
+                    else:
+                        text_hash_by_id[pid] = text_hash
         except Exception as exc:
             findings.append(f"passages.jsonl unreadable: {exc}")
-        if len(set(jsonl_ids)) != len(jsonl_ids):
-            findings.append("passages.jsonl contains duplicate ids")
+        for pid in sorted(conflicting_ids, key=str):
+            findings.append(f"passages.jsonl id {pid!r} appears with different text")
 
         offsets: dict[str, int] = {}
         offsets_ok = False
@@ -2893,10 +2903,11 @@ def verify_command(self, args) -> int:
             findings.append(f"passages.idx unreadable: {exc}")
 
         if offsets_ok and (offsets or jsonl_ids):
-            if len(offsets) != len(jsonl_ids):
+            unique_jsonl_ids = set(jsonl_ids)
+            if len(offsets) != len(unique_jsonl_ids):
                 findings.append(
                     f"passages.idx has {len(offsets)} entries but "
-                    f"passages.jsonl has {len(jsonl_ids)} lines"
+                    f"passages.jsonl has {len(unique_jsonl_ids)} unique ids"
                 )
             if set(offsets) != set(jsonl_ids):
                 findings.append("passages.idx keys do not match passages.jsonl ids")
@@ -2983,11 +2994,15 @@ def _verify_ivf(self, prefix: Path, offsets: Optional[dict[str, int]]) -> list[s
                     findings.append(f"id_to_passage key {key!r} is negative")
             except ValueError:
                 findings.append(f"id_to_passage key {key!r} is not an integer")
-        if len(id_to_passage) != len(passage_to_id):
-            findings.append("id_to_passage and passage_to_id have different sizes")
-        for key, pid in id_to_passage.items():
-            if str(passage_to_id.get(pid)) != key:
-                findings.append(f"id_to_passage[{key!r}]={pid!r} is not inverted in passage_to_id")
+        if len(passage_to_id) != len(set(id_to_passage.values())):
+            findings.append("passage_to_id size does not match unique id_to_passage values")
+        for pid, fid in passage_to_id.items():
+            if id_to_passage.get(str(fid)) != pid:
+                findings.append(
+                    f"passage_to_id[{pid!r}]={fid!r} does not map back in id_to_passage"
+                )
+        if set(passage_to_id) != set(id_to_passage.values()):
+            findings.append("passage_to_id keys do not match id_to_passage values")
         if offsets is not None and set(id_to_passage.values()) != set(offsets):
             findings.append("id_to_passage values do not match passages.idx keys")
         numeric_ids = [

tests/test_cli_verify.py

@@ -3,6 +3,7 @@
 import json
 import pickle
 from pathlib import Path
+from typing import Any
 
 import numpy as np
 from leann.cli import LeannCLI
@@ -83,6 +84,50 @@ def _make_ivf_index(tmp_path: Path, passage_ids: list[str]) -> Path:
     return prefix
 
 
+def _make_ivf_index_with_entries(tmp_path: Path, entries: list[tuple[Any, Any]]) -> Path:
+    # Like _make_ivf_index but accepts explicit (pid, text) pairs so a pid may
+    # repeat (content-hash id scheme). idx and passage_to_id are last-wins.
+    index_dir = tmp_path / ".leann" / "indexes" / "idx"
+    index_dir.mkdir(parents=True)
+    prefix = index_dir / "documents.leann"
+
+    Path(str(prefix) + ".meta.json").write_text(
+        json.dumps(
+            {
+                "backend_name": "ivf",
+                "embedding_model": "m",
+                "embedding_mode": "sentence-transformers",
+                "dimensions": 4,
+                "backend_kwargs": {},
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    offsets: dict[str, int] = {}
+    with open(str(prefix) + ".passages.jsonl", "wb") as f:
+        for pid, text in entries:
+            offsets[pid] = f.tell()
+            line = json.dumps({"id": pid, "text": text, "metadata": {}}) + "\n"
+            f.write(line.encode("utf-8"))
+    with open(str(prefix) + ".passages.idx", "wb") as f:
+        pickle.dump(offsets, f)
+
+    id_map = {
+        "id_to_passage": {str(i): pid for i, (pid, _) in enumerate(entries)},
+        "passage_to_id": {pid: i for i, (pid, _) in enumerate(entries)},
+        "next_id": len(entries),
+    }
+    Path(str(prefix).removesuffix(".leann") + ".ivf_id_map.json").write_text(
+        json.dumps(id_map), encoding="utf-8"
+    )
+
+    _write_faiss_index(
+        Path(str(prefix).removesuffix(".leann") + ".index"), num_vectors=len(entries)
+    )
+    return prefix
+
+
 def _run_verify(index_name: str = "idx") -> int:
     cli = LeannCLI()
     args = cli.create_parser().parse_args(["verify", index_name])
@@ -316,3 +361,104 @@ def test_verify_flags_mapped_id_absent_from_index_vectors(tmp_path, monkeypatch,
     # Assert
     assert rc == 1
     assert "missing 1 of 2 mapped ids" in out
+
+
+def test_verify_passes_on_healthy_index_with_duplicate_content_hash_ids(
+    tmp_path, monkeypatch, capsys
+):
+    # Arrange: byte-identical chunks legitimately share one content-hash id
+    _make_ivf_index_with_entries(
+        tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")]
+    )
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+    out = capsys.readouterr().out
+
+    # Assert
+    assert rc == 0
+    assert "duplicate" not in out
+    assert "do not match" not in out
+
+
+def test_verify_fails_on_duplicate_id_with_different_text(tmp_path, monkeypatch, capsys):
+    # Arrange
+    _make_ivf_index_with_entries(
+        tmp_path, [("dup", "text one"), ("dup", "text two"), ("p1", "other text")]
+    )
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+    out = capsys.readouterr().out
+
+    # Assert
+    assert rc == 1
+    assert out.strip()
+
+
+def test_verify_fails_on_duplicate_id_with_type_coerced_text(tmp_path, monkeypatch, capsys):
+    # Arrange: same id, texts 1 (int) vs "1" (str) must count as different text
+    _make_ivf_index_with_entries(tmp_path, [("dup", 1), ("dup", "1")])
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+    out = capsys.readouterr().out
+
+    # Assert
+    assert rc == 1
+    assert "different text" in out
+
+
+def test_verify_reports_conflicts_with_heterogeneous_id_types(tmp_path, monkeypatch, capsys):
+    # Arrange: conflicting ids of mixed types (int and str) must not crash sorted()
+    _make_ivf_index_with_entries(
+        tmp_path, [(1, "text a"), (1, "text b"), ("x", "text c"), ("x", "text d")]
+    )
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+    out = capsys.readouterr().out
+
+    # Assert
+    assert rc == 1
+    assert "different text" in out
+
+
+def test_verify_fails_when_passage_to_id_label_maps_to_other_pid(tmp_path, monkeypatch):
+    # Arrange: passage_to_id["p1"] points at a faiss label owned by "dup"
+    prefix = _make_ivf_index_with_entries(
+        tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")]
+    )
+    id_map_path = prefix.parent / "documents.ivf_id_map.json"
+    id_map = json.loads(id_map_path.read_text(encoding="utf-8"))
+    id_map["passage_to_id"]["p1"] = 0
+    id_map_path.write_text(json.dumps(id_map), encoding="utf-8")
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+
+    # Assert
+    assert rc != 0
+
+
+def test_verify_fails_when_pid_missing_from_passage_to_id(tmp_path, monkeypatch):
+    # Arrange
+    prefix = _make_ivf_index_with_entries(
+        tmp_path, [("dup", "same text"), ("dup", "same text"), ("p1", "other text")]
+    )
+    id_map_path = prefix.parent / "documents.ivf_id_map.json"
+    id_map = json.loads(id_map_path.read_text(encoding="utf-8"))
+    del id_map["passage_to_id"]["p1"]
+    id_map_path.write_text(json.dumps(id_map), encoding="utf-8")
+    monkeypatch.chdir(tmp_path)
+
+    # Act
+    rc = _run_verify()
+
+    # Assert
+    assert rc != 0