From 48edac551ed359700cab39712712ecce0b6cb38f Mon Sep 17 00:00:00 2001
From: hlm <2514266720@qq.com>
Date: Mon, 28 Sep 2026 23:08:18 +0800
Subject: [PATCH 1/3] feat: pluggable BM25 tokenizers with jieba Chinese word
 segmentation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The original tokenizer splits CJK into single characters, so phrase-level
keyword recall fails for Chinese: query 检索 matches a document that merely
contains 检 and 索 inside different words, and that decoy can outrank the
genuinely relevant chunk.

- new loci.tokenizers registry: "default" (original behavior, no deps) and
  "jieba" (word-level segmentation, lazy import, fails open to default when
  the optional package is missing)
- BM25 accepts an injected tokenizer; module-level tokenize() kept as a
  backward-compatible alias (conftest FakeEmbedder relies on it)
- [bm25] tokenizer config section wired through Retriever; unknown names
  warn and fall back to default at load time
- bench gains --tokenizer so A/B runs need no config edits

A/B on a decoy corpus (query 检索; doc b contains 检+索 in other words):
default ranks decoy 0.416 > correct 0.325; jieba ranks correct 0.595 and
zeroes the decoy. 10 new tests in tests/test_tokenizers.py.
---
 config.example.toml      |  4 ++
 pyproject.toml           |  1 +
 src/loci/bm25.py         | 25 ++++++-----
 src/loci/cli.py          | 11 +++--
 src/loci/config.py       |  7 +++
 src/loci/retriever.py    |  9 +++-
 src/loci/tokenizers.py   | 48 +++++++++++++++++++++
 tests/test_tokenizers.py | 93 ++++++++++++++++++++++++++++++++++++++++
 8 files changed, 183 insertions(+), 15 deletions(-)
 create mode 100644 src/loci/tokenizers.py
 create mode 100644 tests/test_tokenizers.py

diff --git a/config.example.toml b/config.example.toml
index e5598fc..ac826a4 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -31,6 +31,10 @@ search = 5        # number of hits per search
 hybrid = true     # fuse vector search with BM25 (Reciprocal Rank Fusion)
 rrf_k = 60        # RRF smoothing constant; higher = flatter fusion weights
 
+[bm25]
+tokenizer = "default"   # "default" = CJK single-char (no deps) | "jieba" = word-level
+                        # Chinese segmentation (pip install 'loci-rag[zh]' or jieba)
+
 [watch]
 interval = 30     # seconds between scans in `watch` mode
 
diff --git a/pyproject.toml b/pyproject.toml
index 93c7383..5c13242 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,6 +34,7 @@ pdf = ["pymupdf4llm>=0.0.21", "pypdf>=4"]
 ocr = ["rapidocr-onnxruntime>=1.3"]
 docx = ["python-docx>=1.1"]
 rerank = ["sentence-transformers>=3"]
+zh = ["jieba>=0.42"]
 dev = ["pytest>=8"]
 
 [project.scripts]
diff --git a/src/loci/bm25.py b/src/loci/bm25.py
index 3b98d72..ebed051 100644
--- a/src/loci/bm25.py
+++ b/src/loci/bm25.py
@@ -1,28 +1,33 @@
-"""Native BM25 ranking with a CJK-aware tokenizer. No dependencies."""
+"""Native BM25 ranking with pluggable tokenizers. No dependencies by default."""
 from __future__ import annotations
 
 import math
-import re
 
-# Latin words stay whole; CJK text becomes single characters (strong enough
-# for BM25 ranking and keeps the tokenizer dependency-free).
-_TOKEN = re.compile(r"[a-z0-9]+|[\u4e00-\u9fff]")
+from loci.tokenizers import default_tokenize, get_tokenizer
 
 
 def tokenize(text: str) -> list[str]:
-    return _TOKEN.findall(text.lower())
+    """Backward-compatible alias for the default CJK single-char tokenizer."""
+    return default_tokenize(text)
 
 
 class BM25:
-    """Okapi BM25 over a corpus of {key: text}, built once per process."""
+    """Okapi BM25 over a corpus of {key: text}, built once per process.
 
-    def __init__(self, corpus: dict[str, str], k1: float = 1.5, b: float = 0.75):
+    `tokenizer` accepts a callable text->list[str]; defaults to the
+    dependency-free CJK single-char tokenizer. Pass `get_tokenizer("jieba")`
+    for word-level Chinese segmentation (requires the optional jieba package).
+    """
+
+    def __init__(self, corpus: dict[str, str], k1: float = 1.5, b: float = 0.75,
+                 tokenizer=None):
         self.k1, self.b = k1, b
+        self._tok = tokenizer or default_tokenize
         self.tf: dict[str, dict[str, int]] = {}
         self.dl: dict[str, int] = {}
         self.df: dict[str, int] = {}
         for key, text in corpus.items():
-            toks = tokenize(text)
+            toks = self._tok(text)
             freq: dict[str, int] = {}
             for t in toks:
                 freq[t] = freq.get(t, 0) + 1
@@ -42,7 +47,7 @@ def score(self, query: str) -> list[tuple[str, float]]:
             if not dl:
                 continue
             s = 0.0
-            for term in tokenize(query):
+            for term in self._tok(query):
                 df = self.df.get(term)
                 if not df:
                     continue
diff --git a/src/loci/cli.py b/src/loci/cli.py
index 0f268a0..9c52669 100644
--- a/src/loci/cli.py
+++ b/src/loci/cli.py
@@ -39,7 +39,8 @@ def make_retriever(cfg, embedder, store):
                      feedback_path=fb_path,
                      rerank_provider=cfg.retrieval.get("rerank_provider", "llm"),
                      local_rerank_model=cfg.retrieval.get(
-                         "local_rerank_model", "BAAI/bge-reranker-base"))
+                         "local_rerank_model", "BAAI/bge-reranker-base"),
+                     bm25_tokenizer=cfg.bm25.get("tokenizer", "default"))
 
 
 def cmd_ingest(cfg, force: bool = False) -> None:
@@ -240,7 +241,7 @@ def cmd_sync(cfg, direction: str) -> None:
         print(f"  {name}: {direction} done")
 
 
-def cmd_bench(cfg, cases_file: str, k: int = 5) -> None:
+def cmd_bench(cfg, cases_file: str, k: int = 5, tokenizer: str | None = None) -> None:
     import json as _json
     p = Path(cases_file)
     if not p.exists():
@@ -249,6 +250,8 @@ def cmd_bench(cfg, cases_file: str, k: int = 5) -> None:
     cases = [_json.loads(l) for l in p.read_text(encoding="utf-8").splitlines() if l.strip()]
     embedder, store = build(cfg)
     retriever = make_retriever(cfg, embedder, store)
+    if tokenizer:   # per-run override for A/B comparisons
+        retriever.bm25_tokenizer = tokenizer
     for label, hybrid in [("vector-only", False), ("hybrid", True)]:
         retriever.hybrid = hybrid
         ok = 0
@@ -511,6 +514,8 @@ def main() -> None:
     p_bench = sub.add_parser("bench", help="run a retrieval benchmark (hit@k) on a cases file")
     p_bench.add_argument("cases", help="JSONL file with {query, expect} per line")
     p_bench.add_argument("-k", type=int, default=5, help="hits per query (default 5)")
+    p_bench.add_argument("--tokenizer", default=None,
+                         help="BM25 tokenizer override for this run (default|jieba)")
 
     sub.add_parser("stats", help="show what is in the index")
     sub.add_parser("doctor", help="check config, endpoints, and store health")
@@ -558,7 +563,7 @@ def main() -> None:
     elif args.cmd == "sync":
         cmd_sync(cfg, args.direction)
     elif args.cmd == "bench":
-        cmd_bench(cfg, args.cases, k=args.k)
+        cmd_bench(cfg, args.cases, k=args.k, tokenizer=args.tokenizer)
     elif args.cmd == "wiki":
         cmd_wiki_suggest(cfg) if getattr(args, "suggest", False) else cmd_wiki(cfg, args.topic, k=args.k)
     elif args.cmd == "watch":
diff --git a/src/loci/config.py b/src/loci/config.py
index e633223..a3d13c3 100644
--- a/src/loci/config.py
+++ b/src/loci/config.py
@@ -17,6 +17,7 @@
     "retrieval": {"hybrid": True, "rrf_k": 60, "rerank": False,
                   "rerank_provider": "llm",
                   "local_rerank_model": "BAAI/bge-reranker-base"},
+    "bm25": {"tokenizer": "default"},
     "watch": {"interval": 30},
     "memories": {"path": "./memories"},
     "wiki": {"path": "./wiki"},
@@ -34,6 +35,7 @@ class Config:
     chunk: dict = field(default_factory=dict)
     top_k: dict = field(default_factory=dict)
     retrieval: dict = field(default_factory=dict)
+    bm25: dict = field(default_factory=dict)
     watch: dict = field(default_factory=dict)
     memories: dict = field(default_factory=dict)
     wiki: dict = field(default_factory=dict)
@@ -107,4 +109,9 @@ def load(path: str = "config.toml") -> Config:
         cfg.top_k["search"] = max(int(cfg.top_k["search"]), 1)
     except (TypeError, ValueError):
         cfg.top_k["search"] = DEFAULTS["top_k"]["search"]
+    known_tokenizers = ("default", "jieba")
+    if cfg.bm25.get("tokenizer") not in known_tokenizers:
+        print(f"[warn] bm25.tokenizer={cfg.bm25.get('tokenizer')!r} is not one of "
+              f"{known_tokenizers} — using 'default'", file=sys.stderr)
+        cfg.bm25["tokenizer"] = "default"
     return cfg
diff --git a/src/loci/retriever.py b/src/loci/retriever.py
index df60ab9..05285e4 100644
--- a/src/loci/retriever.py
+++ b/src/loci/retriever.py
@@ -9,6 +9,7 @@
 
 from loci.bm25 import BM25
 from loci.reranker import local_rerank
+from loci.tokenizers import get_tokenizer
 
 SYSTEM_PROMPT = (
     "You are a Q&A assistant over the user's personal knowledge base. "
@@ -24,12 +25,14 @@ def __init__(self, embedder, store, top_k: int,
                  rerank: bool = False, llm_cfg: dict | None = None,
                  rerank_provider: str = "llm",
                  local_rerank_model: str = "BAAI/bge-reranker-base",
-                 feedback_path: str | None = None):
+                 feedback_path: str | None = None,
+                 bm25_tokenizer: str = "default"):
         self.embedder, self.store, self.top_k = embedder, store, top_k
         self.hybrid, self.rrf_k = hybrid, rrf_k
         self.rerank, self.llm_cfg = rerank, llm_cfg
         self.rerank_provider = rerank_provider
         self.local_rerank_model = local_rerank_model
+        self.bm25_tokenizer = bm25_tokenizer
         self._penalties: dict[str, float] = {}
         if feedback_path and Path(feedback_path).exists():
             import json as _json
@@ -114,7 +117,9 @@ def _fuse(self, vhits: list[dict], query: str, fetch: int) -> list[dict]:
         ids, docs = self.store.all_chunks()
         if not ids:
             return vhits
-        ranked = BM25(dict(zip(ids, docs))).score(query)[:fetch]
+        ranked = BM25(dict(zip(ids, docs)),
+                      tokenizer=get_tokenizer(self.bm25_tokenizer)
+                      ).score(query)[:fetch]
         scores: dict[str, float] = {}
         for r, h in enumerate(vhits):
             scores[h["id"]] = scores.get(h["id"], 0.0) + 1.0 / (self.rrf_k + r + 1)
diff --git a/src/loci/tokenizers.py b/src/loci/tokenizers.py
new file mode 100644
index 0000000..99f724d
--- /dev/null
+++ b/src/loci/tokenizers.py
@@ -0,0 +1,48 @@
+"""BM25 tokenizers behind a tiny registry so config.toml can select one by name.
+
+`default` splits CJK text into single characters — dependency-free, the
+original behavior. `jieba` does word-level Chinese segmentation, which keeps
+phrases like 向量检索 intact for BM25 matching; it imports the optional
+`jieba` package lazily and degrades to `default` when it is not installed
+(fail-open, per project ground rules)."""
+from __future__ import annotations
+
+import re
+
+# Latin words stay whole; CJK text becomes single characters (strong enough
+# for BM25 ranking and keeps the tokenizer dependency-free).
+_TOKEN = re.compile(r"[a-z0-9]+|[一-鿿]")
+
+_registry: dict[str, object] = {}
+
+
+def register(name: str):
+    def deco(fn):
+        _registry[name] = fn
+        return fn
+    return deco
+
+
+@register("default")
+def default_tokenize(text: str) -> list[str]:
+    return _TOKEN.findall(text.lower())
+
+
+@register("jieba")
+def jieba_tokenize(text: str) -> list[str]:
+    """Word-level Chinese segmentation. Falls back to `default` when the
+    optional jieba package is not installed."""
+    try:
+        import jieba
+        jieba.setLogLevel(20)   # silence the "Building prefix dict" banner
+    except ImportError:
+        return default_tokenize(text)
+    return [t for t in jieba.cut(text.lower()) if t.strip()]
+
+
+def get_tokenizer(name: str = "default"):
+    fn = _registry.get(name or "default")
+    if fn is None:
+        raise ValueError(f"unknown tokenizer {name!r} "
+                         f"(known: {', '.join(sorted(_registry))})")
+    return fn
diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py
new file mode 100644
index 0000000..7bab3b6
--- /dev/null
+++ b/tests/test_tokenizers.py
@@ -0,0 +1,93 @@
+"""Tokenizer plugin tests: default CJK single-char vs jieba word-level."""
+import pytest
+
+from conftest import build_index, make_cfg, write_corpus
+from loci import config
+from loci.bm25 import BM25, tokenize
+from loci.tokenizers import (default_tokenize, get_tokenizer,
+                             jieba_tokenize)
+
+
+# [1] the default tokenizer keeps the original behavior
+
+def test_default_tokenizer_splits_cjk_single_chars():
+    assert tokenize("向量检索") == ["向", "量", "检", "索"]
+
+
+def test_default_tokenizer_keeps_latin_words():
+    assert tokenize("RAG embedding") == ["rag", "embedding"]
+
+
+def test_tokenize_is_alias_for_default():
+    assert tokenize("a b") == default_tokenize("a b")
+
+
+# [2] jieba tokenizer keeps Chinese words intact
+
+def test_jieba_tokenizer_segments_words():
+    pytest.importorskip("jieba")
+    toks = jieba_tokenize("向量检索和混合检索的区别")
+    assert "向量" in toks and "检索" in toks
+    assert "检索" == toks[toks.index("向量") + 1]   # 向量检索 stays one phrase
+
+
+def test_jieba_tokenizer_keeps_latin_runs():
+    pytest.importorskip("jieba")
+    toks = jieba_tokenize("embedding 模型选型")
+    assert "embedding" in toks and "模型" in toks
+
+
+# [3] registry lookups and unknown-name handling
+
+def test_get_tokenizer_known_names():
+    assert get_tokenizer("default") is default_tokenize
+    assert get_tokenizer("jieba") is jieba_tokenize
+
+
+def test_get_tokenizer_unknown_name_raises():
+    with pytest.raises(ValueError):
+        get_tokenizer("nope")
+
+
+# [4] BM25 accepts an injected tokenizer
+
+def test_bm25_accepts_injected_tokenizer():
+    corpus = {"a": "向量检索是混合检索的一部分", "b": "向量数据库存储高维向量"}
+    bm = BM25(corpus, tokenizer=jieba_tokenize)
+    scores = dict(bm.score("混合检索"))
+    assert scores.get("a", 0.0) > 0.0
+    assert "b" not in scores                     # no word overlap under jieba
+
+
+def test_bm25_default_is_unchanged():
+    corpus = {"a": "向量检索是混合检索的一部分", "b": "向量数据库存储高维向量"}
+    bm = BM25(corpus)
+    assert bm._tok is default_tokenize           # default behavior preserved
+    scores = dict(bm.score("混合检索"))
+    assert scores.get("a", 0.0) > 0.0           # single-char overlap still scores
+
+
+# [5] retriever runs hybrid search end-to-end with the jieba tokenizer
+
+def test_retriever_hybrid_with_jieba_tokenizer(tmp_path):
+    pytest.importorskip("jieba")
+    sources = write_corpus(tmp_path, {
+        "a.md": "# A\n混合检索把向量检索和BM25关键词检索融合起来",
+        "b.md": "# B\n完全没有关系的其他内容",
+    })
+    cfg = make_cfg(tmp_path, sources)
+    _, _, retriever = build_index(cfg)
+    retriever.bm25_tokenizer = "jieba"
+    hits = retriever.search("混合检索")
+    assert hits and hits[0]["source"].replace("\\", "/").endswith("/a.md")
+
+
+# [6] config validates tokenizer names and warns on unknown values
+
+def test_config_unknown_tokenizer_falls_back(tmp_path, capsys):
+    p = tmp_path / "c.toml"
+    p.write_text('[bm25]\ntokenizer = "foo"\n[[sources]]\npath = "X"',
+                 encoding="utf-8")
+    cfg = config.load(str(p))
+    assert cfg.bm25["tokenizer"] == "default"
+    assert "bm25.tokenizer" in capsys.readouterr().err

From 2fe64caeb88d08f328f53dfb2aa61b3c08bfae4f Mon Sep 17 00:00:00 2001
From: hlm <2514266720@qq.com>
Date: Tue, 29 Sep 2026 22:10:36 +0800
Subject: [PATCH 2/3] feat: Gradio web UI with evidence panel (`loci webui`,
 optional [ui] extra)

- src/loci/webui.py: chat pane + per-answer evidence panel reusing
  build()/make_retriever() from the CLI pipeline; ask_fn keeps the last
  four turns as history; format_evidence() renders source > section
- cli: new `webui` subcommand with --host/--port overrides
- config: [webui] section (host 127.0.0.1, port 7860)
- pyproject: ui = ["gradio>=4"] optional extra (no new runtime deps)
- tests/test_webui.py: 10 offline tests (evidence formatting, ask
  callback, history pass-through, app construction, config defaults)
---
 config.example.toml |   4 ++
 pyproject.toml      |   1 +
 src/loci/cli.py     |  17 +++++++
 src/loci/config.py  |   2 +
 src/loci/webui.py   |  87 ++++++++++++++++++++++++++++++++++
 tests/test_webui.py | 111 ++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 222 insertions(+)
 create mode 100644 src/loci/webui.py
 create mode 100644 tests/test_webui.py

diff --git a/config.example.toml b/config.example.toml
index ac826a4..cb9b48b 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -38,5 +38,9 @@ tokenizer = "default"   # "default" = CJK single-char (no deps) | "jieba" = word
 [watch]
 interval = 30     # seconds between scans in `watch` mode
 
+[webui]
+host = "127.0.0.1"   # `loci webui` bind address (Gradio UI, pip install 'loci-rag[ui]')
+port = 7860
+
 [store]
 path = "./chroma_db"   # where the vector store lives on disk
diff --git a/pyproject.toml b/pyproject.toml
index 5c13242..ee4b381 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,6 +35,7 @@ ocr = ["rapidocr-onnxruntime>=1.3"]
 docx = ["python-docx>=1.1"]
 rerank = ["sentence-transformers>=3"]
 zh = ["jieba>=0.42"]
+ui = ["gradio>=4"]
 dev = ["pytest>=8"]
 
 [project.scripts]
diff --git a/src/loci/cli.py b/src/loci/cli.py
index 9c52669..979e147 100644
--- a/src/loci/cli.py
+++ b/src/loci/cli.py
@@ -127,6 +127,14 @@ def cmd_ask(cfg, question: str, rerank: bool | None = None,
         print("\n" + verify_answer(cfg.llm, question, reply, hits))
 
 
+def cmd_webui(cfg, host: str | None = None, port: int | None = None) -> None:
+    """Serve the Gradio web UI (optional `ui` extra) over the same pipeline."""
+    from loci.webui import run_app   # deferred: webui imports cli back
+    host = host or cfg.webui.get("host") or "127.0.0.1"
+    port = port or int(cfg.webui.get("port") or 7860)
+    run_app(cfg, host=host, port=port)
+
+
 def cmd_feedback(cfg, verdict: str) -> None:
     """Rate the chunks used in the last ask; bad ratings down-weight chunks."""
     last = Path(cfg.store["path"]) / ".last_ask.json"
@@ -517,6 +525,12 @@ def main() -> None:
     p_bench.add_argument("--tokenizer", default=None,
                          help="BM25 tokenizer override for this run (default|jieba)")
 
+    p_webui = sub.add_parser("webui", help="run the Gradio web UI (optional [ui] extra)")
+    p_webui.add_argument("--host", default=None,
+                         help="bind address (default: [webui] host, or 127.0.0.1)")
+    p_webui.add_argument("--port", type=int, default=None,
+                         help="port (default: [webui] port, or 7860)")
+
     sub.add_parser("stats", help="show what is in the index")
     sub.add_parser("doctor", help="check config, endpoints, and store health")
 
@@ -564,6 +578,9 @@ def main() -> None:
         cmd_sync(cfg, args.direction)
     elif args.cmd == "bench":
         cmd_bench(cfg, args.cases, k=args.k, tokenizer=args.tokenizer)
+    elif args.cmd == "webui":
+        cfg.validate()
+        cmd_webui(cfg, host=args.host, port=args.port)
     elif args.cmd == "wiki":
         cmd_wiki_suggest(cfg) if getattr(args, "suggest", False) else cmd_wiki(cfg, args.topic, k=args.k)
     elif args.cmd == "watch":
diff --git a/src/loci/config.py b/src/loci/config.py
index a3d13c3..99dc4c0 100644
--- a/src/loci/config.py
+++ b/src/loci/config.py
@@ -24,6 +24,7 @@
     "chat": {"auto_extract": True},
     "sync": {"remote": ""},
     "http": {"host": "127.0.0.1", "port": 8765, "token": ""},
+    "webui": {"host": "127.0.0.1", "port": 7860},
     "store": {"path": "./chroma_db"},
 }
 
@@ -42,6 +43,7 @@ class Config:
     chat: dict = field(default_factory=dict)
     sync: dict = field(default_factory=dict)
     http: dict = field(default_factory=dict)
+    webui: dict = field(default_factory=dict)
     store: dict = field(default_factory=dict)
     sources: list = field(default_factory=list)
 
diff --git a/src/loci/webui.py b/src/loci/webui.py
new file mode 100644
index 0000000..142080d
--- /dev/null
+++ b/src/loci/webui.py
@@ -0,0 +1,87 @@
+"""Gradio web UI over the same retrieval pipeline the CLI uses.
+
+Requires the optional `ui` extra (gradio): `pip install 'loci-rag[ui]'`.
+`loci webui` serves a chat pane next to the evidence chunks the answer was
+grounded on, so every answer stays auditable."""
+from __future__ import annotations
+
+import os
+
+
+def format_evidence(hits: list[dict]) -> str:
+    """Render retrieved chunks as markdown evidence (source > section + text)."""
+    lines: list[str] = []
+    for i, h in enumerate(hits, 1):
+        src = h["source"].replace("\\", "/")
+        where = f" > {h['section']}" if h.get("section") else ""
+        lines.append(f"### [{i}] {src}{where}")
+        lines.append("")
+        lines.append("```")
+        lines.append(h["text"][:400].replace("`", "'"))
+        lines.append("```")
+        lines.append("")
+    return "\n".join(lines) if lines else "（无检索结果）"
+
+
+def stats_markdown(cfg, store) -> str:
+    """One-glance pipeline status for the sidebar."""
+    return "\n".join([
+        f"- **chunks**: {store.count()}",
+        f"- **embed**: {cfg.embed['model']}",
+        f"- **llm**: {cfg.llm['model']}",
+        f"- **hybrid**: {cfg.retrieval['hybrid']}, rrf_k={cfg.retrieval['rrf_k']}",
+        f"- **bm25 tokenizer**: {cfg.bm25.get('tokenizer', 'default')}",
+    ])
+
+
+def make_ask_fn(cfg, retriever):
+    """ChatInterface callback: search -> answer with citations -> evidence panel."""
+    def ask_fn(question: str, history: list) -> tuple[str, str]:
+        from loci.retriever import answer
+        if not (question or "").strip():
+            return "请输入问题", "（无）"
+        hits = retriever.search(question)
+        if not hits:
+            return "(nothing relevant in the knowledge base)", "（无检索结果）"
+        msgs: list[dict] = []
+        for turn in (history or [])[-4:]:   # keep the last four turns
+            if isinstance(turn, dict):
+                msgs.append(turn)
+            elif isinstance(turn, (tuple, list)) and len(turn) >= 2:
+                msgs.append({"role": "user", "content": str(turn[0])})
+                msgs.append({"role": "assistant", "content": str(turn[1])})
+        reply = answer(cfg.llm, question, hits, history=msgs or None)
+        return reply, format_evidence(hits)
+    return ask_fn
+
+
+def build_app(cfg):
+    """Build (not launch) the Gradio Blocks app for `loci webui` and tests."""
+    import gradio as gr
+    from loci.cli import build, make_retriever   # deferred: cli imports webui
+    embedder, store = build(cfg)
+    retriever = make_retriever(cfg, embedder, store)
+    ask_fn = make_ask_fn(cfg, retriever)
+
+    with gr.Blocks(title="loci — 本地知识库问答") as app:
+        gr.Markdown("# loci 🧠 本地知识库问答")
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("### 索引状态")
+                stats = gr.Markdown(stats_markdown(cfg, store))
+                gr.Button("刷新状态").click(fn=lambda: stats_markdown(cfg, store),
+                                            outputs=stats)
+            with gr.Column(scale=3):
+                gr.ChatInterface(
+                    fn=ask_fn,
+                    additional_outputs=[gr.Markdown(label="检索证据")],
+                    description="答案基于右侧检索证据生成，引用指向源文档章节",
+                )
+    return app
+
+
+def run_app(cfg, host: str | None = None, port: int | None = None) -> None:
+    host = host or cfg.webui.get("host") or "127.0.0.1"
+    port = port or int(cfg.webui.get("port") or 7860)
+    print(f"loci webui: http://{host}:{port}")
+    build_app(cfg).launch(server_name=host, server_port=port)
diff --git a/tests/test_webui.py b/tests/test_webui.py
new file mode 100644
index 0000000..8d6eb54
--- /dev/null
+++ b/tests/test_webui.py
@@ -0,0 +1,111 @@
+"""Web UI tests: evidence formatting, ask callback, offline app construction.
+
+Everything here is offline (FakeEmbedder); gradio is optional and tests that
+need it are skipped when the [ui] extra is not installed."""
+import pytest
+
+from conftest import build_index, make_cfg, write_corpus
+
+
+# [1] evidence panel formatting
+
+def test_format_evidence_renders_source_section_and_text():
+    from loci.webui import format_evidence
+    hits = [{"source": "/n/01.md", "section": "本地部署", "text": "正文`code`内容"}]
+    md = format_evidence(hits)
+    assert "[1] /n/01.md > 本地部署" in md
+    assert "正文'code'内容" in md          # backticks neutralized: code fence survives
+
+
+def test_format_evidence_empty_and_no_section():
+    from loci.webui import format_evidence
+    assert format_evidence([]) == "（无检索结果）"
+    assert ">" not in format_evidence([{"source": "x.md", "section": "", "text": "t"}])
+
+
+# [2] ask callback: search -> answer -> (reply, evidence)
+
+def test_ask_fn_returns_reply_and_evidence(monkeypatch, tmp_path):
+    import loci.retriever
+    from loci.webui import make_ask_fn
+    sources = write_corpus(tmp_path, {"a.md": "# A\n混合检索融合向量检索和BM25"})
+    cfg = make_cfg(tmp_path, sources)
+    _, _, retriever = build_index(cfg)
+    monkeypatch.setattr(loci.retriever, "answer",
+                        lambda llm_cfg, q, hits, history=None: "混合检索=两者融合")
+    reply, evidence = make_ask_fn(cfg, retriever)("混合检索", [])
+    assert "混合检索" in reply
+    assert "a.md" in evidence and "A" in evidence
+
+
+def test_ask_fn_empty_question(tmp_path):
+    from loci.webui import make_ask_fn
+    cfg = make_cfg(tmp_path, write_corpus(tmp_path, {"a.md": "# A\n内容"}))
+    _, _, retriever = build_index(cfg)
+    assert make_ask_fn(cfg, retriever)("   ", [])[0] == "请输入问题"
+
+
+def test_ask_fn_empty_index_guidance(tmp_path):
+    from loci.webui import make_ask_fn
+    cfg = make_cfg(tmp_path, [])
+    _, _, retriever = build_index(cfg)
+    reply, evidence = make_ask_fn(cfg, retriever)("任意问题", [])
+    assert "nothing relevant" in reply
+    assert evidence == "（无检索结果）"
+
+
+def test_ask_fn_passes_recent_history_to_answer(monkeypatch, tmp_path):
+    import loci.retriever
+    from loci.webui import make_ask_fn
+    sources = write_corpus(tmp_path, {"a.md": "# A\n混合检索融合向量检索和BM25"})
+    cfg = make_cfg(tmp_path, sources)
+    _, _, retriever = build_index(cfg)
+    captured = {}
+    monkeypatch.setattr(loci.retriever, "answer",
+                        lambda llm_cfg, q, hits, history=None: captured.update(history=history))
+    ask_fn = make_ask_fn(cfg, retriever)
+    ask_fn("混合检索", [("你好", "你好，请问"), ("什么是RAG", "RAG是检索增强生成")])
+    assert [m["role"] for m in captured["history"]] == \
+        ["user", "assistant", "user", "assistant"]
+
+
+# [3] stats sidebar + app construction (offline)
+
+def test_stats_markdown_lists_pipeline_status(tmp_path):
+    from loci.webui import stats_markdown
+    sources = write_corpus(tmp_path, {"a.md": "# A\n内容"})
+    cfg = make_cfg(tmp_path, sources)
+    _, store, _ = build_index(cfg)
+    md = stats_markdown(cfg, store)
+    assert "chunks" in md and "hybrid" in md and "bm25 tokenizer" in md
+
+
+def test_build_app_constructs_blocks_offline(tmp_path):
+    pytest.importorskip("gradio")
+    from loci.webui import build_app
+    sources = write_corpus(tmp_path, {"a.md": "# A\n内容"})
+    cfg = make_cfg(tmp_path, sources)
+    cfg.llm["api_key"] = "test-key"      # build() constructs a real client (no calls)
+    cfg.embed["api_key"] = "test-key"
+    app = build_app(cfg)
+    assert app is not None
+
+
+# [4] config defaults + cli wiring
+
+def test_config_defaults_include_webui(tmp_path):
+    from loci import config
+    p = tmp_path / "c.toml"
+    p.write_text('[[sources]]\npath = "X"', encoding="utf-8")
+    assert config.load(str(p)).webui == {"host": "127.0.0.1", "port": 7860}
+
+
+def test_cmd_webui_passes_host_and_port(monkeypatch):
+    import loci.cli as cli
+    import loci.webui as webui
+    from loci.config import Config
+    called = {}
+    monkeypatch.setattr(webui, "run_app",
+                        lambda c, host=None, port=None: called.update(host=host, port=port))
+    cli.cmd_webui(Config(), host="0.0.0.0", port=9999)
+    assert called == {"host": "0.0.0.0", "port": 9999}

From f57a907e76b571b01491b2a1d19d517b03ad9718 Mon Sep 17 00:00:00 2001
From: hlm <2514266720@qq.com>
Date: Wed, 7 Oct 2026 21:19:21 +0800
Subject: [PATCH 3/3] =?UTF-8?q?feat:=20retrieval=20quality=20pass=20?=
 =?UTF-8?q?=E2=80=94=20per-doc=20diversity=20cap,=20strict=20grounding=20p?=
 =?UTF-8?q?rompt,=20--top=5Fk=20alias,=20missing-extra=20guidance?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- retriever: max_per_doc (default 2) caps chunks per source document in
  one result list, applied after RRF fusion/rerank so surplus chunks give
  way to other docs; [retrieval] max_per_doc config (0 = no cap)
- retriever: SYSTEM_PROMPT rewritten as 4 strict rules — excerpts only,
  explicit "not in KB" answer, bullet structure, mandatory Sources line
- cli: search/ask/bench accept -k/--top_k alias (dest=k keeps compat)
- webui: evidence panel shows document name > section (full path stays in
  the answer citations)
- loaders/tokenizers: missing pdf/docx/ocr/jieba extras now warn once per
  run with the exact pip install command instead of skipping silently
- tests: +14 (diversity cap, prompt rules, --top_k wiring, dep warnings)
---
 .gitignore                      |  2 +
 config.example.toml             |  1 +
 src/loci/cli.py                 | 14 +++++--
 src/loci/config.py              |  3 +-
 src/loci/loaders.py             | 23 +++++++++--
 src/loci/retriever.py           | 46 +++++++++++++++++++---
 src/loci/tokenizers.py          | 13 ++++++-
 src/loci/webui.py               |  8 +++-
 tests/test_cli_args.py          | 50 ++++++++++++++++++++++++
 tests/test_missing_deps.py      | 51 ++++++++++++++++++++++++
 tests/test_retrieval_quality.py | 69 +++++++++++++++++++++++++++++++++
 tests/test_webui.py             |  2 +-
 12 files changed, 263 insertions(+), 19 deletions(-)
 create mode 100644 tests/test_cli_args.py
 create mode 100644 tests/test_missing_deps.py
 create mode 100644 tests/test_retrieval_quality.py

diff --git a/.gitignore b/.gitignore
index 39784be..b033ddb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,5 @@ generated/
 dist/
 build/
 *.egg-info/
+
+.venv/
diff --git a/config.example.toml b/config.example.toml
index cb9b48b..f129d7f 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -30,6 +30,7 @@ search = 5        # number of hits per search
 [retrieval]
 hybrid = true     # fuse vector search with BM25 (Reciprocal Rank Fusion)
 rrf_k = 60        # RRF smoothing constant; higher = flatter fusion weights
+max_per_doc = 2   # max chunks one document may contribute per search (0 = no cap)
 
 [bm25]
 tokenizer = "default"   # "default" = CJK single-char (no deps) | "jieba" = word-level
diff --git a/src/loci/cli.py b/src/loci/cli.py
index 979e147..71064d1 100644
--- a/src/loci/cli.py
+++ b/src/loci/cli.py
@@ -30,6 +30,8 @@ def build(cfg):
 
 
 def make_retriever(cfg, embedder, store):
+    """Build the Retriever from config — the single wiring point every entry
+    (CLI / HTTP API / WebUI) shares, so options can't drift apart."""
     from loci.retriever import Retriever
     import os as _os
     fb_path = _os.path.join(cfg.store["path"], "feedback.jsonl")
@@ -40,7 +42,8 @@ def make_retriever(cfg, embedder, store):
                      rerank_provider=cfg.retrieval.get("rerank_provider", "llm"),
                      local_rerank_model=cfg.retrieval.get(
                          "local_rerank_model", "BAAI/bge-reranker-base"),
-                     bm25_tokenizer=cfg.bm25.get("tokenizer", "default"))
+                     bm25_tokenizer=cfg.bm25.get("tokenizer", "default"),
+                     max_per_doc=cfg.retrieval.get("max_per_doc", 2))
 
 
 def cmd_ingest(cfg, force: bool = False) -> None:
@@ -466,7 +469,8 @@ def main() -> None:
                           help="only files modified on/after DATE (YYYY-MM-DD or YYYY-MM)")
     p_search.add_argument("-e", "--exact", metavar="PHRASE",
                           help="only hits containing this exact phrase")
-    p_search.add_argument("-k", type=int, help="override top_k")
+    p_search.add_argument("-k", "--top_k", dest="k", type=int,
+                          help="override top_k (retrieved chunks per query)")
     p_search.add_argument("--rerank", nargs="?", const=True, default=None,
                           metavar="PROVIDER", help="rerank candidates; optional "
                           "value llm|local overrides [retrieval] rerank_provider")
@@ -477,7 +481,8 @@ def main() -> None:
                        help="scope retrieval to paths containing this substring")
     p_ask.add_argument("--since", metavar="DATE", type=parse_since,
                        help="only files modified on/after DATE (YYYY-MM-DD or YYYY-MM)")
-    p_ask.add_argument("-k", type=int, help="override top_k")
+    p_ask.add_argument("-k", "--top_k", dest="k", type=int,
+                       help="override top_k (retrieved chunks per query)")
     p_ask.add_argument("--rerank", nargs="?", const=True, default=None,
                        metavar="PROVIDER", help="rerank candidates; optional "
                        "value llm|local overrides [retrieval] rerank_provider")
@@ -521,7 +526,8 @@ def main() -> None:
 
     p_bench = sub.add_parser("bench", help="run a retrieval benchmark (hit@k) on a cases file")
     p_bench.add_argument("cases", help="JSONL file with {query, expect} per line")
-    p_bench.add_argument("-k", type=int, default=5, help="hits per query (default 5)")
+    p_bench.add_argument("-k", "--top_k", dest="k", type=int, default=5,
+                         help="hits per query (default 5)")
     p_bench.add_argument("--tokenizer", default=None,
                          help="BM25 tokenizer override for this run (default|jieba)")
 
diff --git a/src/loci/config.py b/src/loci/config.py
index 99dc4c0..6b6969e 100644
--- a/src/loci/config.py
+++ b/src/loci/config.py
@@ -16,7 +16,8 @@
     "top_k": {"search": 5},
     "retrieval": {"hybrid": True, "rrf_k": 60, "rerank": False,
                   "rerank_provider": "llm",
-                  "local_rerank_model": "BAAI/bge-reranker-base"},
+                  "local_rerank_model": "BAAI/bge-reranker-base",
+                  "max_per_doc": 2},   # chunks per document in one result list
     "bm25": {"tokenizer": "default"},
     "watch": {"interval": 30},
     "memories": {"path": "./memories"},
diff --git a/src/loci/loaders.py b/src/loci/loaders.py
index bead535..c19c39c 100644
--- a/src/loci/loaders.py
+++ b/src/loci/loaders.py
@@ -38,6 +38,18 @@
 
 _ocr_engine = None  # lazy singleton (model load is ~1s)
 
+_warned_missing: set[str] = set()   # extras already warned about this run
+
+
+def _warn_missing_extra(extra: str, install: str) -> None:
+    """Warn once per run that files need an optional extra, with the exact
+    install command — silently skipping .pdf/.docx files confuses users."""
+    if extra in _warned_missing:
+        return
+    _warned_missing.add(extra)
+    print(f"[warn] {extra} sources found but the optional dependency is not "
+          f"installed — skipped. Install it with: {install}")
+
 IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tiff"}
 
 SUFFIXES = {".md", ".txt"}
@@ -296,10 +308,12 @@ def _read_text(p: Path) -> str | None:
             except Exception as e:
                 print(f"[warn] pdf unreadable, skipping: {p.name} ({e})")
                 return None
-        return None  # no pdf extra installed; `doctor` mentions it
+        _warn_missing_extra("pdf", "pip install 'loci-rag[pdf]'")
+        return None  # no pdf extra installed
     if suffix == ".docx":
         if not HAS_DOCX:
-            return None  # silently skip; `doctor` mentions the optional extra
+            _warn_missing_extra("docx", "pip install 'loci-rag[docx]'")
+            return None
         try:
             document = _docx.Document(str(p))
             parts = [para.text for para in document.paragraphs if para.text.strip()]
@@ -389,9 +403,10 @@ def _get_ocr():
 
 
 def _ocr_image(p: Path) -> str | None:
-    """OCR one image file; None (skip silently) when OCR isn't installed."""
+    """OCR one image file; None (skip) when OCR isn't installed."""
     if not HAS_OCR:
-        return None  # `doctor` mentions the optional [ocr] extra
+        _warn_missing_extra("ocr", "pip install 'loci-rag[ocr]'")
+        return None
     try:
         result, _ = _get_ocr()(str(p))
         if not result:
diff --git a/src/loci/retriever.py b/src/loci/retriever.py
index 05285e4..a37d5d6 100644
--- a/src/loci/retriever.py
+++ b/src/loci/retriever.py
@@ -11,28 +11,43 @@
 from loci.reranker import local_rerank
 from loci.tokenizers import get_tokenizer
 
+# Strict grounding rules: no fabrication, explicit "not in KB" answer, and a
+# mandatory source list so every claim is traceable to a document section.
 SYSTEM_PROMPT = (
     "You are a Q&A assistant over the user's personal knowledge base. "
-    "Answer only from the provided excerpts; if they don't contain the answer, "
-    "say so plainly — do not invent. End your reply with a list of citations "
-    "in the form [source: file path > section]."
+    "Follow these rules strictly:\n"
+    "1. Answer ONLY from the provided excerpts. Never use outside knowledge "
+    "to fill gaps — do not invent or guess.\n"
+    "2. If the excerpts do not contain the answer, say so plainly in the "
+    "user's language (e.g. \"知识库中没有相关信息\") and stop.\n"
+    "3. Structure the answer as short bullet points; each point must be "
+    "grounded in at least one excerpt.\n"
+    "4. End the reply with a 'Sources:' line citing every excerpt used, in "
+    "the form [source: file path > section]."
 )
 
 
 class Retriever:
+    """Vector (+BM25) retrieval over a Store, fused via RRF.
+
+    `max_per_doc` caps how many chunks one source document may contribute to
+    a single result list, so one long file can't crowd out other relevant
+    docs (0 = no cap)."""
     def __init__(self, embedder, store, top_k: int,
                  hybrid: bool = True, rrf_k: int = 60,
                  rerank: bool = False, llm_cfg: dict | None = None,
                  rerank_provider: str = "llm",
                  local_rerank_model: str = "BAAI/bge-reranker-base",
                  feedback_path: str | None = None,
-                 bm25_tokenizer: str = "default"):
+                 bm25_tokenizer: str = "default",
+                 max_per_doc: int = 2):
         self.embedder, self.store, self.top_k = embedder, store, top_k
         self.hybrid, self.rrf_k = hybrid, rrf_k
         self.rerank, self.llm_cfg = rerank, llm_cfg
         self.rerank_provider = rerank_provider
         self.local_rerank_model = local_rerank_model
         self.bm25_tokenizer = bm25_tokenizer
+        self.max_per_doc = max_per_doc
         self._penalties: dict[str, float] = {}
         if feedback_path and Path(feedback_path).exists():
             import json as _json
@@ -98,7 +113,28 @@ def search(self, query: str, tag: str | None = None,
                 fused = local_rerank(query, fused, self.local_rerank_model)
             elif self.llm_cfg:
                 fused = rerank_hits(self.llm_cfg, query, fused)
-        return fused[:limit]
+        # diversity cap AFTER ranking/filters: the best chunk per doc keeps
+        # its slot, surplus chunks give way to other documents
+        return self._diversify(fused, limit)[:limit]
+
+    def _diversify(self, hits: list[dict], limit: int) -> list[dict]:
+        """Keep at most `max_per_doc` chunks per source document (order kept).
+
+        Applied as the last step before truncation, so a single document that
+        dominates the ranking loses its surplus slots to other docs."""
+        if self.max_per_doc <= 0 or not hits:
+            return hits
+        seen: dict[str, int] = {}
+        out: list[dict] = []
+        for h in hits:
+            src = h.get("source", "")
+            if seen.get(src, 0) >= self.max_per_doc:
+                continue
+            seen[src] = seen.get(src, 0) + 1
+            out.append(h)
+            if len(out) >= limit:
+                break
+        return out
 
     @staticmethod
     def _rrf_merge(ranked_lists: list[list[dict]], rrf_k: int = 60) -> list[dict]:
diff --git a/src/loci/tokenizers.py b/src/loci/tokenizers.py
index 99f724d..50c96ab 100644
--- a/src/loci/tokenizers.py
+++ b/src/loci/tokenizers.py
@@ -28,14 +28,23 @@ def default_tokenize(text: str) -> list[str]:
     return _TOKEN.findall(text.lower())
 
 
+_warned_jieba = False   # warn once per process, not once per query
+
+
 @register("jieba")
 def jieba_tokenize(text: str) -> list[str]:
-    """Word-level Chinese segmentation. Falls back to `default` when the
-    optional jieba package is not installed."""
+    """Word-level Chinese segmentation. Falls back to `default` (with a
+    one-time install hint) when the optional jieba package is not installed."""
+    global _warned_jieba
     try:
         import jieba
         jieba.setLogLevel(20)   # silence the "Building prefix dict" banner
     except ImportError:
+        if not _warned_jieba:
+            _warned_jieba = True
+            print("[warn] bm25.tokenizer = 'jieba' but jieba is not installed — "
+                  "falling back to the single-char tokenizer. "
+                  "Install it with: pip install 'loci-rag[zh]'")
         return default_tokenize(text)
     return [t for t in jieba.cut(text.lower()) if t.strip()]
 
diff --git a/src/loci/webui.py b/src/loci/webui.py
index 142080d..a9a2f84 100644
--- a/src/loci/webui.py
+++ b/src/loci/webui.py
@@ -9,10 +9,14 @@
 
 
 def format_evidence(hits: list[dict]) -> str:
-    """Render retrieved chunks as markdown evidence (source > section + text)."""
+    """Render retrieved chunks as markdown evidence (doc name > section + text).
+
+    The chat pane is for humans, so only the document name is shown here —
+    the full path stays in the answer's [source: ...] citations."""
     lines: list[str] = []
     for i, h in enumerate(hits, 1):
-        src = h["source"].replace("\\", "/")
+        # document name only (chatlog sources keep their "path::title" suffix)
+        src = h["source"].replace("\\", "/").rsplit("/", 1)[-1]
         where = f" > {h['section']}" if h.get("section") else ""
         lines.append(f"### [{i}] {src}{where}")
         lines.append("")
diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py
new file mode 100644
index 0000000..47f00e5
--- /dev/null
+++ b/tests/test_cli_args.py
@@ -0,0 +1,50 @@
+"""CLI argument wiring: --top_k alias applies the same override as -k."""
+import sys
+
+import loci.cli as cli_module
+from conftest import make_cfg
+
+
+def _run_main(args, tmp_path, monkeypatch):
+    captured = {}
+
+    def fake_cmd_search(cfg, query, **kw):
+        captured["top_k"] = cfg.top_k["search"]
+        captured["kw"] = kw
+
+    fake = make_cfg(tmp_path, [])
+    fake.sources = [{"path": str(tmp_path)}]   # pass cfg.validate()
+    fake.llm["api_key"] = "test-key"
+    fake.embed["api_key"] = "test-key"
+    monkeypatch.setattr(cli_module.config, "load", lambda path="config.toml": fake)
+    monkeypatch.setattr(cli_module, "cmd_search", fake_cmd_search)
+    monkeypatch.setattr(sys, "argv", ["loci", *args])
+    cli_module.main()
+    return captured
+
+
+def test_search_top_k_alias_override(tmp_path, monkeypatch):
+    captured = _run_main(["search", "--top_k", "7", "测试查询"], tmp_path, monkeypatch)
+    assert captured["top_k"] == 7
+
+
+def test_search_short_k_still_works(tmp_path, monkeypatch):
+    captured = _run_main(["search", "-k", "3", "测试查询"], tmp_path, monkeypatch)
+    assert captured["top_k"] == 3
+
+
+def test_ask_top_k_alias_override(tmp_path, monkeypatch):
+    captured = {}
+
+    def fake_cmd_ask(cfg, question, **kw):
+        captured["top_k"] = cfg.top_k["search"]
+
+    fake = make_cfg(tmp_path, [])
+    fake.sources = [{"path": str(tmp_path)}]   # pass cfg.validate()
+    fake.llm["api_key"] = "test-key"
+    fake.embed["api_key"] = "test-key"
+    monkeypatch.setattr(cli_module.config, "load", lambda path="config.toml": fake)
+    monkeypatch.setattr(cli_module, "cmd_ask", fake_cmd_ask)
+    monkeypatch.setattr(sys, "argv", ["loci", "ask", "--top_k", "9", "测试问题"])
+    cli_module.main()
+    assert captured["top_k"] == 9
diff --git a/tests/test_missing_deps.py b/tests/test_missing_deps.py
new file mode 100644
index 0000000..83d4c86
--- /dev/null
+++ b/tests/test_missing_deps.py
@@ -0,0 +1,51 @@
+"""Optional-dependency UX: files needing a missing extra are skipped with a
+clear one-time install hint instead of vanishing silently."""
+import sys
+
+import pytest
+
+import loci.loaders as loaders
+
+
+# [1] pdf/docx files with no optional extra installed
+
+def test_missing_pdf_extra_warns_once(tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(loaders, "HAS_PDF", False)
+    monkeypatch.setattr(loaders, "HAS_PDF_TABLES", False)
+    monkeypatch.setattr(loaders, "_warned_missing", set())
+    (tmp_path / "a.pdf").write_bytes(b"%PDF-1.4 fake")
+    (tmp_path / "b.pdf").write_bytes(b"%PDF-1.4 fake")
+    assert loaders.scan_sources([{"path": str(tmp_path)}]) == []
+    out = capsys.readouterr().out
+    assert out.count("[warn] pdf sources") == 1        # once per run, not per file
+    assert "pip install 'loci-rag[pdf]'" in out
+
+
+def test_missing_docx_extra_warns_once(tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(loaders, "HAS_DOCX", False)
+    monkeypatch.setattr(loaders, "_warned_missing", set())
+    (tmp_path / "a.docx").write_bytes(b"PK fake")
+    assert loaders.scan_sources([{"path": str(tmp_path)}]) == []
+    out = capsys.readouterr().out
+    assert "[warn] docx sources" in out
+    assert "pip install 'loci-rag[docx]'" in out
+
+
+def test_installed_extra_produces_no_warning(tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(loaders, "_warned_missing", set())
+    assert loaders.scan_sources([{"path": str(tmp_path)}]) == []
+    assert "[warn] pdf sources" not in capsys.readouterr().out
+
+
+# [2] jieba selected in config but not installed
+
+def test_jieba_missing_warns_once_and_falls_back(monkeypatch, capsys):
+    from loci import tokenizers
+    monkeypatch.setitem(sys.modules, "jieba", None)     # force ImportError
+    monkeypatch.setattr(tokenizers, "_warned_jieba", False)
+    out1 = tokenizers.jieba_tokenize("向量检索")
+    out2 = tokenizers.jieba_tokenize("向量检索")        # second call: no repeat
+    err = capsys.readouterr().out
+    assert out1 == tokenizers.default_tokenize("向量检索")   # fail-open fallback
+    assert out2 == out1
+    assert err.count("loci-rag[zh]") == 1
diff --git a/tests/test_retrieval_quality.py b/tests/test_retrieval_quality.py
new file mode 100644
index 0000000..8d800ba
--- /dev/null
+++ b/tests/test_retrieval_quality.py
@@ -0,0 +1,69 @@
+"""Retrieval-quality improvements: per-document diversity cap and the
+strict grounding system prompt."""
+from collections import Counter
+
+from conftest import build_index, make_cfg, write_corpus
+
+
+# [1] diversity cap: one document can't crowd out the whole result list
+
+def test_diversify_caps_chunks_per_document():
+    from loci.retriever import Retriever
+    r = Retriever(None, None, top_k=5, max_per_doc=2)
+    hits = [{"id": str(i), "source": "a.md"} for i in range(4)] + \
+           [{"id": "b1", "source": "b.md"}, {"id": "c1", "source": "c.md"}]
+    out = r._diversify(hits, 5)
+    sources = [h["source"] for h in out]
+    assert sources.count("a.md") == 2          # surplus a.md chunks dropped
+    assert "b.md" in sources and "c.md" in sources   # slots freed for others
+
+
+def test_diversify_zero_disables_cap():
+    from loci.retriever import Retriever
+    r = Retriever(None, None, top_k=5, max_per_doc=0)
+    hits = [{"id": str(i), "source": "a.md"} for i in range(4)]
+    assert len(r._diversify(hits, 4)) == 4     # 0 = keep the original behavior
+
+
+def test_diversify_preserves_ranking_order():
+    from loci.retriever import Retriever
+    r = Retriever(None, None, top_k=5, max_per_doc=2)
+    hits = [{"id": "a1", "source": "a.md"}, {"id": "b1", "source": "b.md"},
+            {"id": "a2", "source": "a.md"}, {"id": "a3", "source": "a.md"}]
+    out = r._diversify(hits, 5)
+    assert [h["id"] for h in out] == ["a1", "b1", "a2"]
+
+
+def test_search_applies_diversity_cap(tmp_path):
+    sources = write_corpus(tmp_path, {
+        "a.md": "# A\n" + "\n\n".join(f"向量检索段落{i} 内容" for i in range(6)),
+        "b.md": "# B\n向量检索的另一篇文档段落",
+    })
+    sources[0]["chunk_size"] = 20             # force a.md into many chunks
+    sources[0]["chunk_overlap"] = 0
+    cfg = make_cfg(tmp_path, sources)
+    _, _, retriever = build_index(cfg)
+    retriever.top_k = 5
+    hits = retriever.search("向量检索")
+    counts = Counter(h["source"].replace("\\", "/").split("/")[-1] for h in hits)
+    assert counts["a.md"] <= 2                 # cap holds end-to-end
+    assert counts.get("b.md", 0) >= 1          # the other doc gets a slot
+
+
+# [2] system prompt: strict grounding rules
+
+def test_system_prompt_forbids_fabrication():
+    from loci.retriever import SYSTEM_PROMPT
+    assert "do not invent" in SYSTEM_PROMPT
+    assert "Never use outside knowledge" in SYSTEM_PROMPT
+
+
+def test_system_prompt_requires_explicit_unknown_answer():
+    from loci.retriever import SYSTEM_PROMPT
+    assert "知识库中没有相关信息" in SYSTEM_PROMPT
+
+
+def test_system_prompt_mandates_citations():
+    from loci.retriever import SYSTEM_PROMPT
+    assert "Sources:" in SYSTEM_PROMPT
+    assert "[source: file path > section]" in SYSTEM_PROMPT
diff --git a/tests/test_webui.py b/tests/test_webui.py
index 8d6eb54..9ca91d8 100644
--- a/tests/test_webui.py
+++ b/tests/test_webui.py
@@ -13,7 +13,7 @@ def test_format_evidence_renders_source_section_and_text():
     from loci.webui import format_evidence
     hits = [{"source": "/n/01.md", "section": "本地部署", "text": "正文`code`内容"}]
     md = format_evidence(hits)
-    assert "[1] /n/01.md > 本地部署" in md
+    assert "[1] 01.md > 本地部署" in md    # doc name only, not the full path
     assert "正文'code'内容" in md          # backticks neutralized: code fence survives