Skip to content

Commit c4d67f2

Browse files
committed
feat(query): add FTS5 keyword search as hybrid retrieval pre-filter
The query agent navigated the wiki purely by reading index.md, then summaries, concepts, and entities — several LLM turns of latency before any content is seen, and it handles "which page mentions X" lookups poorly. Add a zero-dependency SQLite FTS5 index (bm25() ranking) as a fast first-pass retrieval layer: - openkb/fts.py: builds {kb_dir}/.openkb/fts.db from the markdown under summaries/, concepts/, entities/, sources/, plus per-page rows for long-document JSON. CJK runs are indexed as overlapping bigrams so Chinese queries match properly with the stock unicode61 tokenizer — no external segmenter needed. Mixed-script tokens become adjacency phrases; FTS5 syntax characters are stripped from query tokens so pasted punctuation can never break MATCH. Frontmatter is stripped to keep wikilink lists out of snippets, and snippets are highlighted with marker-wrapped terms. The index refreshes incrementally by (mtime, size) on every search, so freshly added files are searchable without a manual rebuild. - Query agent: new `search` tool, first in the tool list, and the search-strategy prompt now starts with it, falling back to index.md navigation on no hits. Chat sessions inherit it automatically via build_chat_agent(). FTS failures degrade gracefully to navigation instead of breaking the agent loop. - CLI: `openkb fts rebuild` and `openkb fts search` for full rebuilds and quick debugging lookups. On a real ~1000-file wiki: a full build takes ~3s, and entity-style questions that previously took ~2.5 min / 6 agent turns now answer in ~25s / 2 tool calls (search -> read_file).
1 parent ff54396 commit c4d67f2

5 files changed

Lines changed: 513 additions & 4 deletions

File tree

‎openkb/agent/query.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@
2525
{schema_md}
2626
2727
## Search strategy
28-
1. Read index.md to see all documents and concepts with brief summaries.
28+
1. Use the `search` tool with the question's key terms (Chinese and/or
29+
English) first — it is a BM25-ranked full-text index over the ENTIRE wiki
30+
and returns candidate pages with snippets in milliseconds. Hits on long
31+
documents include page numbers you can pass straight to get_page_content.
32+
If `search` returns no hits or the question is about overall KB scope,
33+
read index.md to see all documents and concepts with brief summaries.
2934
Each document is marked (short) or (pageindex) to indicate its type.
3035
2. Read relevant summary pages (summaries/) for document overviews.
3136
Summaries may omit details — if you need more, follow the summary's
@@ -83,6 +88,27 @@ def get_page_content(doc_name: str, pages: str) -> str:
8388
"""
8489
return get_wiki_page_content(doc_name, pages, wiki_root)
8590

91+
@function_tool
92+
def search(query: str, limit: int = 8) -> str:
93+
"""Full-text keyword search over the entire wiki (BM25-ranked FTS5).
94+
95+
Fast first step: returns candidate pages with snippets. Supports
96+
Chinese and English terms (CJK is bigram-indexed). Multi-word queries
97+
are ANDed (falls back to OR when nothing matches). Then verify by
98+
reading the actual pages with read_file / get_page_content.
99+
100+
Args:
101+
query: Keywords, e.g. 'ETA 川普 豁免' or 'transformer attention'.
102+
limit: Max hits (default 8).
103+
"""
104+
from openkb.fts import format_hits, search as fts_search
105+
106+
kb_dir = Path(wiki_root).parent
107+
try:
108+
return format_hits(fts_search(kb_dir, query, limit=limit))
109+
except Exception as exc: # never break the agent loop on FTS issues
110+
return f"Full-text search unavailable ({exc}). Navigate via index.md instead."
111+
86112
@function_tool
87113
def get_image(image_path: str) -> ToolOutputImage | ToolOutputText:
88114
"""View an image from the wiki.
@@ -117,7 +143,7 @@ def get_image(image_path: str) -> ToolOutputImage | ToolOutputText:
117143
return Agent(
118144
name="wiki-query",
119145
instructions=instructions,
120-
tools=[read_file, get_page_content, get_image],
146+
tools=[search, read_file, get_page_content, get_image],
121147
model=f"litellm/{model}",
122148
model_settings=ModelSettings(**model_settings),
123149
)

‎openkb/cli.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1216,6 +1216,44 @@ def save_exploration(kb_dir: Path, question: str, answer: str) -> Path | None:
12161216
return explore_path
12171217

12181218

1219+
@cli.group()
1220+
def fts():
1221+
"""FTS5 full-text index (hybrid retrieval pre-filter for queries)."""
1222+
1223+
1224+
@fts.command(name="rebuild")
1225+
@click.pass_context
1226+
def fts_rebuild(ctx):
1227+
"""Rebuild the full-text index from scratch."""
1228+
kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
1229+
if kb_dir is None:
1230+
click.echo("No knowledge base found. Run `openkb init` first.")
1231+
return
1232+
from openkb.fts import rebuild_index
1233+
1234+
t0 = time.time()
1235+
files_seen, rows = rebuild_index(kb_dir)
1236+
click.echo(
1237+
f"FTS index rebuilt: {files_seen} files, {rows} rows, "
1238+
f"{time.time() - t0:.1f}s -> {kb_dir / '.openkb' / 'fts.db'}"
1239+
)
1240+
1241+
1242+
@fts.command(name="search")
1243+
@click.argument("fts_query")
1244+
@click.option("--limit", default=8, show_default=True, help="Max hits.")
1245+
@click.pass_context
1246+
def fts_search_cmd(ctx, fts_query, limit):
1247+
"""BM25 keyword search over the wiki (debug / quick lookup)."""
1248+
kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
1249+
if kb_dir is None:
1250+
click.echo("No knowledge base found. Run `openkb init` first.")
1251+
return
1252+
from openkb.fts import format_hits, search as fts_search
1253+
1254+
click.echo(format_hits(fts_search(kb_dir, fts_query, limit=limit)))
1255+
1256+
12191257
@cli.command()
12201258
@click.argument("question")
12211259
@click.option("--save", is_flag=True, default=False, help="Save the answer to wiki/explorations/.")

‎openkb/fts.py‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
"""FTS5 hybrid retrieval for OpenKB: a fast BM25-ranked keyword pre-filter
2+
over the whole wiki, complementing the query agent's navigation loop.
3+
4+
The query agent's default strategy (read index.md -> summaries -> concepts ->
5+
sources) is thorough but costs several LLM turns before any content is seen.
6+
This module gives it a millisecond keyword index to jump straight to
7+
candidate pages; the agent still verifies by reading the actual pages.
8+
9+
Design notes:
10+
11+
- Storage: ``{kb_dir}/.openkb/fts.db`` (SQLite FTS5, ``bm25()`` ranking).
12+
- CJK: FTS5's unicode61 tokenizer treats a run of CJK ideographs as ONE
13+
token, so "川普政府" would only match as a whole. We transform CJK runs
14+
into overlapping bigrams ("川普 普政 政府") before indexing and apply the
15+
same transform to query terms (classic CJK bigram trick, no external
16+
segmenter dependency).
17+
- Long docs (``sources/*.json`` page arrays) are indexed one row per page,
18+
so a hit tells the agent exactly which ``pages=`` to pass to
19+
``get_page_content``.
20+
- Freshness: incremental update keyed on (mtime, size) per file, checked on
21+
every search; a missing db triggers a full build. ``openkb fts rebuild``
22+
forces a rebuild.
23+
"""
24+
25+
from __future__ import annotations
26+
27+
import json
28+
import re
29+
import sqlite3
30+
import time
31+
from pathlib import Path
32+
33+
# CJK Unified Ideographs (+ ext A, compat). Runs of these get bigram-ised.
34+
_CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff〇]+")
35+
36+
_INDEX_DIRS = ("summaries", "concepts", "entities", "sources", "explorations")
37+
_KIND = {
38+
"summaries": "summary",
39+
"concepts": "concept",
40+
"entities": "entity",
41+
"sources": "source",
42+
"explorations": "exploration",
43+
}
44+
_FRONTMATTER_RE = re.compile(r"\A---\n.*?\n---\n", re.DOTALL)
45+
_RAW_CAP = 30000 # chars of original text kept per row for snippets
46+
47+
48+
def fts_db_path(kb_dir: Path) -> Path:
49+
return kb_dir / ".openkb" / "fts.db"
50+
51+
52+
def _bigramize(text: str) -> str:
53+
"""Replace each CJK run with space-joined overlapping bigrams."""
54+
55+
def repl(m: re.Match) -> str:
56+
s = m.group(0)
57+
if len(s) == 1:
58+
return s
59+
return " ".join(s[i : i + 2] for i in range(len(s) - 1))
60+
61+
return _CJK_RE.sub(repl, text)
62+
63+
64+
def _connect(db_path: Path) -> sqlite3.Connection:
65+
con = sqlite3.connect(str(db_path))
66+
con.execute(
67+
"CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5("
68+
"content, raw UNINDEXED, kind UNINDEXED, path UNINDEXED, page UNINDEXED,"
69+
" tokenize='unicode61')"
70+
)
71+
con.execute(
72+
"CREATE TABLE IF NOT EXISTS files("
73+
"path TEXT PRIMARY KEY, mtime REAL, size INTEGER)"
74+
)
75+
return con
76+
77+
78+
def _iter_targets(wiki_root: Path):
79+
"""Yield (rel_path, kind, page, text) for every indexable unit."""
80+
for sub in _INDEX_DIRS:
81+
d = wiki_root / sub
82+
if not d.is_dir():
83+
continue
84+
for f in sorted(d.glob("*.md")):
85+
try:
86+
text = f.read_text(encoding="utf-8", errors="replace")
87+
except OSError:
88+
continue
89+
# Drop YAML frontmatter: wikilink lists there drown real mentions
90+
# in snippets and pollute ranking.
91+
text = _FRONTMATTER_RE.sub("", text)
92+
yield (f"{sub}/{f.name}", _KIND[sub], None, text)
93+
src = wiki_root / "sources"
94+
if src.is_dir():
95+
for j in sorted(src.glob("*.json")):
96+
try:
97+
data = json.loads(j.read_text(encoding="utf-8", errors="replace"))
98+
except (OSError, ValueError):
99+
continue
100+
if not isinstance(data, list):
101+
continue
102+
for entry in data:
103+
if isinstance(entry, dict) and "page" in entry:
104+
yield (
105+
f"sources/{j.name}",
106+
"page",
107+
entry["page"],
108+
str(entry.get("content", "")),
109+
)
110+
111+
112+
def _row_for(rel: str, kind: str, page, text: str):
113+
raw = text[:_RAW_CAP]
114+
return (_bigramize(raw), raw, kind, rel, page)
115+
116+
117+
def _sync(con: sqlite3.Connection, wiki_root: Path) -> tuple[int, int]:
118+
"""Incremental update: returns (files_seen, rows_written)."""
119+
cur = con.execute("SELECT path, mtime, size FROM files")
120+
known = {p: (m, s) for p, m, s in cur.fetchall()}
121+
seen: dict[str, list] = {}
122+
files_seen = 0
123+
rows_written = 0
124+
125+
# Group rows by file so each file is replaced atomically-ish.
126+
for rel, kind, page, text in _iter_targets(wiki_root):
127+
seen.setdefault(rel, []).append((rel, kind, page, text))
128+
129+
wiki_files = {rel: (wiki_root / rel) for rel in seen}
130+
stale = [p for p in known if p not in seen]
131+
132+
for rel, rows in seen.items():
133+
f = wiki_files[rel]
134+
try:
135+
st = f.stat()
136+
except OSError:
137+
continue
138+
files_seen += 1
139+
if known.get(rel) == (st.st_mtime, st.st_size):
140+
continue
141+
con.execute("DELETE FROM fts WHERE path = ?", (rel,))
142+
con.executemany(
143+
"INSERT INTO fts (content, raw, kind, path, page) VALUES (?,?,?,?,?)",
144+
[_row_for(rel, kind, page, text) for rel, kind, page, text in rows],
145+
)
146+
con.execute(
147+
"INSERT OR REPLACE INTO files (path, mtime, size) VALUES (?,?,?)",
148+
(rel, st.st_mtime, st.st_size),
149+
)
150+
rows_written += len(rows)
151+
152+
for rel in stale:
153+
con.execute("DELETE FROM fts WHERE path = ?", (rel,))
154+
con.execute("DELETE FROM files WHERE path = ?", (rel,))
155+
156+
con.commit()
157+
return files_seen, rows_written
158+
159+
160+
def ensure_index(kb_dir: Path) -> Path:
161+
"""Build the index if missing, else incrementally refresh it."""
162+
wiki_root = kb_dir / "wiki"
163+
db_path = fts_db_path(kb_dir)
164+
db_path.parent.mkdir(parents=True, exist_ok=True)
165+
con = _connect(db_path)
166+
try:
167+
_sync(con, wiki_root)
168+
finally:
169+
con.close()
170+
return db_path
171+
172+
173+
def rebuild_index(kb_dir: Path) -> tuple[int, int]:
174+
"""Drop and rebuild the index. Returns (files_seen, rows_written)."""
175+
wiki_root = kb_dir / "wiki"
176+
db_path = fts_db_path(kb_dir)
177+
db_path.parent.mkdir(parents=True, exist_ok=True)
178+
con = _connect(db_path)
179+
try:
180+
con.execute("DROP TABLE IF EXISTS fts")
181+
con.execute("DROP TABLE IF EXISTS files")
182+
con.commit()
183+
con.close()
184+
con = _connect(db_path)
185+
stats = _sync(con, wiki_root)
186+
finally:
187+
con.close()
188+
return stats
189+
190+
191+
def _fts_expr(query: str) -> tuple[str, str, list[str]]:
192+
"""Build (AND_expr, OR_expr, raw_terms) from a free-text query."""
193+
clauses_and: list[str] = []
194+
clauses_or: list[str] = []
195+
raw_terms: list[str] = []
196+
for tok in query.split():
197+
# Drop anything that is FTS5 syntax (quotes, parens, stars, colons...)
198+
# so pasted punctuation can never break the MATCH expression.
199+
tok = re.sub(r"[^\w一-鿿豈-﫿]+", "", tok)
200+
if not tok:
201+
continue
202+
raw_terms.append(tok)
203+
subs: list[str] = []
204+
pos = 0
205+
for m in _CJK_RE.finditer(tok):
206+
if m.start() > pos:
207+
subs.append(("latin", tok[pos : m.start()]))
208+
subs.append(("cjk", m.group(0)))
209+
pos = m.end()
210+
if pos < len(tok):
211+
subs.append(("latin", tok[pos:]))
212+
213+
if len(subs) > 1:
214+
# Mixed token (e.g. "ETA公司"): adjacency phrase, no prefix star.
215+
parts = []
216+
for kind, s in subs:
217+
parts.append(_bigramize(s) if kind == "cjk" else s)
218+
clauses_and.append('"' + " ".join(parts) + '"')
219+
clauses_or.append('"' + " ".join(parts) + '"')
220+
elif subs:
221+
kind, s = subs[0]
222+
if kind == "cjk":
223+
expr = '"' + _bigramize(s) + '"'
224+
clauses_and.append(expr)
225+
clauses_or.append(expr)
226+
else:
227+
clauses_and.append(s + "*")
228+
clauses_or.append(s + "*")
229+
return " AND ".join(clauses_and), " OR ".join(clauses_or), raw_terms
230+
231+
232+
def _snippet(raw: str, terms: list[str], width: int = 90) -> str:
233+
low = raw.lower()
234+
best = -1
235+
for t in terms:
236+
i = low.find(t.lower())
237+
if i >= 0 and (best < 0 or i < best):
238+
best = i
239+
flat = re.sub(r"\s+", " ", raw)
240+
if best < 0:
241+
return flat[: width * 2] + ("…" if len(flat) > width * 2 else "")
242+
start = max(0, best - width)
243+
end = min(len(flat), best + width)
244+
# `best` indexes into raw; recompute on flat text approximately.
245+
i = flat.lower().find(terms[0].lower()) if terms else -1
246+
if i >= 0:
247+
start = max(0, i - width)
248+
end = min(len(flat), i + width)
249+
return ("…" if start > 0 else "") + flat[start:end] + ("…" if end < len(flat) else "")
250+
251+
252+
def search(kb_dir: Path, query: str, limit: int = 8) -> list[dict]:
253+
"""BM25-ranked keyword search. Returns [{kind, path, page, snippet, score}]."""
254+
db_path = ensure_index(kb_dir)
255+
and_expr, or_expr, terms = _fts_expr(query)
256+
if not and_expr:
257+
return []
258+
con = sqlite3.connect(str(db_path))
259+
try:
260+
rows = con.execute(
261+
"SELECT kind, path, page, raw, bm25(fts) FROM fts"
262+
" WHERE fts MATCH ? ORDER BY bm25(fts) LIMIT ?",
263+
(and_expr, limit),
264+
).fetchall()
265+
if not rows and or_expr and or_expr != and_expr:
266+
rows = con.execute(
267+
"SELECT kind, path, page, raw, bm25(fts) FROM fts"
268+
" WHERE fts MATCH ? ORDER BY bm25(fts) LIMIT ?",
269+
(or_expr, limit),
270+
).fetchall()
271+
finally:
272+
con.close()
273+
return [
274+
{
275+
"kind": k,
276+
"path": p,
277+
"page": pg,
278+
"snippet": _snippet(raw or "", terms),
279+
"score": round(score, 3),
280+
}
281+
for k, p, pg, raw, score in rows
282+
]
283+
284+
285+
def format_hits(hits: list[dict]) -> str:
286+
"""Render hits as the text returned by the agent's `search` tool."""
287+
if not hits:
288+
return (
289+
"No full-text hits. Fall back to navigation: read index.md, "
290+
"then summaries/concepts/entities."
291+
)
292+
lines = []
293+
for i, h in enumerate(hits, 1):
294+
loc = h["path"] + (f" (page {h['page']})" if h["page"] is not None else "")
295+
lines.append(f"[{i}] {h['kind']} | {loc}\n {h['snippet']}")
296+
lines.append(
297+
"\nNext: read_file() the .md hits; for 'page' hits call "
298+
"get_page_content(doc_name, pages) with doc_name = path minus "
299+
"'sources/' and '.json'."
300+
)
301+
return "\n".join(lines)

0 commit comments

Comments
 (0)