|
| 1 | +"""FTS5 hybrid retrieval for OpenKB: a fast BM25-ranked keyword pre-filter |
| 2 | +over the whole wiki, complementing the query agent's navigation loop. |
| 3 | +
|
| 4 | +The query agent's default strategy (read index.md -> summaries -> concepts -> |
| 5 | +sources) is thorough but costs several LLM turns before any content is seen. |
| 6 | +This module gives it a millisecond keyword index to jump straight to |
| 7 | +candidate pages; the agent still verifies by reading the actual pages. |
| 8 | +
|
| 9 | +Design notes: |
| 10 | +
|
| 11 | +- Storage: ``{kb_dir}/.openkb/fts.db`` (SQLite FTS5, ``bm25()`` ranking). |
| 12 | +- CJK: FTS5's unicode61 tokenizer treats a run of CJK ideographs as ONE |
| 13 | + token, so "川普政府" would only match as a whole. We transform CJK runs |
| 14 | + into overlapping bigrams ("川普 普政 政府") before indexing and apply the |
| 15 | + same transform to query terms (classic CJK bigram trick, no external |
| 16 | + segmenter dependency). |
| 17 | +- Long docs (``sources/*.json`` page arrays) are indexed one row per page, |
| 18 | + so a hit tells the agent exactly which ``pages=`` to pass to |
| 19 | + ``get_page_content``. |
| 20 | +- Freshness: incremental update keyed on (mtime, size) per file, checked on |
| 21 | + every search; a missing db triggers a full build. ``openkb fts rebuild`` |
| 22 | + forces a rebuild. |
| 23 | +""" |
| 24 | + |
| 25 | +from __future__ import annotations |
| 26 | + |
| 27 | +import json |
| 28 | +import re |
| 29 | +import sqlite3 |
| 30 | +import time |
| 31 | +from pathlib import Path |
| 32 | + |
| 33 | +# CJK Unified Ideographs (+ ext A, compat). Runs of these get bigram-ised. |
| 34 | +_CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff〇]+") |
| 35 | + |
| 36 | +_INDEX_DIRS = ("summaries", "concepts", "entities", "sources", "explorations") |
| 37 | +_KIND = { |
| 38 | + "summaries": "summary", |
| 39 | + "concepts": "concept", |
| 40 | + "entities": "entity", |
| 41 | + "sources": "source", |
| 42 | + "explorations": "exploration", |
| 43 | +} |
| 44 | +_FRONTMATTER_RE = re.compile(r"\A---\n.*?\n---\n", re.DOTALL) |
| 45 | +_RAW_CAP = 30000 # chars of original text kept per row for snippets |
| 46 | + |
| 47 | + |
| 48 | +def fts_db_path(kb_dir: Path) -> Path: |
| 49 | + return kb_dir / ".openkb" / "fts.db" |
| 50 | + |
| 51 | + |
| 52 | +def _bigramize(text: str) -> str: |
| 53 | + """Replace each CJK run with space-joined overlapping bigrams.""" |
| 54 | + |
| 55 | + def repl(m: re.Match) -> str: |
| 56 | + s = m.group(0) |
| 57 | + if len(s) == 1: |
| 58 | + return s |
| 59 | + return " ".join(s[i : i + 2] for i in range(len(s) - 1)) |
| 60 | + |
| 61 | + return _CJK_RE.sub(repl, text) |
| 62 | + |
| 63 | + |
| 64 | +def _connect(db_path: Path) -> sqlite3.Connection: |
| 65 | + con = sqlite3.connect(str(db_path)) |
| 66 | + con.execute( |
| 67 | + "CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(" |
| 68 | + "content, raw UNINDEXED, kind UNINDEXED, path UNINDEXED, page UNINDEXED," |
| 69 | + " tokenize='unicode61')" |
| 70 | + ) |
| 71 | + con.execute( |
| 72 | + "CREATE TABLE IF NOT EXISTS files(" |
| 73 | + "path TEXT PRIMARY KEY, mtime REAL, size INTEGER)" |
| 74 | + ) |
| 75 | + return con |
| 76 | + |
| 77 | + |
| 78 | +def _iter_targets(wiki_root: Path): |
| 79 | + """Yield (rel_path, kind, page, text) for every indexable unit.""" |
| 80 | + for sub in _INDEX_DIRS: |
| 81 | + d = wiki_root / sub |
| 82 | + if not d.is_dir(): |
| 83 | + continue |
| 84 | + for f in sorted(d.glob("*.md")): |
| 85 | + try: |
| 86 | + text = f.read_text(encoding="utf-8", errors="replace") |
| 87 | + except OSError: |
| 88 | + continue |
| 89 | + # Drop YAML frontmatter: wikilink lists there drown real mentions |
| 90 | + # in snippets and pollute ranking. |
| 91 | + text = _FRONTMATTER_RE.sub("", text) |
| 92 | + yield (f"{sub}/{f.name}", _KIND[sub], None, text) |
| 93 | + src = wiki_root / "sources" |
| 94 | + if src.is_dir(): |
| 95 | + for j in sorted(src.glob("*.json")): |
| 96 | + try: |
| 97 | + data = json.loads(j.read_text(encoding="utf-8", errors="replace")) |
| 98 | + except (OSError, ValueError): |
| 99 | + continue |
| 100 | + if not isinstance(data, list): |
| 101 | + continue |
| 102 | + for entry in data: |
| 103 | + if isinstance(entry, dict) and "page" in entry: |
| 104 | + yield ( |
| 105 | + f"sources/{j.name}", |
| 106 | + "page", |
| 107 | + entry["page"], |
| 108 | + str(entry.get("content", "")), |
| 109 | + ) |
| 110 | + |
| 111 | + |
| 112 | +def _row_for(rel: str, kind: str, page, text: str): |
| 113 | + raw = text[:_RAW_CAP] |
| 114 | + return (_bigramize(raw), raw, kind, rel, page) |
| 115 | + |
| 116 | + |
| 117 | +def _sync(con: sqlite3.Connection, wiki_root: Path) -> tuple[int, int]: |
| 118 | + """Incremental update: returns (files_seen, rows_written).""" |
| 119 | + cur = con.execute("SELECT path, mtime, size FROM files") |
| 120 | + known = {p: (m, s) for p, m, s in cur.fetchall()} |
| 121 | + seen: dict[str, list] = {} |
| 122 | + files_seen = 0 |
| 123 | + rows_written = 0 |
| 124 | + |
| 125 | + # Group rows by file so each file is replaced atomically-ish. |
| 126 | + for rel, kind, page, text in _iter_targets(wiki_root): |
| 127 | + seen.setdefault(rel, []).append((rel, kind, page, text)) |
| 128 | + |
| 129 | + wiki_files = {rel: (wiki_root / rel) for rel in seen} |
| 130 | + stale = [p for p in known if p not in seen] |
| 131 | + |
| 132 | + for rel, rows in seen.items(): |
| 133 | + f = wiki_files[rel] |
| 134 | + try: |
| 135 | + st = f.stat() |
| 136 | + except OSError: |
| 137 | + continue |
| 138 | + files_seen += 1 |
| 139 | + if known.get(rel) == (st.st_mtime, st.st_size): |
| 140 | + continue |
| 141 | + con.execute("DELETE FROM fts WHERE path = ?", (rel,)) |
| 142 | + con.executemany( |
| 143 | + "INSERT INTO fts (content, raw, kind, path, page) VALUES (?,?,?,?,?)", |
| 144 | + [_row_for(rel, kind, page, text) for rel, kind, page, text in rows], |
| 145 | + ) |
| 146 | + con.execute( |
| 147 | + "INSERT OR REPLACE INTO files (path, mtime, size) VALUES (?,?,?)", |
| 148 | + (rel, st.st_mtime, st.st_size), |
| 149 | + ) |
| 150 | + rows_written += len(rows) |
| 151 | + |
| 152 | + for rel in stale: |
| 153 | + con.execute("DELETE FROM fts WHERE path = ?", (rel,)) |
| 154 | + con.execute("DELETE FROM files WHERE path = ?", (rel,)) |
| 155 | + |
| 156 | + con.commit() |
| 157 | + return files_seen, rows_written |
| 158 | + |
| 159 | + |
| 160 | +def ensure_index(kb_dir: Path) -> Path: |
| 161 | + """Build the index if missing, else incrementally refresh it.""" |
| 162 | + wiki_root = kb_dir / "wiki" |
| 163 | + db_path = fts_db_path(kb_dir) |
| 164 | + db_path.parent.mkdir(parents=True, exist_ok=True) |
| 165 | + con = _connect(db_path) |
| 166 | + try: |
| 167 | + _sync(con, wiki_root) |
| 168 | + finally: |
| 169 | + con.close() |
| 170 | + return db_path |
| 171 | + |
| 172 | + |
| 173 | +def rebuild_index(kb_dir: Path) -> tuple[int, int]: |
| 174 | + """Drop and rebuild the index. Returns (files_seen, rows_written).""" |
| 175 | + wiki_root = kb_dir / "wiki" |
| 176 | + db_path = fts_db_path(kb_dir) |
| 177 | + db_path.parent.mkdir(parents=True, exist_ok=True) |
| 178 | + con = _connect(db_path) |
| 179 | + try: |
| 180 | + con.execute("DROP TABLE IF EXISTS fts") |
| 181 | + con.execute("DROP TABLE IF EXISTS files") |
| 182 | + con.commit() |
| 183 | + con.close() |
| 184 | + con = _connect(db_path) |
| 185 | + stats = _sync(con, wiki_root) |
| 186 | + finally: |
| 187 | + con.close() |
| 188 | + return stats |
| 189 | + |
| 190 | + |
| 191 | +def _fts_expr(query: str) -> tuple[str, str, list[str]]: |
| 192 | + """Build (AND_expr, OR_expr, raw_terms) from a free-text query.""" |
| 193 | + clauses_and: list[str] = [] |
| 194 | + clauses_or: list[str] = [] |
| 195 | + raw_terms: list[str] = [] |
| 196 | + for tok in query.split(): |
| 197 | + # Drop anything that is FTS5 syntax (quotes, parens, stars, colons...) |
| 198 | + # so pasted punctuation can never break the MATCH expression. |
| 199 | + tok = re.sub(r"[^\w一-鿿豈-]+", "", tok) |
| 200 | + if not tok: |
| 201 | + continue |
| 202 | + raw_terms.append(tok) |
| 203 | + subs: list[str] = [] |
| 204 | + pos = 0 |
| 205 | + for m in _CJK_RE.finditer(tok): |
| 206 | + if m.start() > pos: |
| 207 | + subs.append(("latin", tok[pos : m.start()])) |
| 208 | + subs.append(("cjk", m.group(0))) |
| 209 | + pos = m.end() |
| 210 | + if pos < len(tok): |
| 211 | + subs.append(("latin", tok[pos:])) |
| 212 | + |
| 213 | + if len(subs) > 1: |
| 214 | + # Mixed token (e.g. "ETA公司"): adjacency phrase, no prefix star. |
| 215 | + parts = [] |
| 216 | + for kind, s in subs: |
| 217 | + parts.append(_bigramize(s) if kind == "cjk" else s) |
| 218 | + clauses_and.append('"' + " ".join(parts) + '"') |
| 219 | + clauses_or.append('"' + " ".join(parts) + '"') |
| 220 | + elif subs: |
| 221 | + kind, s = subs[0] |
| 222 | + if kind == "cjk": |
| 223 | + expr = '"' + _bigramize(s) + '"' |
| 224 | + clauses_and.append(expr) |
| 225 | + clauses_or.append(expr) |
| 226 | + else: |
| 227 | + clauses_and.append(s + "*") |
| 228 | + clauses_or.append(s + "*") |
| 229 | + return " AND ".join(clauses_and), " OR ".join(clauses_or), raw_terms |
| 230 | + |
| 231 | + |
| 232 | +def _snippet(raw: str, terms: list[str], width: int = 90) -> str: |
| 233 | + low = raw.lower() |
| 234 | + best = -1 |
| 235 | + for t in terms: |
| 236 | + i = low.find(t.lower()) |
| 237 | + if i >= 0 and (best < 0 or i < best): |
| 238 | + best = i |
| 239 | + flat = re.sub(r"\s+", " ", raw) |
| 240 | + if best < 0: |
| 241 | + return flat[: width * 2] + ("…" if len(flat) > width * 2 else "") |
| 242 | + start = max(0, best - width) |
| 243 | + end = min(len(flat), best + width) |
| 244 | + # `best` indexes into raw; recompute on flat text approximately. |
| 245 | + i = flat.lower().find(terms[0].lower()) if terms else -1 |
| 246 | + if i >= 0: |
| 247 | + start = max(0, i - width) |
| 248 | + end = min(len(flat), i + width) |
| 249 | + return ("…" if start > 0 else "") + flat[start:end] + ("…" if end < len(flat) else "") |
| 250 | + |
| 251 | + |
| 252 | +def search(kb_dir: Path, query: str, limit: int = 8) -> list[dict]: |
| 253 | + """BM25-ranked keyword search. Returns [{kind, path, page, snippet, score}].""" |
| 254 | + db_path = ensure_index(kb_dir) |
| 255 | + and_expr, or_expr, terms = _fts_expr(query) |
| 256 | + if not and_expr: |
| 257 | + return [] |
| 258 | + con = sqlite3.connect(str(db_path)) |
| 259 | + try: |
| 260 | + rows = con.execute( |
| 261 | + "SELECT kind, path, page, raw, bm25(fts) FROM fts" |
| 262 | + " WHERE fts MATCH ? ORDER BY bm25(fts) LIMIT ?", |
| 263 | + (and_expr, limit), |
| 264 | + ).fetchall() |
| 265 | + if not rows and or_expr and or_expr != and_expr: |
| 266 | + rows = con.execute( |
| 267 | + "SELECT kind, path, page, raw, bm25(fts) FROM fts" |
| 268 | + " WHERE fts MATCH ? ORDER BY bm25(fts) LIMIT ?", |
| 269 | + (or_expr, limit), |
| 270 | + ).fetchall() |
| 271 | + finally: |
| 272 | + con.close() |
| 273 | + return [ |
| 274 | + { |
| 275 | + "kind": k, |
| 276 | + "path": p, |
| 277 | + "page": pg, |
| 278 | + "snippet": _snippet(raw or "", terms), |
| 279 | + "score": round(score, 3), |
| 280 | + } |
| 281 | + for k, p, pg, raw, score in rows |
| 282 | + ] |
| 283 | + |
| 284 | + |
| 285 | +def format_hits(hits: list[dict]) -> str: |
| 286 | + """Render hits as the text returned by the agent's `search` tool.""" |
| 287 | + if not hits: |
| 288 | + return ( |
| 289 | + "No full-text hits. Fall back to navigation: read index.md, " |
| 290 | + "then summaries/concepts/entities." |
| 291 | + ) |
| 292 | + lines = [] |
| 293 | + for i, h in enumerate(hits, 1): |
| 294 | + loc = h["path"] + (f" (page {h['page']})" if h["page"] is not None else "") |
| 295 | + lines.append(f"[{i}] {h['kind']} | {loc}\n {h['snippet']}") |
| 296 | + lines.append( |
| 297 | + "\nNext: read_file() the .md hits; for 'page' hits call " |
| 298 | + "get_page_content(doc_name, pages) with doc_name = path minus " |
| 299 | + "'sources/' and '.json'." |
| 300 | + ) |
| 301 | + return "\n".join(lines) |
0 commit comments