diff --git a/src/pkb_x/cli.py b/src/pkb_x/cli.py index 0756845..8a4d563 100644 --- a/src/pkb_x/cli.py +++ b/src/pkb_x/cli.py @@ -1,11 +1,16 @@ from __future__ import annotations import argparse +import datetime as dt +import json import sys +from dataclasses import asdict from .config import load_settings from .extractor import Extractor +from .index import reindex from .oauth import authenticate +from .search import browse, search from .storage import ensure_dirs @@ -30,14 +35,67 @@ def cmd_extract(args: argparse.Namespace) -> int: ) finally: extractor.close() + index_stats = reindex(settings) print("Extraction complete") for key, value in stats.items(): print(f"{key}: {value}") print(f"Raw data: {settings.raw_dir}") print(f"Markdown: {settings.markdown_dir}") + print(f"Search index: {settings.search_index_path}") + print(f"indexed: {index_stats.indexed}") + print(f"index_skipped: {index_stats.skipped}") + print(f"index_deleted: {index_stats.deleted}") return 0 if stats["errors"] == 0 else 1 +def cmd_index(args: argparse.Namespace) -> int: + settings = load_settings() + stats = reindex(settings, full=args.full) + print("Index complete") + print(f"Search index: {settings.search_index_path}") + print(f"scanned: {stats.scanned}") + print(f"indexed: {stats.indexed}") + print(f"skipped: {stats.skipped}") + print(f"deleted: {stats.deleted}") + return 0 + + +def cmd_search(args: argparse.Namespace) -> int: + settings = load_settings() + hits = search( + settings, + args.query, + kind=args.kind, + author=args.author, + since=args.since, + until=args.until, + limit=args.limit, + ) + if args.json: + print(json.dumps([asdict(hit) for hit in hits], ensure_ascii=False, indent=2)) + else: + _print_search_hits(settings, hits) + return 0 + + +def cmd_browse(args: argparse.Namespace) -> int: + settings = load_settings() + hits = browse( + settings, + kind=args.kind, + author=args.author, + since=args.since, + until=args.until, + random=args.random, + limit=args.limit, + ) + if args.json: + print(json.dumps([asdict(hit) for hit in hits], ensure_ascii=False, indent=2)) + else: + _print_browse_hits(settings, hits) + return 0 + + def cmd_init(_: argparse.Namespace) -> int: settings = load_settings() ensure_dirs(settings.data_dir) @@ -73,6 +131,22 @@ def build_parser() -> argparse.ArgumentParser: ) extract_parser.set_defaults(func=cmd_extract) + index_parser = subparsers.add_parser("index", help="Build or refresh the local Markdown search index.") + index_parser.add_argument("--full", action="store_true", help="Re-index all Markdown files even if body hashes match.") + index_parser.set_defaults(func=cmd_index) + + search_parser = subparsers.add_parser("search", help="Search indexed Markdown with FTS5 query syntax.") + search_parser.add_argument("query", help="FTS5 MATCH query.") + _add_discovery_filters(search_parser) + search_parser.add_argument("--json", action="store_true", help="Write results as JSON.") + search_parser.set_defaults(func=cmd_search) + + browse_parser = subparsers.add_parser("browse", help="Browse indexed Markdown without a search query.") + _add_discovery_filters(browse_parser) + browse_parser.add_argument("--random", action="store_true", help="Return a random result order.") + browse_parser.add_argument("--json", action="store_true", help="Write results as JSON.") + browse_parser.set_defaults(func=cmd_browse) + return parser @@ -89,5 +163,61 @@ def main(argv: list[str] | None = None) -> int: return 1 +def _add_discovery_filters(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--kind", choices=("bookmark", "linked-page"), default=None, help="Limit results by document kind.") + parser.add_argument("--author", default=None, help="Limit results by author handle or linked-page host.") + parser.add_argument("--since", type=_iso_date, default=None, help="Limit results to documents on or after YYYY-MM-DD.") + parser.add_argument("--until", type=_iso_date, default=None, help="Limit results to documents before YYYY-MM-DD.") + parser.add_argument("--limit", type=_positive_int, default=20, help="Maximum result count.") + + +def _iso_date(value: str) -> str: + try: + dt.date.fromisoformat(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("expected YYYY-MM-DD") from exc + return value + + +def _positive_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("expected an integer") from exc + if parsed < 1: + raise argparse.ArgumentTypeError("expected a positive integer") + return parsed + + +def _print_search_hits(settings, hits) -> None: + for hit in hits: + print(f"{settings.markdown_dir / hit.path}:1") + meta = _metadata(hit.author, hit.created_at, f"score {hit.score:.4f}") + if meta: + print(f" {meta}") + if hit.title: + print(f" {hit.title}") + if hit.source_url: + print(f" {hit.source_url}") + if hit.snippet: + print(f" {hit.snippet}") + + +def _print_browse_hits(settings, hits) -> None: + for hit in hits: + print(f"{settings.markdown_dir / hit.path}:1") + meta = _metadata(hit.author, hit.created_at, hit.kind) + if meta: + print(f" {meta}") + if hit.title: + print(f" {hit.title}") + if hit.source_url: + print(f" {hit.source_url}") + + +def _metadata(*parts: str | None) -> str: + return " | ".join(part for part in parts if part) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/pkb_x/config.py b/src/pkb_x/config.py index 6df35da..71ef549 100644 --- a/src/pkb_x/config.py +++ b/src/pkb_x/config.py @@ -47,6 +47,10 @@ def markdown_dir(self) -> Path: def state_path(self) -> Path: return self.data_dir / ".state" / "extractor.sqlite" + @property + def search_index_path(self) -> Path: + return self.data_dir / ".state" / "search.sqlite" + def load_settings() -> Settings: load_dotenv() diff --git a/src/pkb_x/index.py b/src/pkb_x/index.py new file mode 100644 index 0000000..34a9c47 --- /dev/null +++ b/src/pkb_x/index.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .config import Settings +from .storage import utc_now + + +KIND_BOOKMARK = "bookmark" +KIND_LINKED_PAGE = "linked-page" +INDEXED_KINDS = (KIND_BOOKMARK, KIND_LINKED_PAGE) + + +@dataclass +class IndexStats: + scanned: int = 0 + indexed: int = 0 + skipped: int = 0 + deleted: int = 0 + + +@dataclass(frozen=True) +class ParsedMarkdown: + frontmatter: dict[str, Any] + body: str + + +@dataclass(frozen=True) +class MarkdownDocument: + kind: str + path: str + source_url: str | None + title: str | None + author: str | None + created_at: str | None + conversation_id: str | None + links_json: str | None + body: str + content_hash: str + + +def connect(settings: Settings) -> sqlite3.Connection: + settings.search_index_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(settings.search_index_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + return conn + + +def initialize(conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + source_url TEXT, + title TEXT, + author TEXT, + created_at TEXT, + conversation_id TEXT, + links_json TEXT, + indexed_at TEXT NOT NULL, + content_hash TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '' + ) + """ + ) + _ensure_documents_body_column(conn) + conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_kind ON documents(kind)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_author ON documents(author)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_created_at ON documents(created_at)") + conn.execute( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( + title, + body, + content='documents', + content_rowid='id', + tokenize='porter unicode61 remove_diacritics 2' + ) + """ + ) + conn.commit() + + +def reindex(settings: Settings, *, full: bool = False) -> IndexStats: + stats = IndexStats() + markdown_dir = settings.markdown_dir + conn = connect(settings) + try: + initialize(conn) + documents = list(_iter_markdown_documents(markdown_dir)) + seen_paths = {document.path for document in documents} + now = utc_now() + with conn: + for document in documents: + stats.scanned += 1 + existing = conn.execute( + """ + SELECT + id, + kind, + path, + source_url, + title, + author, + created_at, + conversation_id, + links_json, + content_hash, + body + FROM documents + WHERE path = ? + """, + (document.path,), + ).fetchone() + if existing and _document_matches_row(document, existing) and not full: + stats.skipped += 1 + continue + row_id = _upsert_document(conn, document, indexed_at=now, existing_id=existing["id"] if existing else None) + _replace_fts_row(conn, row_id, document.title, document.body) + stats.indexed += 1 + deleted_rows = conn.execute("SELECT id, path FROM documents").fetchall() + for row in deleted_rows: + if str(row["path"]) in seen_paths: + continue + conn.execute("DELETE FROM documents_fts WHERE rowid = ?", (row["id"],)) + conn.execute("DELETE FROM documents WHERE id = ?", (row["id"],)) + stats.deleted += 1 + return stats + finally: + conn.close() + + +def parse_markdown(text: str) -> ParsedMarkdown: + if not text.startswith("---\n"): + return ParsedMarkdown({}, text) + lines = text.splitlines(keepends=True) + end_index: int | None = None + for index, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + end_index = index + break + if end_index is None: + return ParsedMarkdown({}, text) + frontmatter_lines = [line.rstrip("\n") for line in lines[1:end_index]] + body = "".join(lines[end_index + 1 :]).lstrip("\n") + return ParsedMarkdown(_parse_frontmatter_lines(frontmatter_lines), body) + + +def _ensure_documents_body_column(conn: sqlite3.Connection) -> None: + columns = set() + for row in conn.execute("PRAGMA table_info(documents)").fetchall(): + columns.add(str(row["name"] if isinstance(row, sqlite3.Row) else row[1])) + if "body" not in columns: + conn.execute("ALTER TABLE documents ADD COLUMN body TEXT NOT NULL DEFAULT ''") + + +def _iter_markdown_documents(markdown_dir: Path) -> list[MarkdownDocument]: + documents: list[MarkdownDocument] = [] + for kind, relative_root in ( + (KIND_BOOKMARK, Path("bookmarks")), + (KIND_LINKED_PAGE, Path("linked-pages")), + ): + root = markdown_dir / relative_root + if not root.exists(): + continue + for path in sorted(root.rglob("*.md")): + documents.append(_load_markdown_document(markdown_dir, path, kind)) + return documents + + +def _load_markdown_document(markdown_dir: Path, path: Path, kind: str) -> MarkdownDocument: + parsed = parse_markdown(path.read_text(encoding="utf-8")) + frontmatter = parsed.frontmatter + relative_path = path.relative_to(markdown_dir).as_posix() + body = parsed.body + content_hash = hashlib.sha256(body.encode("utf-8")).hexdigest() + source_url = _optional_str(frontmatter.get("url")) + if kind == KIND_LINKED_PAGE: + author = _linked_page_author(frontmatter) + else: + author = _optional_str(frontmatter.get("author")) + links = frontmatter.get("links") + links_json = json.dumps(links, ensure_ascii=False, sort_keys=True) if isinstance(links, list) else None + return MarkdownDocument( + kind=kind, + path=relative_path, + source_url=source_url, + title=_optional_str(frontmatter.get("title")) or _first_heading(body), + author=author, + created_at=_optional_str(frontmatter.get("created_at")), + conversation_id=_optional_str(frontmatter.get("conversation_id")), + links_json=links_json, + body=body, + content_hash=content_hash, + ) + + +def _upsert_document( + conn: sqlite3.Connection, + document: MarkdownDocument, + *, + indexed_at: str, + existing_id: int | None, +) -> int: + values = ( + document.kind, + document.path, + document.source_url, + document.title, + document.author, + document.created_at, + document.conversation_id, + document.links_json, + indexed_at, + document.content_hash, + document.body, + ) + if existing_id is not None: + conn.execute("DELETE FROM documents_fts WHERE rowid = ?", (existing_id,)) + conn.execute( + """ + UPDATE documents + SET kind = ?, + path = ?, + source_url = ?, + title = ?, + author = ?, + created_at = ?, + conversation_id = ?, + links_json = ?, + indexed_at = ?, + content_hash = ?, + body = ? + WHERE id = ? + """, + (*values, existing_id), + ) + return int(existing_id) + cursor = conn.execute( + """ + INSERT INTO documents ( + kind, + path, + source_url, + title, + author, + created_at, + conversation_id, + links_json, + indexed_at, + content_hash, + body + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + return int(cursor.lastrowid) + + +def _document_matches_row(document: MarkdownDocument, row: sqlite3.Row) -> bool: + return ( + row["kind"] == document.kind + and row["path"] == document.path + and row["source_url"] == document.source_url + and row["title"] == document.title + and row["author"] == document.author + and row["created_at"] == document.created_at + and row["conversation_id"] == document.conversation_id + and row["links_json"] == document.links_json + and row["content_hash"] == document.content_hash + and row["body"] == document.body + ) + + +def _replace_fts_row(conn: sqlite3.Connection, row_id: int, title: str | None, body: str) -> None: + conn.execute( + "INSERT INTO documents_fts(rowid, title, body) VALUES (?, ?, ?)", + (row_id, title or "", body), + ) + + +def _parse_frontmatter_lines(lines: list[str]) -> dict[str, Any]: + values: dict[str, Any] = {} + current_list_key: str | None = None + for line in lines: + if not line.strip(): + continue + stripped = line.strip() + if current_list_key and stripped.startswith("- "): + current = values.setdefault(current_list_key, []) + if isinstance(current, list): + current.append(_parse_scalar(stripped[2:].strip())) + continue + current_list_key = None + if ":" not in line: + continue + key, raw_value = line.split(":", 1) + key = key.strip() + if not key: + continue + raw_value = raw_value.strip() + if raw_value: + values[key] = _parse_scalar(raw_value) + else: + values[key] = [] + current_list_key = key + return values + + +def _parse_scalar(raw_value: str) -> Any: + if raw_value == "null": + return None + if len(raw_value) >= 2 and raw_value[0] == '"' and raw_value[-1] == '"': + try: + return json.loads(raw_value) + except json.JSONDecodeError: + return raw_value[1:-1] + if len(raw_value) >= 2 and raw_value[0] == "'" and raw_value[-1] == "'": + return raw_value[1:-1] + return raw_value + + +def _first_heading(body: str) -> str | None: + for line in body.splitlines(): + if line.startswith("# "): + title = line[2:].strip() + return title or None + return None + + +def _linked_page_author(frontmatter: dict[str, Any]) -> str | None: + url = _optional_str(frontmatter.get("final_url")) or _optional_str(frontmatter.get("url")) + if not url or "://" not in url: + return None + host = url.split("://", 1)[1].split("/", 1)[0].lower() + return host or None + + +def _optional_str(value: Any) -> str | None: + if value is None: + return None + text = str(value) + return text if text else None diff --git a/src/pkb_x/search.py b/src/pkb_x/search.py new file mode 100644 index 0000000..745ba4b --- /dev/null +++ b/src/pkb_x/search.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from typing import Literal + +from .config import Settings +from .index import connect, initialize + + +DocumentKind = Literal["bookmark", "linked-page"] + + +@dataclass(frozen=True) +class SearchHit: + path: str + kind: str + score: float + title: str | None + author: str | None + created_at: str | None + source_url: str | None + snippet: str + + +@dataclass(frozen=True) +class BrowseHit: + path: str + kind: str + title: str | None + author: str | None + created_at: str | None + source_url: str | None + + +def search( + settings: Settings, + query: str, + *, + kind: DocumentKind | None = None, + author: str | None = None, + since: str | None = None, + until: str | None = None, + limit: int = 20, +) -> list[SearchHit]: + if limit <= 0: + return [] + conn = connect(settings) + try: + initialize(conn) + rows = conn.execute( + """ + SELECT + d.path, + d.kind, + d.title, + d.author, + d.created_at, + d.source_url, + bm25(documents_fts, 5.0, 1.0) AS score, + snippet(documents_fts, 1, '**', '**', ' ... ', 16) AS snippet + FROM documents_fts + JOIN documents d ON d.id = documents_fts.rowid + WHERE documents_fts MATCH :query + AND (:kind IS NULL OR d.kind = :kind) + AND (:author IS NULL OR d.author = :author) + AND (:since IS NULL OR d.created_at >= :since) + AND (:until IS NULL OR d.created_at < :until) + ORDER BY score + LIMIT :limit + """, + { + "query": query, + "kind": kind, + "author": author, + "since": since, + "until": until, + "limit": limit, + }, + ).fetchall() + return [_search_hit(row) for row in rows] + finally: + conn.close() + + +def browse( + settings: Settings, + *, + kind: DocumentKind | None = None, + author: str | None = None, + since: str | None = None, + until: str | None = None, + random: bool = False, + limit: int = 20, +) -> list[BrowseHit]: + if limit <= 0: + return [] + conn = connect(settings) + try: + initialize(conn) + order_by = "RANDOM()" if random else "d.created_at IS NULL, d.created_at DESC, d.indexed_at DESC, d.path ASC" + rows = conn.execute( + f""" + SELECT + d.path, + d.kind, + d.title, + d.author, + d.created_at, + d.source_url + FROM documents d + WHERE (:kind IS NULL OR d.kind = :kind) + AND (:author IS NULL OR d.author = :author) + AND (:since IS NULL OR d.created_at >= :since) + AND (:until IS NULL OR d.created_at < :until) + ORDER BY {order_by} + LIMIT :limit + """, + { + "kind": kind, + "author": author, + "since": since, + "until": until, + "limit": limit, + }, + ).fetchall() + return [_browse_hit(row) for row in rows] + finally: + conn.close() + + +def _search_hit(row: sqlite3.Row) -> SearchHit: + return SearchHit( + path=str(row["path"]), + kind=str(row["kind"]), + score=float(row["score"]), + title=row["title"], + author=row["author"], + created_at=row["created_at"], + source_url=row["source_url"], + snippet=str(row["snippet"] or ""), + ) + + +def _browse_hit(row: sqlite3.Row) -> BrowseHit: + return BrowseHit( + path=str(row["path"]), + kind=str(row["kind"]), + title=row["title"], + author=row["author"], + created_at=row["created_at"], + source_url=row["source_url"], + ) diff --git a/tests/test_index_search.py b/tests/test_index_search.py new file mode 100644 index 0000000..ca37759 --- /dev/null +++ b/tests/test_index_search.py @@ -0,0 +1,348 @@ +import hashlib +import json +import sqlite3 + +from pkb_x import cli +from pkb_x.config import Settings +from pkb_x.index import IndexStats, initialize, parse_markdown, reindex +from pkb_x.search import browse, search + + +def test_initialize_creates_schema_and_migrates_body_column(tmp_path): + db_path = tmp_path / "search.sqlite" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + conn.execute( + """ + CREATE TABLE documents ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + source_url TEXT, + title TEXT, + author TEXT, + created_at TEXT, + conversation_id TEXT, + links_json TEXT, + indexed_at TEXT NOT NULL, + content_hash TEXT NOT NULL + ) + """ + ) + + initialize(conn) + + columns = {row["name"] for row in conn.execute("PRAGMA table_info(documents)").fetchall()} + assert "body" in columns + assert conn.execute("SELECT name FROM sqlite_master WHERE name = 'documents_fts'").fetchone() + assert conn.execute("SELECT name FROM sqlite_master WHERE name = 'idx_documents_kind'").fetchone() + finally: + conn.close() + + +def test_parse_markdown_frontmatter_all_fields_missing_and_malformed(): + parsed = parse_markdown( + """--- +id: "123" +url: "https://x.com/alice/status/123" +author: "@alice" +created_at: "2025-01-02T03:04:05.000Z" +conversation_id: "100" +thread_error: null +links: + - "https://example.com/a" + - "https://example.com/b" +--- + +# Body + +Only this body should be indexed. +""" + ) + + assert parsed.frontmatter["id"] == "123" + assert parsed.frontmatter["thread_error"] is None + assert parsed.frontmatter["links"] == ["https://example.com/a", "https://example.com/b"] + assert parsed.body.startswith("# Body") + + no_frontmatter = parse_markdown("# Plain\n\nText") + assert no_frontmatter.frontmatter == {} + assert no_frontmatter.body == "# Plain\n\nText" + + malformed = parse_markdown("---\nauthor: \"@alice\"\n# no closing marker\n") + assert malformed.frontmatter == {} + assert malformed.body.startswith("---") + + +def test_reindex_indexes_bookmark_and_linked_page_rows(tmp_path): + settings = _settings(tmp_path) + bookmark = _write_bookmark( + settings, + "2025-01-02", + "123", + author="@alice", + body="# X bookmark 123 by @alice\n\nLanding pages need searchable SEO tactics.\n", + ) + linked_page = _write_linked_page( + settings, + "example.com", + "article", + url="https://example.com/article", + title="Example Article", + body="# Example Article\n\nPython sqlite indexing notes.\n", + ) + + stats = reindex(settings) + + assert stats.scanned == 2 + assert stats.indexed == 2 + assert stats.skipped == 0 + conn = sqlite3.connect(settings.search_index_path) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute("SELECT * FROM documents ORDER BY kind").fetchall() + bookmark_row = next(row for row in rows if row["kind"] == "bookmark") + linked_row = next(row for row in rows if row["kind"] == "linked-page") + assert bookmark_row["path"] == "bookmarks/2025-01-02/123.md" + assert bookmark_row["source_url"] == "https://x.com/alice/status/123" + assert bookmark_row["author"] == "@alice" + assert bookmark_row["conversation_id"] == "100" + assert json.loads(bookmark_row["links_json"]) == ["https://example.com/article"] + assert bookmark_row["content_hash"] == hashlib.sha256(parse_markdown(bookmark.read_text()).body.encode()).hexdigest() + assert linked_row["path"] == "linked-pages/example.com/article.md" + assert linked_row["source_url"] == "https://example.com/article" + assert linked_row["title"] == "Example Article" + assert linked_row["author"] == "example.com" + assert conn.execute("SELECT rowid FROM documents_fts WHERE documents_fts MATCH 'seo'").fetchone() + assert conn.execute("SELECT rowid FROM documents_fts WHERE documents_fts MATCH 'conversation_id'").fetchone() is None + assert conn.execute("SELECT rowid FROM documents_fts WHERE documents_fts MATCH 'sqlite'").fetchone() + finally: + conn.close() + assert linked_page.exists() + + +def test_reindex_skips_unchanged_full_reindexes_and_removes_deleted_files(tmp_path): + settings = _settings(tmp_path) + bookmark = _write_bookmark( + settings, + "2025-01-02", + "123", + author="@alice", + body="# X bookmark 123 by @alice\n\nIncremental search body.\n", + ) + + assert reindex(settings).indexed == 1 + skipped = reindex(settings) + assert skipped.scanned == 1 + assert skipped.indexed == 0 + assert skipped.skipped == 1 + + _write_bookmark( + settings, + "2025-01-02", + "123", + author="@bob", + body="# X bookmark 123 by @alice\n\nIncremental search body.\n", + ) + metadata_changed = reindex(settings) + assert metadata_changed.scanned == 1 + assert metadata_changed.indexed == 1 + assert metadata_changed.skipped == 0 + assert [hit.path for hit in browse(settings, author="@bob")] == ["bookmarks/2025-01-02/123.md"] + assert browse(settings, author="@alice") == [] + + full = reindex(settings, full=True) + assert full.indexed == 1 + assert full.skipped == 0 + + bookmark.unlink() + deleted = reindex(settings) + assert deleted.scanned == 0 + assert deleted.deleted == 1 + assert search(settings, "incremental") == [] + + +def test_search_returns_snippets_and_applies_filters(tmp_path): + settings = _settings(tmp_path) + _write_bookmark( + settings, + "2025-01-02", + "123", + author="@alice", + body="# X bookmark 123 by @alice\n\nGrowth loops need durable landing page copy.\n", + ) + _write_bookmark( + settings, + "2025-02-02", + "456", + author="@bob", + body="# X bookmark 456 by @bob\n\nGrowth loops also need onboarding analytics.\n", + ) + _write_linked_page( + settings, + "example.com", + "python", + url="https://example.com/python", + title="Python Notes", + body="# Python Notes\n\nSqlite search examples for linked pages.\n", + ) + reindex(settings) + + hits = search(settings, "growth") + assert {hit.path for hit in hits} == {"bookmarks/2025-01-02/123.md", "bookmarks/2025-02-02/456.md"} + assert all(isinstance(hit.score, float) for hit in hits) + assert "**Growth**" in hits[0].snippet + + assert [hit.author for hit in search(settings, "growth", author="@alice")] == ["@alice"] + assert [hit.path for hit in search(settings, "growth", since="2025-02-01")] == ["bookmarks/2025-02-02/456.md"] + assert [hit.path for hit in search(settings, "growth", until="2025-02-01")] == ["bookmarks/2025-01-02/123.md"] + assert [hit.kind for hit in search(settings, "sqlite", kind="linked-page")] == ["linked-page"] + assert search(settings, "sqlite", kind="bookmark") == [] + + +def test_cli_search_json_output_has_stable_keys(monkeypatch, capsys, tmp_path): + settings = _settings(tmp_path) + _write_bookmark( + settings, + "2025-01-02", + "123", + author="@alice", + body="# X bookmark 123 by @alice\n\nJson output should include growth.\n", + ) + reindex(settings) + monkeypatch.setenv("PKB_DATA_DIR", str(tmp_path)) + + exit_code = cli.main(["search", "growth", "--json"]) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert list(payload[0]) == ["path", "kind", "score", "title", "author", "created_at", "source_url", "snippet"] + assert payload[0]["path"] == "bookmarks/2025-01-02/123.md" + assert payload[0]["author"] == "@alice" + + +def test_browse_orders_by_newest_and_random_returns_limited_results(tmp_path): + settings = _settings(tmp_path) + _write_bookmark(settings, "2025-01-02", "123", author="@alice", body="# Older\n\nDiscovery one.\n") + _write_bookmark(settings, "2025-03-02", "456", author="@bob", body="# Newer\n\nDiscovery two.\n") + _write_bookmark(settings, "2025-02-02", "789", author="@alice", body="# Middle\n\nDiscovery three.\n") + reindex(settings) + + ordered = browse(settings, limit=3) + assert [hit.path for hit in ordered] == [ + "bookmarks/2025-03-02/456.md", + "bookmarks/2025-02-02/789.md", + "bookmarks/2025-01-02/123.md", + ] + assert [hit.path for hit in browse(settings, author="@alice", limit=5)] == [ + "bookmarks/2025-02-02/789.md", + "bookmarks/2025-01-02/123.md", + ] + random_hits = browse(settings, random=True, limit=2) + assert len(random_hits) == 2 + assert {hit.path for hit in random_hits} <= {hit.path for hit in ordered} + + +def test_cli_browse_json_output(monkeypatch, capsys, tmp_path): + settings = _settings(tmp_path) + _write_bookmark(settings, "2025-01-02", "123", author="@alice", body="# Browse\n\nBrowse json body.\n") + reindex(settings) + monkeypatch.setenv("PKB_DATA_DIR", str(tmp_path)) + + exit_code = cli.main(["browse", "--json"]) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert list(payload[0]) == ["path", "kind", "title", "author", "created_at", "source_url"] + assert payload[0]["kind"] == "bookmark" + + +def test_extract_command_reindexes_after_extraction(monkeypatch, tmp_path): + settings = _settings(tmp_path) + calls = [] + + class FakeExtractor: + def __init__(self, loaded_settings): + assert loaded_settings == settings + + def run(self, **kwargs): + calls.append(("run", kwargs)) + return { + "bookmark_pages": 0, + "bookmarks": 0, + "skipped_bookmarks": 0, + "missing_bookmarks": 0, + "threads": 0, + "links": 0, + "skipped_links": 0, + "errors": 0, + } + + def close(self): + calls.append(("close", None)) + + monkeypatch.setattr(cli, "load_settings", lambda: settings) + monkeypatch.setattr(cli, "Extractor", FakeExtractor) + monkeypatch.setattr(cli, "reindex", lambda loaded_settings: _fake_index_stats(calls, loaded_settings, settings)) + + exit_code = cli.cmd_extract( + cli.argparse.Namespace(max_pages=None, no_links=False, refresh=False, refresh_links=False) + ) + + assert exit_code == 0 + assert calls == [ + ("run", {"max_pages": None, "fetch_links": True, "refresh": False, "refresh_links": False}), + ("close", None), + ("reindex", False), + ] + + +def _settings(tmp_path): + return Settings(client_id="", client_secret=None, redirect_uri="", data_dir=tmp_path) + + +def _write_bookmark(settings, date, post_id, *, author, body): + path = settings.markdown_dir / "bookmarks" / date / f"{post_id}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f"""--- +id: "{post_id}" +url: "https://x.com/{author.lstrip('@')}/status/{post_id}" +author: "{author}" +created_at: "{date}T03:04:05.000Z" +conversation_id: "100" +links: + - "https://example.com/article" +--- + +{body}""", + encoding="utf-8", + ) + return path + + +def _write_linked_page(settings, host, slug, *, url, title, body): + path = settings.markdown_dir / "linked-pages" / host / f"{slug}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f"""--- +url: "{url}" +final_url: "{url}" +canonical_url: null +title: "{title}" +content_type: "text/html" +status_code: "200" +error: null +--- + +{body}""", + encoding="utf-8", + ) + return path + + +def _fake_index_stats(calls, loaded_settings, expected_settings): + assert loaded_settings == expected_settings + calls.append(("reindex", False)) + return IndexStats()