diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6e3ba60c..75df7277 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -66,3 +66,8 @@ fixes). Newest entries at the bottom. already-working Windows `cp314` installs. Completing the wheel matrix is the correct fix; the next 0.3.8 patch release will carry full `cp314` coverage once CI confirms the new platforms build cleanly. + +## 2026-09-06: Restore CLI parsing after duplicate index command registration + +- Remove the seven duplicate index command parsers and handlers introduced by #269 after #285 had already supplied them. This fixes `argparse.ArgumentError: conflicting subparser: index-browser`, which prevented every CLI command, including help, from running. +- Retain the #285 command interface (`--max-count`, `--index-name`, `--no-recompute`), application readers, and shared builder; native indexing and standalone wheel reader packaging are unchanged. diff --git a/docs/user-scripts-tr.md b/docs/user-scripts-tr.md index 070ae852..6002face 100644 --- a/docs/user-scripts-tr.md +++ b/docs/user-scripts-tr.md @@ -179,7 +179,7 @@ ollama list C: Daha küçük bir dataset ile test edin: ```bash ---max-items 1000 +--max-count 1000 ``` ## İleri Düzey Kullanım diff --git a/docs/user-scripts.md b/docs/user-scripts.md index 447ac94e..22f2d369 100644 --- a/docs/user-scripts.md +++ b/docs/user-scripts.md @@ -177,7 +177,7 @@ ollama list Try with a smaller dataset: ```bash ---max-items 1000 +--max-count 1000 ``` ## Advanced Usage diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py index 61b2a363..34b9f923 100644 --- a/packages/leann-core/src/leann/cli.py +++ b/packages/leann-core/src/leann/cli.py @@ -486,79 +486,6 @@ def add_embedding_args(target_parser: argparse.ArgumentParser) -> None: ), ) - # Browser Index Command - browser_parser = subparsers.add_parser("index-browser", help="Index browser history") - browser_parser.add_argument( - "browser_type", choices=["chrome", "brave"], help="Type of browser" - ) - browser_parser.add_argument( - "--profile", type=str, default="Default", help="Profile name (default: Default)" - ) - browser_parser.add_argument( - "--index-name", type=str, default=None, help="Custom index name" - ) - browser_parser.add_argument( - "--max-items", type=int, default=1000, help="Max history items to index" - ) - add_embedding_args(browser_parser) - - # Email indexing command - email_parser = subparsers.add_parser("index-email", help="Index Apple Mail emails") - email_parser.add_argument( - "index_name", nargs="?", default="apple-mail", help="Index name (default: apple-mail)" - ) - email_parser.add_argument( - "--max-items", type=int, default=2000, help="Max emails to index (default: 2000)" - ) - add_embedding_args(email_parser) - - # Calendar indexing command - calendar_parser = subparsers.add_parser( - "index-calendar", help="Index Apple Calendar events" - ) - calendar_parser.add_argument( - "index_name", - nargs="?", - default="apple-calendar", - help="Index name (default: apple-calendar)", - ) - calendar_parser.add_argument( - "--max-items", type=int, default=1000, help="Max events to index (default: 1000)" - ) - add_embedding_args(calendar_parser) - - # WeChat indexing command - wechat_parser = subparsers.add_parser("index-wechat", help="Index WeChat chat history") - wechat_parser.add_argument( - "index_name", nargs="?", default="wechat", help="Index name (default: wechat)" - ) - wechat_parser.add_argument( - "--export-dir", - type=str, - default="./wechat_export", - help="Directory containing exported WeChat data (default: ./wechat_export)", - ) - wechat_parser.add_argument( - "--max-items", type=int, default=1000, help="Max messages to index (default: 1000)" - ) - add_embedding_args(wechat_parser) - - # iMessage indexing command - imessage_parser = subparsers.add_parser("index-imessage", help="Index iMessage history") - imessage_parser.add_argument( - "index_name", nargs="?", default="imessage", help="Index name (default: imessage)" - ) - imessage_parser.add_argument( - "--db-path", - type=str, - default=None, - help="Path to chat.db (default: ~/Library/Messages/chat.db)", - ) - imessage_parser.add_argument( - "--max-items", type=int, default=1000, help="Max messages to index (default: 1000)" - ) - add_embedding_args(imessage_parser) - # Slack indexing command slack_parser = subparsers.add_parser("index-slack", help="Index Slack workspace via MCP") slack_parser.add_argument( @@ -580,38 +507,6 @@ def add_embedding_args(target_parser: argparse.ArgumentParser) -> None: ) add_embedding_args(slack_parser) - # ChatGPT indexing command - chatgpt_parser = subparsers.add_parser("index-chatgpt", help="Index ChatGPT export") - chatgpt_parser.add_argument( - "index_name", nargs="?", default="chatgpt", help="Index name (default: chatgpt)" - ) - chatgpt_parser.add_argument( - "--export-path", - type=str, - required=True, - help="Path to ChatGPT export file (.html/.zip) or directory", - ) - chatgpt_parser.add_argument( - "--max-items", type=int, default=1000, help="Max items to index (default: 1000)" - ) - add_embedding_args(chatgpt_parser) - - # Claude indexing command - claude_parser = subparsers.add_parser("index-claude", help="Index Claude export") - claude_parser.add_argument( - "index_name", nargs="?", default="claude", help="Index name (default: claude)" - ) - claude_parser.add_argument( - "--export-path", - type=str, - required=True, - help="Path to Claude export file (.json/.zip) or directory", - ) - claude_parser.add_argument( - "--max-items", type=int, default=1000, help="Max items to index (default: 1000)" - ) - add_embedding_args(claude_parser) - # Watch command watch_parser = subparsers.add_parser( "watch", @@ -2615,220 +2510,6 @@ def _load_chunk_ids_by_file( chunk_ids_by_file.setdefault(file_path, []).append(str(chunk_id)) return chunk_ids_by_file - async def index_browser(self, args): - """Build an index from browser history.""" - from .readers import ChromeHistoryReader - - browser_type = args.browser_type - profile = args.profile - index_name = args.index_name or f"{browser_type}-history" - - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print(f"🌐 Indexing {browser_type.capitalize()} history (profile: {profile})...") - - paths = ChromeHistoryReader.find_browser_paths() - if browser_type not in paths: - print(f"❌ Could not find {browser_type} profile directory automatically.") - return - - profile_path = paths[browser_type] / profile - - reader = ChromeHistoryReader() - documents = reader.load_data( - chrome_profile_path=str(profile_path), max_count=args.max_items - ) - - if not documents: - print("❌ No history entries found to index.") - return - - print(f"📚 Loaded {len(documents)} entries. Building index...") - - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ Browser history index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - - async def index_email(self, args): - """Build an index from Apple Mail emails.""" - from .readers import AppleMailReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("📧 Indexing Apple Mail emails...") - - reader = AppleMailReader() - documents = reader.load_data(max_count=args.max_items) - - if not documents: - print("❌ No emails found to index. Make sure Full Disk Access is granted.") - return - - print(f"📚 Loaded {len(documents)} emails. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ Email index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - - async def index_calendar(self, args): - """Build an index from Apple Calendar events.""" - from .readers import AppleCalendarReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("📅 Indexing Apple Calendar events...") - - reader = AppleCalendarReader() - documents = reader.load_data(max_count=args.max_items) - - if not documents: - print("❌ No calendar events found to index. Make sure Full Disk Access is granted.") - return - - print(f"📚 Loaded {len(documents)} events. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ Calendar index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - - async def index_wechat(self, args): - """Build an index from WeChat chat history.""" - from .readers import WeChatHistoryReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("💬 Indexing WeChat chat history...") - - reader = WeChatHistoryReader() - export_dir = getattr(args, "export_dir", "./wechat_export") - documents = reader.load_data(wechat_export_dir=export_dir, max_count=args.max_items) - - if not documents: - print("❌ No WeChat data found. Make sure WeChat is exported first.") - return - - print(f"📚 Loaded {len(documents)} messages. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ WeChat index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - - async def index_imessage(self, args): - """Build an index from iMessage history.""" - from .readers import IMessageReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("💬 Indexing iMessage history...") - - reader = IMessageReader() - documents = reader.load_data(max_count=args.max_items) - - if not documents: - print("❌ No iMessage data found. Make sure Full Disk Access is granted.") - return - - print(f"📚 Loaded {len(documents)} messages. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ iMessage index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - async def index_slack(self, args): """Build an index from Slack workspace via MCP.""" from .readers import SlackMCPReader @@ -2872,86 +2553,6 @@ async def index_slack(self, args): print(f"✅ Slack index built at: {index_path}") print(f' Usage: leann search {index_name} "query"') - async def index_chatgpt(self, args): - """Build an index from ChatGPT export.""" - from .readers import ChatGPTReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("🤖 Indexing ChatGPT export...") - - reader = ChatGPTReader() - documents = reader.load_data(export_path=args.export_path) - - if not documents: - print("❌ No ChatGPT data found. Make sure export file is valid.") - return - - print(f"📚 Loaded {len(documents)} items. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ ChatGPT index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - - async def index_claude(self, args): - """Build an index from Claude export.""" - from .readers import ClaudeReader - - index_name = args.index_name - index_dir = self.indexes_dir / index_name - index_path = str(index_dir / "documents.leann") - - print("🤖 Indexing Claude export...") - - reader = ClaudeReader() - documents = reader.load_data(export_path=args.export_path) - - if not documents: - print("❌ No Claude data found. Make sure export file is valid.") - return - - print(f"📚 Loaded {len(documents)} items. Building index...") - index_dir.mkdir(parents=True, exist_ok=True) - - embedding_options = {} - if args.embedding_mode == "ollama": - embedding_options["host"] = resolve_ollama_host(None) - - builder = LeannBuilder( - backend_name="hnsw", - embedding_model=args.embedding_model, - embedding_mode=args.embedding_mode, - embedding_options=embedding_options or None, - is_recompute=False, - is_compact=False, - ) - - for doc in documents: - builder.add_text(doc.text, metadata=doc.metadata) - - builder.build_index(index_path) - print(f"✅ Claude index built at: {index_path}") - print(f' Usage: leann search {index_name} "query"') - async def build_index(self, args): docs_paths = args.docs # Use current directory name if index_name not provided diff --git a/tests/test_cli_index_commands.py b/tests/test_cli_index_commands.py new file mode 100644 index 00000000..d92d24cb --- /dev/null +++ b/tests/test_cli_index_commands.py @@ -0,0 +1,199 @@ +"""Exercise index command parsing and dispatch without accessing personal data.""" + +import asyncio +import os +import shutil +import sqlite3 +import sys +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock, call + +import pytest +from leann import cli as cli_module +from leann.cli import LeannCLI +from llama_index.core import Document + +SOURCES = { + "browser": ("apps.history_data.history", "ChromeHistoryReader", "browser_history"), + "email": ("apps.email_data.LEANN_email_reader", "EmlxReader", "email"), + "calendar": (None, None, "calendar"), + "imessage": ("apps.imessage_data.imessage_reader", "IMessageReader", "imessage"), + "wechat": ("apps.history_data.wechat_history", "WeChatHistoryReader", "wechat"), + "chatgpt": ("apps.chatgpt_data.chatgpt_reader", "ChatGPTReader", "chatgpt"), + "claude": ("apps.claude_data.claude_reader", "ClaudeReader", "claude"), +} + + +@pytest.fixture +def cli(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + monkeypatch.setattr( + os.path, + "expanduser", + lambda path: str(tmp_path / path[2:]) if path.startswith("~/") else path, + ) + return LeannCLI() + + +def command_args(source, tmp_path): + argv = [f"index-{source}"] + if source == "browser": + argv.append("brave") + elif source == "wechat": + argv.extend(["--export-dir", str(tmp_path / "export")]) + elif source in {"chatgpt", "claude"}: + argv.extend(["--export-path", str(tmp_path / "export.json")]) + return argv + + +@pytest.mark.parametrize("source", SOURCES) +def test_index_command_defaults(cli, source, tmp_path): + args = cli.create_parser().parse_args(command_args(source, tmp_path)) + assert args.index_name == SOURCES[source][2] + assert args.max_count == 1000 + assert not args.no_recompute + + +@pytest.mark.parametrize("source", SOURCES) +@pytest.mark.parametrize("empty", [False, True], ids=["documents", "empty"]) +@pytest.mark.parametrize("no_recompute", [False, True], ids=["recompute", "stored"]) +def test_index_command_dispatch(cli, source, empty, no_recompute, tmp_path, monkeypatch): + # Install fake modules before dispatch: no application reader is imported or run. + documents = [] if empty else [Document(text="Synthetic text", metadata={"source": source})] + module_name, reader_name, _ = SOURCES[source] + reader_class = Mock() + reader_class.return_value.load_data.return_value = documents + if module_name: + assert reader_name is not None + parts = module_name.split(".") + for end in range(1, len(parts) + 1): + name = ".".join(parts[:end]) + module = ModuleType(name) + module.__path__ = [] + monkeypatch.setitem(sys.modules, name, module) + module = sys.modules[module_name] + setattr(module, reader_name, reader_class) + if source == "email": + monkeypatch.setattr( + module, + "find_all_messages_directories", + Mock(return_value=[tmp_path / "mail"]), + raising=False, + ) + else: + # The legacy calendar handler embeds its reader. Exercise its actual SQL + # against a synthetic database, redirecting its fixed scratch path. + calendar_cache = tmp_path / "Library" / "Calendars" / "Calendar Cache" + calendar_cache.parent.mkdir(parents=True) + connection = sqlite3.connect(calendar_cache) + connection.execute( + "CREATE TABLE CI_EVENT (summary, description, location, start_date, end_date)" + ) + if not empty: + connection.executemany( + "INSERT INTO CI_EVENT VALUES (?, ?, ?, ?, ?)", + [(f"Synthetic event {i}", "Details", "Room", i, i + 1) for i in range(3)], + ) + connection.commit() + monkeypatch.setattr(shutil, "copy2", Mock()) + monkeypatch.setattr(sqlite3, "connect", Mock(return_value=connection)) + original_exists = os.path.exists + monkeypatch.setattr( + os.path, + "exists", + lambda path: False + if path == "/tmp/leann_calendar_index_copy" + else original_exists(path), + ) + + builder_class = Mock() + register = Mock() + monkeypatch.setattr(cli_module, "LeannBuilder", builder_class) + monkeypatch.setattr(cli, "register_project_dir", register) + argv = [ + *command_args(source, tmp_path), + "--index-name", + "synthetic-index", + "--max-count", + "2", + "--embedding-model", + "synthetic-model", + "--embedding-mode", + "ollama", + "--embedding-host", + "http://127.0.0.1:9999", + ] + if no_recompute: + argv.append("--no-recompute") + args = cli.create_parser().parse_args(argv) + asyncio.run(cli.run(args)) + + if source == "browser": + reader_class.return_value.load_data.assert_called_once_with( + chrome_profile_path=str( + tmp_path / "Library/Application Support/BraveSoftware/Brave-Browser/Default" + ), + max_count=2, + ) + elif source == "email": + reader_class.return_value.load_data.assert_called_once_with( + str(tmp_path / "mail"), max_count=2 + ) + elif source == "imessage": + reader_class.assert_called_once_with(concatenate_conversations=True) + reader_class.return_value.load_data.assert_called_once_with() + elif source == "wechat": + reader_class.return_value.load_data.assert_called_once_with( + input_dir=str(tmp_path / "export"), + max_count=2, + concatenate_messages=True, + ) + elif source in {"chatgpt", "claude"}: + reader_class.assert_called_once_with(concatenate_conversations=True) + reader_class.return_value.load_data.assert_called_once_with( + input_dir=str(tmp_path / "export.json"), + max_count=2, + ) + + if empty: + builder_class.assert_not_called() + register.assert_not_called() + return + + builder_class.assert_called_once_with( + backend_name="hnsw", + embedding_model="synthetic-model", + embedding_mode="ollama", + embedding_options={"host": "http://127.0.0.1:9999"}, + is_recompute=not no_recompute, + ) + builder = builder_class.return_value + if source == "calendar": + assert builder.add_text.call_count == 2 + assert "Synthetic event 2" in builder.add_text.call_args_list[0].args[0] + assert "Synthetic event 1" in builder.add_text.call_args_list[1].args[0] + else: + assert builder.add_text.call_args_list == [ + call(doc.text, metadata=doc.metadata) for doc in documents + ] + builder.build_index.assert_called_once_with(cli.get_index_path("synthetic-index")) + register.assert_called_once_with() + + +def test_top_level_help(cli, capsys): + with pytest.raises(SystemExit) as exc: + cli.create_parser().parse_args(["--help"]) + assert exc.value.code == 0 + output = capsys.readouterr().out + for source in SOURCES: + assert f"index-{source}" in output + + +def test_unrelated_list_dispatch(cli, monkeypatch): + list_indexes = Mock() + monkeypatch.setattr(cli, "list_indexes", list_indexes) + args = cli.create_parser().parse_args(["list", "--max-depth", "2"]) + asyncio.run(cli.run(args)) + list_indexes.assert_called_once_with(2)