diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c06b14..69f30cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,9 @@ jobs: PYTHONPATH: . run: python -m pytest -q + - name: Verify native SeekDB retrieval + run: timeout 120s python scripts/verify_seekdb_native.py + - name: Compile backend modules run: python -m compileall -q backend diff --git a/README.md b/README.md index e84e925..a9aa56e 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,8 @@ Key backend choices: - Python 3.10+ - Node.js 18+ - A text model endpoint configured in `config.yaml` -- Optional: embedding, rerank, and speech model endpoints +- An embedding endpoint when using the default strict SeekDB mode +- Optional: rerank and speech model endpoints ### Backend @@ -111,16 +112,19 @@ cp config_example.yaml config.yaml Edit `config.yaml` and fill in the profiles you want to use: - `api.models.text_model`: chat, RAG answers, Studio artifacts, podcast scripts -- `api.models.embedding_model`: optional vector retrieval +- `api.models.embedding_model`: required for ingestion and vector retrieval when `seekdb_allow_sqlite_fallback` is `false` - `api.models.rerank_model`: optional rerank - `api.models.audio_model`: optional speech/audio generation - `api.models.image_model`: slide image generation - `api.models.edit_model`: single-slide image editing -- `storage.seekdb_path`: local knowledge database +- `storage.seekdb_path`: local repository base path; SQLite keeps business metadata and native SeekDB keeps chunk vectors and retrieval indexes beside it +- `storage.seekdb_allow_sqlite_fallback`: compatibility-only SQLite retrieval fallback; keep `false` to make SeekDB authoritative - `documents.chunking`: Chonkie/simple chunking settings Keep `config.yaml` local. It is ignored by git and should not contain committed keys. +Once sources are indexed, the runtime settings API rejects changes to the embedding model, adapter, or base URL because existing vectors would no longer be compatible. Delete and re-import the sources before changing that embedding identity. API-key rotation does not require re-indexing. + Start the backend: ```bash diff --git a/README_zh.md b/README_zh.md index c23c448..afc1926 100644 --- a/README_zh.md +++ b/README_zh.md @@ -99,7 +99,8 @@ backend/ - Python 3.10+ - Node.js 18+ - 一个可用的文本模型接口 -- 可选：embedding、rerank、语音模型接口 +- 默认严格 SeekDB 模式下需要可用的 embedding 接口 +- 可选：rerank、语音模型接口 ### 后端 @@ -111,16 +112,19 @@ cp config_example.yaml config.yaml 编辑 `config.yaml`，填写模型与存储配置： - `api.models.text_model`：聊天、RAG 回答、Studio artifact、播客脚本 -- `api.models.embedding_model`：可选向量召回 +- `api.models.embedding_model`：当 `seekdb_allow_sqlite_fallback` 为 `false` 时，文档摄取与向量召回必填 - `api.models.rerank_model`：可选 rerank - `api.models.audio_model`：可选语音/音频生成 - `api.models.image_model`：Slide Deck 图片生成 - `api.models.edit_model`：单页图片编辑 -- `storage.seekdb_path`：本地知识库路径 +- `storage.seekdb_path`：本地 repository 基础路径；SQLite 保存业务元数据，原生 SeekDB 在相邻目录保存 chunk 向量与检索索引 +- `storage.seekdb_allow_sqlite_fallback`：仅用于兼容的 SQLite 检索回退；保持 `false` 时 SeekDB 是检索权威存储 - `documents.chunking`：Chonkie/simple chunk 参数 `config.yaml` 只用于本地运行，已被 git 忽略，不要提交真实 key。 +已有 source 完成索引后，运行时配置接口会拒绝修改 embedding 模型、adapter 或 base URL，避免用不兼容的新查询向量检索旧索引。需要先删除并重新导入 source；仅轮换 API key 不需要重建索引。 + 启动后端： ```bash diff --git a/backend/api/routes/config.py b/backend/api/routes/config.py index 83321fe..a9761f2 100644 --- a/backend/api/routes/config.py +++ b/backend/api/routes/config.py @@ -2,9 +2,15 @@ from __future__ import annotations -from fastapi import APIRouter +from fastapi import APIRouter, HTTPException -from ...config import ModelProfile, ModelProfiles, get_settings, update_runtime_model_profiles +from ...config import ( + ModelProfile, + ModelProfiles, + get_settings, + model_profile_identity, + update_runtime_model_profiles, +) from ...dependencies import DependencyContainer from ..schemas.config import PublicModelProfile, RuntimeConfigResponse, RuntimeConfigUpdate @@ -24,9 +30,12 @@ def _public_profile(profile: ModelProfile) -> PublicModelProfile: ) -def _response(message: str = "") -> RuntimeConfigResponse: +async def _response(message: str = "") -> RuntimeConfigResponse: settings = get_settings() models = settings.api.models + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + storage_status = stats.get("storage", {}) return RuntimeConfigResponse( models={ name: _public_profile(getattr(models, name)) @@ -40,7 +49,18 @@ def _response(message: str = "") -> RuntimeConfigResponse: }, storage={ "vector_store_type": settings.vector_store_type, + "configured_vector_store_type": settings.vector_store_type, "seekdb_path": settings.seekdb_path, + "seekdb_allow_sqlite_fallback": settings.seekdb_allow_sqlite_fallback, + "actual_vector_backend": storage_status.get("vector_backend", stats.get("backend", "unknown")), + "native_available": storage_status.get("native_available", False), + "embedding_required": ( + settings.vector_store_type == "seekdb" + and not settings.seekdb_allow_sqlite_fallback + ), + "embedding_configured": bool( + models.embedding_model.model and models.embedding_model.api_key + ), }, message=message, ) @@ -48,15 +68,38 @@ def _response(message: str = "") -> RuntimeConfigResponse: @router.get("", response_model=RuntimeConfigResponse) async def get_runtime_config() -> RuntimeConfigResponse: - return _response() + return await _response() @router.post("", response_model=RuntimeConfigResponse) async def update_runtime_config(request: RuntimeConfigUpdate) -> RuntimeConfigResponse: + requested_embedding = request.models.get("embedding_model") + if requested_embedding is not None: + settings = get_settings() + current_profile = settings.api.models.embedding_model + requested_values = requested_embedding.model_dump(exclude_none=True) + if requested_values.get("api_key") == "": + requested_values.pop("api_key") + next_profile = current_profile.model_copy(update=requested_values) + if model_profile_identity(next_profile) != model_profile_identity(current_profile): + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + indexed_vector_chunks = stats.get( + "indexed_vector_chunks", + stats.get("total_chunks", 0), + ) + if indexed_vector_chunks > 0: + raise HTTPException( + status_code=409, + detail=( + "The embedding model, adapter, or base URL cannot change while indexed " + "sources exist. Delete and re-index the sources first." + ), + ) profile_data = { name: profile.model_dump(exclude_none=True) for name, profile in request.models.items() } update_runtime_model_profiles(profile_data) - DependencyContainer.reset_runtime_caches() - return _response("Runtime model configuration updated.") + DependencyContainer.reset_runtime_caches(preserve_repository=True) + return await _response("Runtime model configuration updated.") diff --git a/backend/api/routes/documents.py b/backend/api/routes/documents.py index 1277ded..084bbbb 100644 --- a/backend/api/routes/documents.py +++ b/backend/api/routes/documents.py @@ -61,6 +61,10 @@ async def upload_document( char_count=source.char_count ) + except HTTPException: + if os.path.exists(file_path): + os.remove(file_path) + raise except Exception as e: # 清理文件 if os.path.exists(file_path): diff --git a/backend/api/routes/sources.py b/backend/api/routes/sources.py index 3c5e2df..c313c07 100644 --- a/backend/api/routes/sources.py +++ b/backend/api/routes/sources.py @@ -45,6 +45,11 @@ def _detail(source: KnowledgeSource) -> SourceDetail: return SourceDetail(**_info(source).model_dump(), text=source.text) +def _raise_if_failed(source: KnowledgeSource) -> None: + if source.status.value == "error": + raise HTTPException(status_code=422, detail=source.error or "Source ingestion failed") + + @router.post("/text", response_model=SourceDetail) async def create_text_source( request: SourceCreateTextRequest, @@ -55,6 +60,7 @@ async def create_text_source( text=request.text, metadata=request.metadata, ) + _raise_if_failed(source) return _detail(source) @@ -76,7 +82,12 @@ async def upload_source( filename=filename, mime_type=file.content_type, ) + _raise_if_failed(source) return _detail(source) + except HTTPException: + if file_path.exists(): + file_path.unlink() + raise except Exception as exc: if file_path.exists(): file_path.unlink() diff --git a/backend/config.py b/backend/config.py index 273f16d..7f3aeea 100644 --- a/backend/config.py +++ b/backend/config.py @@ -7,6 +7,8 @@ from __future__ import annotations +import hashlib +import json import os from functools import lru_cache from pathlib import Path @@ -36,6 +38,18 @@ class ModelProfile(BaseModel): stream: bool = True +def model_profile_identity(profile: ModelProfile) -> str: + """Return a stable, secret-free identity for embedding compatibility checks.""" + + payload = { + "adapter": profile.adapter.strip(), + "base_url": profile.base_url.strip().rstrip("/"), + "model": profile.model.strip(), + } + serialized = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + class ModelProfiles(BaseModel): """Configured model roles.""" @@ -84,6 +98,7 @@ class Settings(BaseSettings): vector_store_type: Literal["chroma", "faiss", "seekdb"] = "seekdb" chroma_persist_dir: str = "./data/chroma" seekdb_path: str = "./data/seekdb.db" + seekdb_allow_sqlite_fallback: bool = False # 嵌入模型设置 (OpenAI API) embedding_model: str = "text-embedding-3-small" @@ -138,7 +153,12 @@ def _flatten_config(data: dict[str, Any]) -> dict[str, Any]: storage = data.get("storage") or {} if isinstance(storage, dict): - for key in ("vector_store_type", "chroma_persist_dir", "seekdb_path"): + for key in ( + "vector_store_type", + "chroma_persist_dir", + "seekdb_path", + "seekdb_allow_sqlite_fallback", + ): if key in storage: flattened[key] = storage[key] diff --git a/backend/core/services/source_service.py b/backend/core/services/source_service.py index e12cc75..0b54bec 100644 --- a/backend/core/services/source_service.py +++ b/backend/core/services/source_service.py @@ -97,10 +97,12 @@ async def _finish_source( for chunk, embedding in zip(chunks, embeddings): chunk.embedding = embedding source.chunk_count = len(chunks) - source.status = SourceStatus.READY source.updated_at = utc_now() await self.repository.save_source(source) await self.repository.save_chunks(source.id, chunks) + source.status = SourceStatus.READY + source.updated_at = utc_now() + await self.repository.save_source(source) return source async def _fail_source(self, source: KnowledgeSource, exc: Exception) -> KnowledgeSource: diff --git a/backend/dependencies.py b/backend/dependencies.py index e073f71..df5be28 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -6,7 +6,7 @@ """ from typing import Any, Optional -from .config import ModelProfile, Settings, get_settings +from .config import ModelProfile, Settings, get_settings, model_profile_identity from .core.interfaces.vector_store import VectorStoreInterface from .core.interfaces.knowledge_repository import KnowledgeRepositoryInterface from .core.interfaces.llm_provider import LLMProviderInterface @@ -22,11 +22,18 @@ class DependencyContainer: _slide_deck_service: Optional[Any] = None @classmethod - def reset_runtime_caches(cls) -> None: + def reset_runtime_caches(cls, *, preserve_repository: bool = False) -> None: """Drop cached services that depend on model settings.""" + repository = cls._knowledge_repository cls._vector_store = None cls._slide_deck_service = None + if preserve_repository: + return + cls._knowledge_repository = None + close_repository = getattr(repository, "close_sync", None) + if callable(close_repository): + close_repository() @classmethod def get_vector_store( @@ -86,9 +93,23 @@ def get_knowledge_repository( force_new: bool = False, ) -> KnowledgeRepositoryInterface: if cls._knowledge_repository is None or force_new: + if force_new and cls._knowledge_repository is not None: + cls._vector_store = None + cls._slide_deck_service = None + close_repository = getattr(cls._knowledge_repository, "close_sync", None) + if callable(close_repository): + close_repository() settings = settings or get_settings() from .infrastructure.repositories.seekdb_repository import SeekDBRepository - cls._knowledge_repository = SeekDBRepository(settings.seekdb_path) + cls._knowledge_repository = SeekDBRepository( + settings.seekdb_path, + allow_sqlite_vector_fallback=settings.seekdb_allow_sqlite_fallback, + embedding_profile_id=model_profile_identity(settings.api.models.embedding_model), + ) + elif settings is not None: + embedding_profile_id = model_profile_identity(settings.api.models.embedding_model) + if hasattr(cls._knowledge_repository, "embedding_profile_id"): + cls._knowledge_repository.embedding_profile_id = embedding_profile_id return cls._knowledge_repository @classmethod diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 6396f7b..97e2682 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -1,9 +1,8 @@ -"""SeekDB repository wrapper with embedded pyseekdb preference. +"""SQLite entity repository with a native SeekDB knowledge-chunk index. -The pyseekdb SDK is loaded lazily. When it is unavailable, the repository keeps -the same durable interface on top of SQLite so development and tests remain -deterministic while deployment can use pyseekdb embedded mode by installing the -SDK. +In strict mode, SeekDB owns chunk vectors and retrieval. SQLite keeps source and +chunk metadata without embeddings. SQLite vector scoring is retained only for +the explicitly enabled compatibility fallback. """ from __future__ import annotations @@ -13,15 +12,23 @@ import math import re import sqlite3 +from datetime import datetime, timezone from pathlib import Path from typing import Any +from uuid import uuid4 from ...core.interfaces.knowledge_repository import KnowledgeRepositoryInterface from ...domain.slide_deck import SlideAsset, SlideDeckExport, SlideDeckJob, SlideDeckProject from ...domain.source import Artifact, Job, KnowledgeChunk, KnowledgeSource, Note, SourceStatus, utc_now +from ..vector_stores.seekdb_chunk_index import SeekDBChunkIndex logger = logging.getLogger(__name__) +_AUTO_NATIVE_INDEX: Any = object() +_NATIVE_UNAVAILABLE_MESSAGE = ( + "native SeekDB vector index is unavailable; enable seekdb_allow_sqlite_fallback for SQLite fallback" +) + def _dump_model(model: Any) -> str: return model.model_dump_json() @@ -80,23 +87,37 @@ def _bm25_scores(query: str, documents: list[str]) -> list[float]: class SeekDBRepository(KnowledgeRepositoryInterface): - def __init__(self, db_path: str | Path) -> None: + def __init__( + self, + db_path: str | Path, + native_chunk_index: object = _AUTO_NATIVE_INDEX, + allow_sqlite_vector_fallback: bool = False, + embedding_profile_id: str | None = None, + ) -> None: self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) - self.seekdb_client = self._try_pyseekdb(self.db_path.parent / f"{self.db_path.stem}.seekdb") + self.seekdb_path = self.db_path.parent / f"{self.db_path.stem}.seekdb" + self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback + self.embedding_profile_id = embedding_profile_id + self.native_chunk_index = ( + self._try_native_chunk_index(self.seekdb_path) + if native_chunk_index is _AUTO_NATIVE_INDEX + else native_chunk_index + ) self._conn = sqlite3.connect(self.db_path, check_same_thread=False) self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") self._init_schema() + self._storage_initialized = False + self._migration_has_unresolved_sources = False + self._closed = False - @staticmethod - def _try_pyseekdb(path: Path) -> Any | None: + def _try_native_chunk_index(self, path: Path) -> Any | None: try: - import pyseekdb # type: ignore - except Exception: - return None - try: - return pyseekdb.Client(path=str(path)) - except Exception: + return SeekDBChunkIndex(path) + except Exception as exc: + if not self.allow_sqlite_vector_fallback: + logger.warning("Native SeekDB vector index is unavailable: %s", exc) return None def _init_schema(self) -> None: @@ -115,10 +136,31 @@ def _init_schema(self) -> None: content TEXT NOT NULL, chunk_index INTEGER NOT NULL, embedding TEXT, + vector_state TEXT NOT NULL DEFAULT 'legacy', payload TEXT NOT NULL, FOREIGN KEY(source_id) REFERENCES sources(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks(source_id); + CREATE TABLE IF NOT EXISTS chunk_index_state ( + source_id TEXT PRIMARY KEY, + revision TEXT NOT NULL, + chunk_count INTEGER NOT NULL, + embedding_dimension INTEGER, + embedding_profile TEXT, + storage_mode TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS chunk_sync_operations ( + source_id TEXT PRIMARY KEY, + revision TEXT NOT NULL, + operation TEXT NOT NULL, + payload TEXT, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS repository_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); CREATE TABLE IF NOT EXISTS artifacts ( id TEXT PRIMARY KEY, payload TEXT NOT NULL, @@ -169,10 +211,67 @@ def _init_schema(self) -> None: CREATE INDEX IF NOT EXISTS idx_slide_deck_jobs_deck_id ON slide_deck_jobs(deck_id); """ ) + chunk_columns = { + row["name"] for row in self._conn.execute("PRAGMA table_info(chunks)").fetchall() + } + if "vector_state" not in chunk_columns: + self._conn.execute( + "ALTER TABLE chunks ADD COLUMN vector_state TEXT NOT NULL DEFAULT 'legacy'" + ) + state_columns = { + row["name"] for row in self._conn.execute("PRAGMA table_info(chunk_index_state)").fetchall() + } + if "embedding_profile" not in state_columns: + self._conn.execute("ALTER TABLE chunk_index_state ADD COLUMN embedding_profile TEXT") + if self.embedding_profile_id: + self._conn.execute( + """ + UPDATE chunk_index_state + SET embedding_profile = ? + WHERE embedding_profile IS NULL AND storage_mode = 'seekdb' + """, + (self.embedding_profile_id,), + ) self._conn.commit() async def close(self) -> None: - self._conn.close() + self.close_sync() + + def close_sync(self) -> None: + if self._closed: + return + try: + close_native = getattr(self.native_chunk_index, "close", None) + if callable(close_native): + close_native() + finally: + self._conn.close() + self._closed = True + + def storage_status(self) -> dict[str, Any]: + if self.native_chunk_index is not None: + try: + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() + status = dict(self.native_chunk_index.status()) + status.setdefault("seekdb_path", str(self.seekdb_path)) + status.setdefault("native_available", True) + return status + except Exception as exc: + return { + "vector_backend": ( + "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable" + ), + "seekdb_path": str(self.seekdb_path), + "native_available": False, + "error": str(exc), + } + return { + "vector_backend": "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable", + "seekdb_path": str(self.seekdb_path), + "native_available": False, + } async def save_source(self, source: KnowledgeSource) -> KnowledgeSource: self._conn.execute( @@ -207,81 +306,962 @@ async def list_sources(self) -> list[KnowledgeSource]: return [KnowledgeSource.model_validate_json(row["payload"]) for row in rows] async def delete_source(self, source_id: str) -> bool: - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) - self._conn.commit() + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + + await self._ensure_storage_initialized() + + if self.native_chunk_index is None: + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + return True + + previous_native_chunks: list[KnowledgeChunk] | None = None + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) + revision = uuid4().hex + self._record_sync_operation(source_id, revision, "delete", None) + try: + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_source_delete(source_id, revision) + except Exception: + self._compensate_native_failure(source_id, revision, previous_native_chunks) + raise return True + async def delete_source_metadata_only(self, source_id: str) -> None: + """Rollback local metadata without touching native SeekDB.""" + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_sync_operations WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + + if self.native_chunk_index is not None and chunks and not self.allow_sqlite_vector_fallback: + missing_embeddings = [chunk.id for chunk in chunks if not chunk.embedding] + if missing_embeddings: + raise RuntimeError( + "native SeekDB chunk writes require embeddings for all chunks; " + f"missing embeddings for: {', '.join(missing_embeddings)}" + ) + native_eligible_chunks = self._native_eligible_chunks(chunks) + if len(native_eligible_chunks) != len(chunks): + raise RuntimeError("native SeekDB chunk writes require same-dimension embeddings for all chunks") + + await self._ensure_storage_initialized() + + if self.native_chunk_index is None: + with self._conn: + self._replace_sqlite_chunks(source_id, chunks, persist_embedding=True) + self._upsert_chunk_index_state( + source_id, + uuid4().hex, + chunks, + storage_mode="sqlite_fallback", + ) + return + + native_chunks = self._native_eligible_chunks(chunks) + previous_native_chunks: list[KnowledgeChunk] | None = None + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) + revision = uuid4().hex + persist_embedding = self.allow_sqlite_vector_fallback + self._record_sync_operation( + source_id, + revision, + "replace", + self._replacement_operation_payload(chunks, persist_embedding), + ) + + try: + self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) + self._finalize_chunk_replace( + source_id, + revision, + chunks, + persist_embedding=persist_embedding, + ) + except Exception: + self._compensate_native_failure(source_id, revision, previous_native_chunks) + raise + + async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: + await self._ensure_storage_initialized() + state = self._chunk_index_states([source_id]).get(source_id) + if state is not None and state["storage_mode"] == "sqlite_legacy": + return self._get_sqlite_chunks_sync(source_id) + if self.native_chunk_index is not None: + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + try: + chunks = get_native_chunks(source_id) + if chunks or not self.allow_sqlite_vector_fallback: + return chunks + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + if not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + return self._get_sqlite_chunks_sync(source_id) + + async def backfill_native_chunks(self) -> int: + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + return 0 + raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() + self._recover_pending_sync_operations() + written = self._backfill_native_chunks_sync() + if not self.allow_sqlite_vector_fallback and not self._migration_has_unresolved_sources: + self._set_repository_meta("native_chunk_migration_v2", "complete") + self._reconcile_processing_sources() + self._storage_initialized = True + return written + + async def initialize_storage(self) -> int: + """Recover interrupted writes and migrate legacy SQLite vectors into SeekDB.""" + if self._storage_initialized: + return 0 + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + self._reconcile_processing_sources() + self._storage_initialized = True + return 0 + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() + self._recover_pending_sync_operations() + written = 0 + if ( + not self.allow_sqlite_vector_fallback + and self._get_repository_meta("native_chunk_migration_v2") != "complete" + ): + written = self._backfill_native_chunks_sync() + if not self._migration_has_unresolved_sources: + self._set_repository_meta("native_chunk_migration_v2", "complete") + self._reconcile_processing_sources() + self._storage_initialized = True + return written + + async def _ensure_storage_initialized(self) -> None: + if not self._storage_initialized: + await self.initialize_storage() + + def _backfill_native_chunks_sync(self) -> int: + written = 0 + self._migration_has_unresolved_sources = False + source_rows = self._conn.execute( + "SELECT payload FROM sources WHERE status != ? ORDER BY created_at DESC", + (SourceStatus.DELETED.value,), + ).fetchall() + sources = [KnowledgeSource.model_validate_json(row["payload"]) for row in source_rows] + reconciled_states = {"seekdb"} + if self.allow_sqlite_vector_fallback: + reconciled_states.add("sqlite_fallback_reconciled") + reconciled_placeholders = ",".join("?" for _ in reconciled_states) + for source in sources: + state_row = self._conn.execute( + "SELECT source_id FROM chunk_index_state WHERE source_id = ?", + (source.id,), + ).fetchone() + if state_row: + unreconciled_row = self._conn.execute( + f""" + SELECT 1 + FROM chunks + WHERE source_id = ? + AND vector_state NOT IN ({reconciled_placeholders}) + LIMIT 1 + """, + (source.id, *reconciled_states), + ).fetchone() + if unreconciled_row is None: + continue + sqlite_rows = self._conn.execute( + "SELECT payload, vector_state FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", + (source.id,), + ).fetchall() + chunks = [KnowledgeChunk.model_validate_json(row["payload"]) for row in sqlite_rows] + if not chunks: + with self._conn: + self._upsert_chunk_index_state( + source.id, + uuid4().hex, + [], + storage_mode="seekdb", + ) + continue + if state_row and all(row["vector_state"] in reconciled_states for row in sqlite_rows): + continue + if state_row and all(row["vector_state"] == "legacy_needs_embedding" for row in sqlite_rows): + self._migration_has_unresolved_sources = True + continue + chunks_with_embeddings = self._native_eligible_chunks(chunks) + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + previous_native_chunks = get_native_chunks(source.id) if callable(get_native_chunks) else None + if len(chunks_with_embeddings) != len(chunks): + if all(row["vector_state"] in {"legacy", "seekdb"} for row in sqlite_rows) and self._chunks_match_without_vectors( + chunks, + previous_native_chunks or [], + ): + with self._conn: + self._conn.execute( + "UPDATE chunks SET vector_state = 'seekdb' WHERE source_id = ?", + (source.id,), + ) + self._upsert_chunk_index_state( + source.id, + uuid4().hex, + previous_native_chunks or [], + storage_mode="seekdb", + ) + continue + logger.warning( + "Legacy source %s has missing or inconsistent embeddings", + source.id, + ) + revision = uuid4().hex + if not self.allow_sqlite_vector_fallback: + self._migration_has_unresolved_sources = True + self._record_sync_operation(source.id, revision, "legacy_clear", None) + try: + self.native_chunk_index.upsert_source_chunks(source.id, []) + self._finalize_legacy_source(source.id, revision, chunks) + except Exception: + self._compensate_native_failure(source.id, revision, previous_native_chunks) + raise + continue + self._record_sync_operation( + source.id, + revision, + "replace", + self._replacement_operation_payload(chunks, persist_embedding=True), + ) + try: + self.native_chunk_index.upsert_source_chunks(source.id, []) + self._finalize_chunk_replace( + source.id, + revision, + chunks, + persist_embedding=True, + storage_mode="sqlite_fallback", + vector_state="sqlite_fallback_reconciled", + ) + except Exception: + self._compensate_native_failure(source.id, revision, previous_native_chunks) + raise + continue + revision = uuid4().hex + persist_embedding = self.allow_sqlite_vector_fallback + finalize_mode = "replace" if persist_embedding else "scrub" + self._record_sync_operation( + source.id, + revision, + "replace", + self._replacement_operation_payload( + chunks, + persist_embedding, + finalize_mode=finalize_mode, + ), + ) + try: + self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + if finalize_mode == "scrub": + self._finalize_chunk_migration(source.id, revision, chunks) + else: + self._finalize_chunk_replace( + source.id, + revision, + chunks, + persist_embedding=persist_embedding, + ) + except Exception: + self._compensate_native_failure(source.id, revision, previous_native_chunks) + raise + written += len(chunks_with_embeddings) + return written + + async def search_chunks( + self, + query: str, + source_ids: list[str] | None = None, + top_k: int = 5, + query_embedding: list[float] | None = None, + rerank_provider: object | None = None, + ) -> list[dict]: + if source_ids == []: + return [] + await self._ensure_storage_initialized() + if self.native_chunk_index is not None and query_embedding is None: + query_text = query.strip() + if not query_text: + raise RuntimeError("native SeekDB full-text search requires query text") + requested_source_ids = source_ids if source_ids is not None else self._current_source_ids() + selected_source_ids = self._native_source_ids_with_chunks(requested_source_ids) + text_search = getattr(self.native_chunk_index, "text_search", None) + results: list[dict] = [] + if not callable(text_search): + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB full-text search is unavailable") + selected_source_ids = [] + elif selected_source_ids: + try: + results.extend(text_search(query_text, selected_source_ids, top_k)) + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + selected_source_ids = [] + if self.allow_sqlite_vector_fallback: + native_source_id_set = set(selected_source_ids) + fallback_source_ids = [ + source_id for source_id in requested_source_ids if source_id not in native_source_id_set + ] + if fallback_source_ids: + results.extend( + await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(fallback_source_ids), + top_k, + None, + None, + ) + ) + else: + legacy_source_ids = self._legacy_source_ids(requested_source_ids) + if legacy_source_ids: + legacy_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(legacy_source_ids), + top_k, + None, + None, + ) + for result in legacy_results: + result["backend"] = "sqlite_legacy_text" + results.extend(legacy_results) + results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + return await self._maybe_rerank(query_text, results, top_k, rerank_provider) + if self.native_chunk_index is not None and query_embedding is not None: + if not query_embedding: + if self.allow_sqlite_vector_fallback: + rows = self._chunk_rows(source_ids) + return await self._search_sqlite_chunk_rows(query, rows, top_k, None, rerank_provider) + raise RuntimeError("native SeekDB vector search requires a non-empty query embedding") + requested_source_ids = source_ids if source_ids is not None else self._current_source_ids() + selected_source_ids = self._native_search_source_ids(requested_source_ids, query_embedding) + results: list[dict] = [] + hybrid_search = getattr(self.native_chunk_index, "hybrid_search", None) + if selected_source_ids: + try: + if callable(hybrid_search): + results.extend( + hybrid_search( + query_text=query, + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + ) + else: + results.extend(self.native_chunk_index.search(query_embedding, selected_source_ids, top_k)) + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + selected_source_ids = [] + if self.allow_sqlite_vector_fallback: + native_source_id_set = set(selected_source_ids) + fallback_source_ids = [ + source_id for source_id in requested_source_ids if source_id not in native_source_id_set + ] + if fallback_source_ids: + fallback_query_embedding = ( + None + if self._has_embedding_profile_mismatch(fallback_source_ids) + else query_embedding + ) + fallback_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(fallback_source_ids), + top_k, + fallback_query_embedding, + None, + ) + results.extend(fallback_results) + else: + legacy_source_ids = self._legacy_source_ids(requested_source_ids) + if legacy_source_ids: + legacy_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(legacy_source_ids), + top_k, + None, + None, + ) + for result in legacy_results: + result["backend"] = "sqlite_legacy_text" + results.extend(legacy_results) + results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + return await self._maybe_rerank(query, results, top_k, rerank_provider) + + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + + rows = self._chunk_rows(source_ids) + return await self._search_sqlite_chunk_rows(query, rows, top_k, query_embedding, rerank_provider) + + def _current_source_ids(self) -> list[str]: + rows = self._conn.execute( + "SELECT id FROM sources WHERE status != ? ORDER BY created_at DESC", + (SourceStatus.DELETED.value,), + ).fetchall() + return [row["id"] for row in rows] + + @staticmethod + def _native_eligible_chunks(chunks: list[KnowledgeChunk]) -> list[KnowledgeChunk]: + if not chunks: + return [] + dimension: int | None = None + for chunk in chunks: + if not chunk.embedding: + return [] + if dimension is None: + dimension = len(chunk.embedding) + elif len(chunk.embedding) != dimension: + return [] + return chunks + + def _native_search_source_ids(self, source_ids: list[str], query_embedding: list[float]) -> list[str]: + if not source_ids: + return [] + query_dimension = len(query_embedding) + states = self._chunk_index_states(source_ids) + seekdb_states = { + source_id: states[source_id] + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "seekdb" + and states[source_id]["chunk_count"] > 0 + } + if not self.allow_sqlite_vector_fallback: + profile_mismatches = [ + source_id + for source_id, state in seekdb_states.items() + if self.embedding_profile_id + and state["embedding_profile"] + and state["embedding_profile"] != self.embedding_profile_id + ] + if profile_mismatches: + raise RuntimeError( + "Selected sources were indexed with a different embedding profile; " + "re-index the sources before vector retrieval: " + + ", ".join(profile_mismatches) + ) + dimension_mismatches = [ + source_id + for source_id, state in seekdb_states.items() + if state["embedding_dimension"] != query_dimension + ] + if dimension_mismatches: + raise RuntimeError( + "Selected sources use an incompatible embedding dimension; " + "re-index the sources before vector retrieval: " + + ", ".join(dimension_mismatches) + ) + return [ + source_id + for source_id, state in seekdb_states.items() + if state["embedding_dimension"] == query_dimension + and ( + not self.embedding_profile_id + or not state["embedding_profile"] + or state["embedding_profile"] == self.embedding_profile_id + ) + ] + + def _native_source_ids_with_chunks(self, source_ids: list[str]) -> list[str]: + states = self._chunk_index_states(source_ids) + return [ + source_id + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "seekdb" + and states[source_id]["chunk_count"] > 0 + ] + + def _legacy_source_ids(self, source_ids: list[str]) -> list[str]: + states = self._chunk_index_states(source_ids) + return [ + source_id + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "sqlite_legacy" + and states[source_id]["chunk_count"] > 0 + ] + + def _has_embedding_profile_mismatch(self, source_ids: list[str]) -> bool: + if not self.embedding_profile_id: + return False + states = self._chunk_index_states(source_ids) + return any( + state["embedding_profile"] + and state["embedding_profile"] != self.embedding_profile_id + for state in states.values() + ) + + def indexed_vector_chunk_count(self) -> int: + row = self._conn.execute( + """ + SELECT COALESCE(SUM(chunk_count), 0) AS total + FROM chunk_index_state + WHERE storage_mode = 'seekdb' + """ + ).fetchone() + return int(row["total"]) + + def _reconcile_processing_sources(self) -> int: + rows = self._conn.execute( + """ + SELECT sources.payload, chunk_index_state.chunk_count AS indexed_chunk_count + FROM sources + JOIN chunk_index_state ON chunk_index_state.source_id = sources.id + WHERE sources.status = ? + """, + (SourceStatus.PROCESSING.value,), + ).fetchall() + recovered = 0 + with self._conn: + for row in rows: + source = KnowledgeSource.model_validate_json(row["payload"]) + if not source.text or source.chunk_count != row["indexed_chunk_count"]: + continue + source.status = SourceStatus.READY + source.error = None + source.updated_at = utc_now() + self._conn.execute( + """ + UPDATE sources + SET status = ?, payload = ?, updated_at = ? + WHERE id = ? AND status = ? + """, + ( + source.status.value, + _dump_model(source), + source.updated_at.isoformat(), + source.id, + SourceStatus.PROCESSING.value, + ), + ) + recovered += 1 + return recovered + + def _record_sync_operation( + self, + source_id: str, + revision: str, + operation: str, + payload: str | None, + ) -> None: + with self._conn: + self._conn.execute( + """ + INSERT INTO chunk_sync_operations(source_id, revision, operation, payload, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + revision=excluded.revision, + operation=excluded.operation, + payload=excluded.payload, + created_at=excluded.created_at + """, + (source_id, revision, operation, payload, datetime.now(timezone.utc).isoformat()), + ) + + def _discard_sync_operation(self, source_id: str, revision: str) -> None: + try: + with self._conn: + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + except Exception: + logger.exception("Failed to discard completed chunk sync operation for %s", source_id) + + def _compensate_native_failure( + self, + source_id: str, + revision: str, + previous_native_chunks: list[KnowledgeChunk] | None, + ) -> None: + if previous_native_chunks is None: + logger.error( + "Cannot restore SeekDB source %s because its previous state is unavailable; " + "the original sync operation remains pending", + source_id, + ) + return + try: + self._record_sync_operation( + source_id, + revision, + "native_restore", + json.dumps([chunk.model_dump_json() for chunk in previous_native_chunks]), + ) + except Exception: + logger.exception( + "Failed to persist native restore operation for %s; original operation remains pending", + source_id, + ) + return + try: + self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks for %s; restore remains pending", source_id) + return + self._discard_sync_operation(source_id, revision) + + @staticmethod + def _replacement_operation_payload( + chunks: list[KnowledgeChunk], + persist_embedding: bool, + *, + finalize_mode: str = "replace", + ) -> str: + return json.dumps( + { + "chunks": [chunk.model_dump_json() for chunk in chunks], + "persist_embedding": persist_embedding, + "finalize_mode": finalize_mode, + } + ) + + @staticmethod + def _load_replacement_operation(payload: str | None) -> tuple[list[KnowledgeChunk], bool, str]: + if payload is None: + raise ValueError("replace operation is missing its payload") + value = json.loads(payload) + chunks = [KnowledgeChunk.model_validate_json(item) for item in value.get("chunks", [])] + return chunks, bool(value.get("persist_embedding", False)), value.get("finalize_mode", "replace") + + def _recover_pending_sync_operations(self) -> None: + if self.native_chunk_index is None: + return + rows = self._conn.execute( + """ + SELECT source_id, revision, operation, payload + FROM chunk_sync_operations + ORDER BY created_at ASC + """ + ).fetchall() + for row in rows: + source_id = row["source_id"] + revision = row["revision"] + if row["operation"] == "replace": + chunks, persist_embedding, finalize_mode = self._load_replacement_operation(row["payload"]) + self.native_chunk_index.upsert_source_chunks( + source_id, + self._native_eligible_chunks(chunks), + ) + if finalize_mode == "scrub": + self._finalize_chunk_migration(source_id, revision, chunks) + else: + self._finalize_chunk_replace( + source_id, + revision, + chunks, + persist_embedding=persist_embedding, + ) + elif row["operation"] == "delete": + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_source_delete(source_id, revision) + elif row["operation"] == "native_restore": + payload = json.loads(row["payload"] or "[]") + chunks = [KnowledgeChunk.model_validate_json(item) for item in payload] + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + self._discard_sync_operation(source_id, revision) + elif row["operation"] == "legacy_clear": + chunks = self._get_sqlite_chunks_sync(source_id) + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_legacy_source(source_id, revision, chunks) + else: + raise ValueError(f"Unknown chunk sync operation: {row['operation']}") + + def _finalize_chunk_replace( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + *, + persist_embedding: bool, + storage_mode: str | None = None, + vector_state: str | None = None, + ) -> None: + eligible_chunks = self._native_eligible_chunks(chunks) + resolved_storage_mode = storage_mode or ( + "seekdb" if not chunks or len(eligible_chunks) == len(chunks) else "sqlite_fallback" + ) + with self._conn: + self._replace_sqlite_chunks( + source_id, + chunks, + persist_embedding=persist_embedding, + vector_state=vector_state, + ) + self._upsert_chunk_index_state( + source_id, + revision, + eligible_chunks if resolved_storage_mode == "seekdb" else chunks, + storage_mode=resolved_storage_mode, + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _finalize_source_delete(self, source_id: str, revision: str) -> None: + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _finalize_chunk_migration( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + ) -> None: + with self._conn: + for chunk in chunks: + self._conn.execute( + """ + UPDATE chunks + SET embedding = NULL, payload = ?, vector_state = 'seekdb' + WHERE id = ? AND source_id = ? + """, + (self._chunk_payload_without_embedding(chunk), chunk.id, source_id), + ) + self._upsert_chunk_index_state( + source_id, + revision, + chunks, + storage_mode="seekdb", + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _finalize_legacy_source( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + ) -> None: + with self._conn: + self._conn.execute( + "UPDATE chunks SET vector_state = 'legacy_needs_embedding' WHERE source_id = ?", + (source_id,), + ) + self._upsert_chunk_index_state( + source_id, + revision, + chunks, + storage_mode="sqlite_legacy", + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _replace_sqlite_chunks( + self, + source_id: str, + chunks: list[KnowledgeChunk], + *, + persist_embedding: bool, + vector_state: str | None = None, + ) -> None: self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) self._conn.executemany( """ - INSERT INTO chunks(id, source_id, content, chunk_index, embedding, payload) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO chunks(id, source_id, content, chunk_index, embedding, vector_state, payload) + VALUES (?, ?, ?, ?, ?, ?, ?) """, [ - ( - chunk.id, - chunk.source_id, - chunk.content, - chunk.chunk_index, - json.dumps(chunk.embedding) if chunk.embedding is not None else None, - _dump_model(chunk), + self._sqlite_chunk_row( + chunk, + persist_embedding=persist_embedding, + vector_state=vector_state, ) for chunk in chunks ], ) - self._conn.commit() - if self.seekdb_client is not None and chunks: # pragma: no cover - requires SDK/runtime - try: - collection = self.seekdb_client.get_or_create_collection(name="chunks") - payload: dict[str, Any] = { - "ids": [chunk.id for chunk in chunks], - "documents": [chunk.content for chunk in chunks], - "metadatas": [chunk.metadata for chunk in chunks], - } - if all(chunk.embedding is not None for chunk in chunks): - payload["embeddings"] = [chunk.embedding for chunk in chunks] - collection.upsert(**payload) - except Exception as exc: - logger.warning("Skipping optional pyseekdb chunk mirror after save failure: %s", exc) + def _upsert_chunk_index_state( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + *, + storage_mode: str, + ) -> None: + dimension = len(chunks[0].embedding or []) if chunks and chunks[0].embedding else None + self._conn.execute( + """ + INSERT INTO chunk_index_state( + source_id, revision, chunk_count, embedding_dimension, + embedding_profile, storage_mode, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + revision=excluded.revision, + chunk_count=excluded.chunk_count, + embedding_dimension=excluded.embedding_dimension, + embedding_profile=excluded.embedding_profile, + storage_mode=excluded.storage_mode, + updated_at=excluded.updated_at + """, + ( + source_id, + revision, + len(chunks), + dimension, + self.embedding_profile_id, + storage_mode, + datetime.now(timezone.utc).isoformat(), + ), + ) - async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: + def _chunk_index_states(self, source_ids: list[str]) -> dict[str, sqlite3.Row]: + if not source_ids: + return {} + placeholders = ",".join("?" for _ in source_ids) + rows = self._conn.execute( + f""" + SELECT source_id, chunk_count, embedding_dimension, embedding_profile, storage_mode + FROM chunk_index_state + WHERE source_id IN ({placeholders}) + """, + source_ids, + ).fetchall() + return {row["source_id"]: row for row in rows} + + def _get_repository_meta(self, key: str) -> str | None: + row = self._conn.execute("SELECT value FROM repository_meta WHERE key = ?", (key,)).fetchone() + return row["value"] if row else None + + def _set_repository_meta(self, key: str, value: str) -> None: + with self._conn: + self._conn.execute( + """ + INSERT INTO repository_meta(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value + """, + (key, value), + ) + + def _get_sqlite_chunks_sync(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( "SELECT payload FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", (source_id,), ).fetchall() return [KnowledgeChunk.model_validate_json(row["payload"]) for row in rows] - async def search_chunks( + @staticmethod + def _chunk_payload_without_embedding(chunk: KnowledgeChunk) -> str: + return chunk.model_copy(update={"embedding": None}).model_dump_json() + + @staticmethod + def _chunks_match_without_vectors( + sqlite_chunks: list[KnowledgeChunk], + native_chunks: list[KnowledgeChunk], + ) -> bool: + if len(sqlite_chunks) != len(native_chunks): + return False + + def signature(chunk: KnowledgeChunk) -> dict[str, Any]: + return chunk.model_dump(exclude={"embedding", "created_at"}) + + sqlite_signatures = {chunk.id: signature(chunk) for chunk in sqlite_chunks} + native_signatures = {chunk.id: signature(chunk) for chunk in native_chunks} + return sqlite_signatures == native_signatures + + def _sqlite_chunk_row( self, - query: str, - source_ids: list[str] | None = None, - top_k: int = 5, - query_embedding: list[float] | None = None, - rerank_provider: object | None = None, - ) -> list[dict]: + chunk: KnowledgeChunk, + *, + persist_embedding: bool | None = None, + vector_state: str | None = None, + ) -> tuple[Any, ...]: + if persist_embedding is None: + persist_embedding = self.allow_sqlite_vector_fallback + return ( + chunk.id, + chunk.source_id, + chunk.content, + chunk.chunk_index, + json.dumps(chunk.embedding) if persist_embedding and chunk.embedding is not None else None, + vector_state or ("sqlite_fallback" if persist_embedding else "seekdb"), + _dump_model(chunk) if persist_embedding else self._chunk_payload_without_embedding(chunk), + ) + + def _chunk_rows(self, source_ids: list[str] | None = None) -> list[sqlite3.Row]: + if source_ids == []: + return [] params: list[Any] = [] sql = "SELECT payload, content, embedding FROM chunks" - if source_ids: + if source_ids is not None: placeholders = ",".join("?" for _ in source_ids) sql += f" WHERE source_id IN ({placeholders})" params.extend(source_ids) - rows = self._conn.execute(sql, params).fetchall() + return self._conn.execute(sql, params).fetchall() + async def _search_sqlite_chunk_rows( + self, + query: str, + rows: list[sqlite3.Row], + top_k: int, + query_embedding: list[float] | None, + rerank_provider: object | None, + ) -> list[dict]: bm25_scores = _bm25_scores(query, [row["content"] for row in rows]) results = [] for row, bm25_score in zip(rows, bm25_scores): chunk = KnowledgeChunk.model_validate_json(row["payload"]) vector_score = 0.0 + has_comparable_embedding = False if query_embedding and row["embedding"]: - vector_score = _cosine(query_embedding, json.loads(row["embedding"])) + row_embedding = json.loads(row["embedding"]) + has_comparable_embedding = len(row_embedding) == len(query_embedding) + if has_comparable_embedding: + vector_score = _cosine(query_embedding, row_embedding) score = bm25_score + max(vector_score, 0.0) - if bm25_score > 0 or query_embedding: + if bm25_score > 0 or has_comparable_embedding: results.append({"chunk": chunk, "score": score}) results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + return await self._maybe_rerank(query, results, top_k, rerank_provider) + + async def _maybe_rerank( + self, + query: str, + results: list[dict], + top_k: int, + rerank_provider: object | None, + ) -> list[dict]: if rerank_provider and results: try: return await rerank_provider.rerank(query, results, top_k=top_k) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py new file mode 100644 index 0000000..840bc68 --- /dev/null +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -0,0 +1,304 @@ +"""Native SeekDB vector index for knowledge chunks.""" + +from __future__ import annotations + +import hashlib +import importlib +import logging +from pathlib import Path +from typing import Any + +from ...domain.source import KnowledgeChunk + +logger = logging.getLogger(__name__) + + +class SeekDBUnavailableError(RuntimeError): + """Raised when the native pyseekdb chunk index cannot be initialized.""" + + +class SeekDBChunkIndex: + """Source-scoped native pyseekdb chunk vector index.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.path.mkdir(parents=True, exist_ok=True) + self._closed = False + try: + self.pyseekdb = importlib.import_module("pyseekdb") + self.client = self.pyseekdb.Client(path=str(self.path)) + except Exception as exc: + raise SeekDBUnavailableError("Native SeekDB chunk index is unavailable") from exc + + def close(self) -> None: + if self._closed: + return + client_exit = getattr(self.client, "__exit__", None) + if callable(client_exit): + client_exit(None, None, None) + self._closed = True + + def probe(self) -> None: + """Force the lazy embedded client to connect before request-time work starts.""" + + list_collections = getattr(self.client, "list_collections", None) + if not callable(list_collections): + raise SeekDBUnavailableError("Native SeekDB client does not support a readiness probe") + try: + list_collections() + except Exception as exc: + raise SeekDBUnavailableError("Native SeekDB readiness probe failed") from exc + + def collection_name(self, source_id: str) -> str: + digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] + return f"chunks_{digest}" + + def _first_embedding_dimension(self, chunks: list[KnowledgeChunk]) -> int: + if not chunks: + raise ValueError("At least one chunk with an embedding is required") + embedding = chunks[0].embedding + if not embedding: + raise ValueError("Chunk embeddings must be non-empty") + return len(embedding) + + def _collection(self, source_id: str, dimension: int | None = None) -> Any: + name = self.collection_name(source_id) + if dimension is None: + return self.client.get_collection(name=name, embedding_function=None) + configuration = self.pyseekdb.HNSWConfiguration(dimension=dimension) + return self.client.get_or_create_collection( + name=name, + configuration=configuration, + embedding_function=None, + ) + + def _existing_collection(self, source_id: str) -> Any | None: + try: + return self._collection(source_id) + except Exception as exc: + if self._is_missing_collection_error(exc): + return None + raise + + @staticmethod + def _is_missing_collection_error(exc: Exception) -> bool: + if isinstance(exc, KeyError): + return True + message = str(exc).lower() + missing_markers = ( + "not found", + "not exist", + "doesn't exist", + "failed to resolve collection metadata", + ) + return "collection" in message and any(marker in message for marker in missing_markers) + + @staticmethod + def _collection_ids(collection: Any) -> list[str]: + result = collection.get(include=[]) + ids = result.get("ids", []) + if isinstance(ids, str): + return [ids] + return list(ids or []) + + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + existing_collection = self._existing_collection(source_id) + if not chunks: + if existing_collection is None: + return + existing_ids = self._collection_ids(existing_collection) + if existing_ids: + existing_collection.delete(ids=existing_ids) + existing_collection.refresh_index() + return + + dimension = self._first_embedding_dimension(chunks) + embeddings: list[list[float]] = [] + metadatas: list[dict[str, Any]] = [] + for chunk in chunks: + if chunk.source_id != source_id: + raise ValueError("chunk.source_id must match source_id") + if not chunk.embedding or len(chunk.embedding) != dimension: + raise ValueError("All chunks must have same-dimension non-empty embeddings") + embeddings.append(chunk.embedding) + metadata = dict(chunk.metadata) + metadata["source_id"] = chunk.source_id + metadata["chunk_index"] = chunk.chunk_index + metadata["payload"] = chunk.model_dump_json() + metadatas.append(metadata) + + if existing_collection is not None: + existing_dimension = getattr(existing_collection, "dimension", None) + if existing_dimension is not None and existing_dimension != dimension: + raise ValueError( + f"Existing SeekDB collection dimension {existing_dimension} " + f"does not match chunk embedding dimension {dimension}" + ) + collection = existing_collection + existing_ids = self._collection_ids(collection) + else: + collection = self._collection(source_id, dimension=dimension) + existing_ids = [] + + new_ids = [chunk.id for chunk in chunks] + collection.upsert( + ids=new_ids, + documents=[chunk.content for chunk in chunks], + metadatas=metadatas, + embeddings=embeddings, + ) + collection.refresh_index() + new_id_set = set(new_ids) + stale_ids = [chunk_id for chunk_id in existing_ids if chunk_id not in new_id_set] + if stale_ids: + collection.delete(ids=stale_ids) + collection.refresh_index() + + def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: + if not chunk_ids: + return + try: + collection = self._existing_collection(source_id) + if collection is None: + return + collection.delete(ids=chunk_ids) + collection.refresh_index() + except Exception as exc: + logger.warning("Failed to delete SeekDB chunks for source %s: %s", source_id, exc) + + def get_source_chunks(self, source_id: str) -> list[KnowledgeChunk]: + collection = self._existing_collection(source_id) + if collection is None: + return [] + result = collection.get(include=["metadatas"]) + metadatas = result.get("metadatas", []) or [] + chunks: list[KnowledgeChunk] = [] + for metadata in metadatas: + try: + chunks.append(KnowledgeChunk.model_validate_json(metadata["payload"])) + except Exception as exc: + raise ValueError("Malformed SeekDB chunk payload") from exc + return sorted(chunks, key=lambda chunk: chunk.chunk_index) + + def search( + self, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("Query embedding is required") + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + for source_id in source_ids: + collection = self._collection(source_id) + query_result = collection.query( + query_embeddings=[query_embedding], + n_results=top_k, + include=include, + ) + results.extend(self._rows_to_results(query_result, score_mode="distance")) + + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + + def text_search( + self, + query_text: str, + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + query_text = query_text.strip() + if not query_text: + raise ValueError("Query text is required") + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + n_results = max(top_k, 1) + for source_id in source_ids: + collection = self._collection(source_id) + collection_hybrid_search = getattr(collection, "hybrid_search", None) + if not callable(collection_hybrid_search): + raise RuntimeError("SeekDB collection does not support native full-text search") + query_result = collection_hybrid_search( + query={"where_document": {"$contains": query_text}, "n_results": n_results}, + knn=None, + rank=None, + n_results=n_results, + include=include, + ) + results.extend(self._rows_to_results(query_result, score_mode="similarity")) + + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + + def hybrid_search( + self, + query_text: str, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("Query embedding is required") + query_text = query_text.strip() + if not query_text: + return self.search(query_embedding, source_ids, top_k) + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + n_results = max(top_k, 1) + for source_id in source_ids: + collection = self._collection(source_id) + collection_hybrid_search = getattr(collection, "hybrid_search", None) + if not callable(collection_hybrid_search): + return self.search(query_embedding, source_ids, top_k) + + query_result = collection_hybrid_search( + query={"where_document": {"$contains": query_text}, "n_results": n_results}, + knn={"query_embeddings": query_embedding, "n_results": n_results}, + rank={"rrf": {}}, + n_results=n_results, + include=include, + ) + results.extend(self._rows_to_results(query_result, score_mode="similarity")) + + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + + @staticmethod + def _result_group(value: Any) -> list[Any]: + if not value: + return [] + first = value[0] + return first if isinstance(first, list) else value + + def _rows_to_results(self, query_result: dict[str, Any], score_mode: str) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + ids = self._result_group(query_result.get("ids", [])) + distances = self._result_group(query_result.get("distances", [])) + metadatas = self._result_group(query_result.get("metadatas", [])) + for index, _chunk_id in enumerate(ids): + try: + metadata = metadatas[index] + chunk = KnowledgeChunk.model_validate_json(metadata["payload"]) + except Exception as exc: + raise ValueError("Malformed SeekDB chunk result payload") from exc + raw_score = float(distances[index]) if index < len(distances) else 0.0 + if score_mode == "distance": + score = 1.0 / (1.0 + max(raw_score, 0.0)) + else: + score = raw_score + results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) + return results + + def status(self) -> dict[str, Any]: + return { + "vector_backend": "seekdb", + "seekdb_path": str(self.path), + "native_available": True, + } diff --git a/backend/infrastructure/vector_stores/seekdb_vector_store.py b/backend/infrastructure/vector_stores/seekdb_vector_store.py index d5ce655..999eafd 100644 --- a/backend/infrastructure/vector_stores/seekdb_vector_store.py +++ b/backend/infrastructure/vector_stores/seekdb_vector_store.py @@ -28,7 +28,10 @@ async def add_chunks( chunks: list[dict[str, Any]], embeddings: Optional[list[list[float]]] = None, ) -> None: + self._validate_strict_native_write(chunks, embeddings) + existing = await self.repository.get_source(doc_id) + created_source = existing is None if existing is None: existing = KnowledgeSource( id=doc_id, @@ -51,7 +54,41 @@ async def add_chunks( metadata={"source_id": doc_id, "chunk_index": index, **metadata}, ) ) - await self.repository.save_chunks(doc_id, knowledge_chunks) + try: + await self.repository.save_chunks(doc_id, knowledge_chunks) + except Exception: + if created_source: + await self._rollback_created_source(doc_id) + raise + + def _validate_strict_native_write( + self, + chunks: list[dict[str, Any]], + embeddings: Optional[list[list[float]]], + ) -> None: + if getattr(self.repository, "allow_sqlite_vector_fallback", True): + return + if getattr(self.repository, "native_chunk_index", None) is None: + raise RuntimeError( + "native SeekDB vector index is unavailable; " + "enable seekdb_allow_sqlite_fallback for SQLite fallback" + ) + if not chunks: + return + + provided_embeddings = embeddings or [] + has_missing_embedding = len(provided_embeddings) < len(chunks) or any( + embedding is None or len(embedding) == 0 for embedding in provided_embeddings[: len(chunks)] + ) + if has_missing_embedding: + raise RuntimeError("native SeekDB chunk writes require embeddings for all chunks") + + async def _rollback_created_source(self, doc_id: str) -> None: + rollback_metadata = getattr(self.repository, "delete_source_metadata_only", None) + if rollback_metadata is not None: + await rollback_metadata(doc_id) + return + await self.repository.delete_source(doc_id) async def search( self, @@ -95,7 +132,27 @@ async def get_document_chunks(self, doc_id: str) -> list[dict[str, Any]]: for chunk in chunks ] + def storage_status(self) -> dict[str, Any]: + repository_status = getattr(self.repository, "storage_status", None) + if callable(repository_status): + return repository_status() + return {"vector_backend": "unknown", "native_available": False} + async def get_stats(self) -> dict[str, Any]: + initialize_storage = getattr(self.repository, "initialize_storage", None) + if callable(initialize_storage): + await initialize_storage() sources = await self.repository.list_sources() chunk_count = sum(source.chunk_count for source in sources) - return {"total_documents": len(sources), "total_chunks": chunk_count, "backend": "seekdb"} + indexed_chunk_count = getattr(self.repository, "indexed_vector_chunk_count", None) + indexed_vector_chunks = ( + indexed_chunk_count() if callable(indexed_chunk_count) else chunk_count + ) + status = self.storage_status() + return { + "total_documents": len(sources), + "total_chunks": chunk_count, + "indexed_vector_chunks": indexed_vector_chunks, + "backend": status.get("vector_backend", "unknown"), + "storage": status, + } diff --git a/backend/main.py b/backend/main.py index c0df7e8..73505f9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,6 +10,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from .config import get_settings from .api.routes.chat import router as chat_router @@ -31,8 +32,19 @@ async def lifespan(app: FastAPI): os.makedirs(settings.upload_dir, exist_ok=True) os.makedirs(settings.output_dir, exist_ok=True) os.makedirs(settings.chroma_persist_dir, exist_ok=True) - - yield + + from .dependencies import DependencyContainer + + if settings.vector_store_type == "seekdb": + repository = DependencyContainer.get_knowledge_repository(settings=settings) + initialize_storage = getattr(repository, "initialize_storage", None) + if callable(initialize_storage): + await initialize_storage() + + try: + yield + finally: + DependencyContainer.reset_runtime_caches() # 创建应用 @@ -67,7 +79,45 @@ async def lifespan(app: FastAPI): # 健康检查 @app.get("/health") async def health_check(): - return {"status": "healthy", "version": settings.app_version} + from .dependencies import get_vector_store + + current_settings = get_settings() + embedding_profile = current_settings.api.models.embedding_model + embedding_configured = bool(embedding_profile.model and embedding_profile.api_key) + try: + vector_store = get_vector_store() + stats = await vector_store.get_stats() + storage = stats.get("storage", {}) + native_available = storage.get("native_available", False) + except Exception as exc: + return JSONResponse( + status_code=503, + content={ + "status": "unhealthy", + "version": current_settings.app_version, + "storage": { + "actual_vector_backend": "unavailable", + "native_available": False, + "embedding_configured": embedding_configured, + "error": str(exc), + }, + }, + ) + strict_seekdb = ( + current_settings.vector_store_type == "seekdb" + and not current_settings.seekdb_allow_sqlite_fallback + ) + healthy = not strict_seekdb or (native_available and embedding_configured) + payload = { + "status": "healthy" if healthy else "unhealthy", + "version": current_settings.app_version, + "storage": { + "actual_vector_backend": storage.get("vector_backend", stats.get("backend", "unknown")), + "native_available": native_available, + "embedding_configured": embedding_configured, + }, + } + return JSONResponse(status_code=200 if healthy else 503, content=payload) # 向量存储统计 diff --git a/config_example.yaml b/config_example.yaml index f322550..5aecbb6 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -48,6 +48,7 @@ api: storage: vector_store_type: "seekdb" seekdb_path: "./data/seekdb.db" + seekdb_allow_sqlite_fallback: false chroma_persist_dir: "./data/chroma" documents: diff --git a/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md b/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md new file mode 100644 index 0000000..300e30c --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md @@ -0,0 +1,1405 @@ +# SeekDB Primary Vector Retrieval Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make SeekDB the real primary vector and hybrid retrieval backend for NotebookLM-Lite RAG chunks, with SQLite kept only for entity metadata and explicit fallback. + +**Architecture:** Keep the current repository interface and SQLite-backed CRUD tables for sources, artifacts, notes, jobs, and slide-deck state. Move chunk embedding writes and retrieval into a dedicated SeekDB native chunk index using `pyseekdb` collections with explicit HNSW dimensions and `embedding_function=None`; SQLite chunk rows remain a metadata/fallback copy, not the default vector search engine. + +**Tech Stack:** FastAPI, Python 3.12, `pyseekdb>=1.3.0`, SQLite metadata tables, LiteLLM embeddings, pytest, Playwright smoke validation. + +--- + +## Current Problem + +Current code sets `vector_store_type: seekdb`, but `backend/infrastructure/repositories/seekdb_repository.py` writes embeddings into SQLite as JSON and searches by reading SQLite rows and calculating BM25/cosine in Python. The optional `pyseekdb` mirror is attempted after SQLite commit and fails in real uploads with: + +```text +Skipping optional pyseekdb chunk mirror after save failure: execute sql failed OB_INVALID_ARGUMENT(1210): Incorrect arguments to %s +``` + +Local SDK inspection shows the failure is caused by creating a `pyseekdb` collection without matching explicit embedding settings and then searching before refreshing the native index. A working explicit-embedding path, verified against the installed `pyseekdb`, is: + +```python +import pyseekdb + +client = pyseekdb.Client(path="/tmp/seekdb") +collection = client.create_collection( + name="chunks_source_abc", + configuration=pyseekdb.HNSWConfiguration(dimension=3), + embedding_function=None, +) +collection.upsert( + ids=["chunk_1"], + documents=["content"], + metadatas=[{"source_id": "source_1", "payload": "{}"}], + embeddings=[[0.1, 0.2, 0.3]], +) +collection.refresh_index() +collection.query( + query_embeddings=[0.1, 0.2, 0.3], + n_results=1, + include=["documents", "metadatas", "distances"], +) +``` + +This plan makes that working path the default RAG chunk path. + +## File Structure + +- Create `backend/infrastructure/vector_stores/seekdb_chunk_index.py` + - Owns all native `pyseekdb` collection creation, chunk upsert/delete/query, index refresh, collection naming, score normalization, and backend status. +- Modify `backend/infrastructure/repositories/seekdb_repository.py` + - Delegates chunk vector writes/searches to `SeekDBChunkIndex`. + - Keeps SQLite rows for source/chunk payload metadata and explicit fallback. + - Removes the optional mirror warning path. +- Modify `backend/config.py` + - Adds explicit fallback configuration and default primary semantics. +- Modify `config_example.yaml` + - Documents that SeekDB native vector retrieval is default and fallback is opt-in. +- Modify `backend/infrastructure/vector_stores/seekdb_vector_store.py` + - Reports the actual backend status from the repository. +- Modify `backend/api/routes/config.py` and `backend/api/schemas/config.py` + - Exposes actual storage status to the frontend config modal. +- Modify `backend/main.py` + - Extends `/health` with actual vector backend status. +- Modify tests: + - `tests/test_seekdb_chunk_index.py` + - `tests/test_chunking_and_rerank.py` + - `tests/test_source_knowledge_base.py` + - `tests/test_runtime_config_api.py` + - `tests/test_config_settings.py` + +## Decisions + +- The scope is RAG chunk vector storage and retrieval. SQLite may remain the metadata store for non-vector entities in this iteration. +- SeekDB native mode is the default for `vector_store_type: seekdb`. +- SQLite vector search is available only when `seekdb_allow_sqlite_fallback: true` or in tests that explicitly construct the repository with fallback enabled. +- The repository must never silently label SQLite vector search as SeekDB. +- Multi-source filtering will use one SeekDB collection per source. This avoids relying on metadata `where` filtering, which did not work reliably in local SDK checks. + +## Collection Model + +For source `source_id`, native chunks are stored in: + +```text +chunks_ +``` + +Each collection uses the embedding dimension of the first chunk with an embedding: + +```python +pyseekdb.HNSWConfiguration(dimension=len(first_embedding)) +embedding_function=None +``` + +Each metadata row contains enough data to reconstruct `KnowledgeChunk`: + +```python +{ + "source_id": chunk.source_id, + "chunk_index": chunk.chunk_index, + "payload": chunk.model_dump_json(), +} +``` + +Searching selected sources queries each selected source collection, merges results by normalized score, applies optional rerank, and returns the existing `list[dict]` shape: + +```python +{"chunk": KnowledgeChunk(...), "score": 0.82} +``` + +--- + +### Task 1: Add a Native SeekDB Chunk Index + +**Files:** +- Create: `backend/infrastructure/vector_stores/seekdb_chunk_index.py` +- Test: `tests/test_seekdb_chunk_index.py` + +- [ ] **Step 1: Write failing tests for native collection creation and query** + +Create `tests/test_seekdb_chunk_index.py`: + +```python +import sys +import types +from pathlib import Path + +import pytest + +from backend.domain.source import KnowledgeChunk +from backend.infrastructure.vector_stores.seekdb_chunk_index import ( + SeekDBChunkIndex, + SeekDBUnavailableError, +) + + +class FakeHNSWConfiguration: + def __init__(self, dimension: int, distance: str = "cosine") -> None: + self.dimension = dimension + self.distance = distance + + +class FakeCollection: + def __init__(self, name: str, dimension: int) -> None: + self.name = name + self.dimension = dimension + self.upsert_calls = [] + self.delete_calls = [] + self.query_calls = [] + self.refresh_calls = 0 + self.rows = {} + + def upsert(self, ids, documents, metadatas, embeddings): + self.upsert_calls.append( + { + "ids": ids, + "documents": documents, + "metadatas": metadatas, + "embeddings": embeddings, + } + ) + for index, chunk_id in enumerate(ids): + self.rows[chunk_id] = { + "document": documents[index], + "metadata": metadatas[index], + "embedding": embeddings[index], + } + + def delete(self, ids=None, **kwargs): + self.delete_calls.append({"ids": ids, **kwargs}) + if ids: + for chunk_id in ids: + self.rows.pop(chunk_id, None) + + def query(self, query_embeddings, n_results, include): + self.query_calls.append( + { + "query_embeddings": query_embeddings, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[0.0 + index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + + def count(self): + return len(self.rows) + + def refresh_index(self): + self.refresh_calls += 1 + + +class FakeClient: + instances = [] + + def __init__(self, path: str) -> None: + self.path = path + self.collections = {} + self.created = [] + FakeClient.instances.append(self) + + def get_or_create_collection(self, name, configuration=None, embedding_function=None): + if embedding_function is not None: + raise AssertionError("explicit LiteLLM embeddings require embedding_function=None") + if configuration is None: + raise AssertionError("native SeekDB collection must declare vector dimension") + if name not in self.collections: + self.created.append((name, configuration.dimension)) + self.collections[name] = FakeCollection(name, configuration.dimension) + return self.collections[name] + + def get_collection(self, name, embedding_function=None): + if name not in self.collections: + raise KeyError(name) + return self.collections[name] + + +@pytest.fixture(autouse=True) +def fake_pyseekdb(monkeypatch, request): + if request.node.name == "test_real_pyseekdb_upsert_refresh_and_query_round_trip": + yield + return + FakeClient.instances.clear() + module = types.SimpleNamespace(Client=FakeClient, HNSWConfiguration=FakeHNSWConfiguration) + monkeypatch.setitem(sys.modules, "pyseekdb", module) + yield + + +def chunk(chunk_id: str, source_id: str, embedding: list[float]) -> KnowledgeChunk: + return KnowledgeChunk( + id=chunk_id, + source_id=source_id, + content=f"{chunk_id} content", + chunk_index=0, + embedding=embedding, + metadata={"source_id": source_id}, + ) + + +def test_upsert_creates_dimensioned_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + client = FakeClient.instances[0] + assert client.created == [(index.collection_name("source-a"), 3)] + collection = client.collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["ids"] == ["chunk-a"] + assert collection.upsert_calls[0]["embeddings"] == [[0.1, 0.2, 0.3]] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert "payload" in collection.upsert_calls[0]["metadatas"][0] + assert collection.refresh_calls == 1 + + +def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("source-b", [chunk("chunk-b", "source-b", [0.9, 0.1, 0.1])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a", "source-b"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert all(item["backend"] == "seekdb" for item in results) + client = FakeClient.instances[0] + assert client.collections[index.collection_name("source-a")].query_calls + assert client.collections[index.collection_name("source-b")].query_calls + + +def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: Path, monkeypatch): + monkeypatch.setitem(sys.modules, "pyseekdb", None) + + with pytest.raises(SeekDBUnavailableError): + SeekDBChunkIndex(tmp_path / "native.seekdb") + + +def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path, monkeypatch): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=1, + ) + + assert results[0]["chunk"].id == "real-chunk" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_seekdb_chunk_index.py -q +``` + +Expected: FAIL with `ModuleNotFoundError` or import error for `seekdb_chunk_index`. + +- [ ] **Step 3: Implement the native chunk index** + +Create `backend/infrastructure/vector_stores/seekdb_chunk_index.py`: + +```python +"""Native SeekDB chunk vector index. + +This module owns pyseekdb usage. The repository may keep SQLite metadata rows, +but RAG vector search should go through this index when vector_store_type=seekdb. +""" + +from __future__ import annotations + +import hashlib +import logging +from pathlib import Path +from typing import Any + +from ...domain.source import KnowledgeChunk + +logger = logging.getLogger(__name__) + + +class SeekDBUnavailableError(RuntimeError): + """Raised when native SeekDB is required but the SDK cannot be initialized.""" + + +class SeekDBChunkIndex: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.path.mkdir(parents=True, exist_ok=True) + try: + import pyseekdb # type: ignore + except Exception as exc: + raise SeekDBUnavailableError("pyseekdb is required for native SeekDB vector retrieval") from exc + + self._pyseekdb = pyseekdb + try: + self.client = pyseekdb.Client(path=str(self.path)) + except Exception as exc: + raise SeekDBUnavailableError(f"failed to initialize pyseekdb at {self.path}") from exc + + @staticmethod + def collection_name(source_id: str) -> str: + digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] + return f"chunks_{digest}" + + @staticmethod + def _first_embedding_dimension(chunks: list[KnowledgeChunk]) -> int: + for chunk in chunks: + if chunk.embedding: + return len(chunk.embedding) + raise ValueError("SeekDB native indexing requires chunk embeddings") + + def _collection(self, source_id: str, dimension: int | None = None) -> Any: + name = self.collection_name(source_id) + if dimension is None: + return self.client.get_collection(name=name, embedding_function=None) + return self.client.get_or_create_collection( + name=name, + configuration=self._pyseekdb.HNSWConfiguration(dimension=dimension), + embedding_function=None, + ) + + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if not chunks: + return + dimension = self._first_embedding_dimension(chunks) + missing = [chunk.id for chunk in chunks if not chunk.embedding or len(chunk.embedding) != dimension] + if missing: + raise ValueError(f"all chunks require {dimension}-dimensional embeddings: {missing}") + + collection = self._collection(source_id, dimension=dimension) + collection.upsert( + ids=[chunk.id for chunk in chunks], + documents=[chunk.content for chunk in chunks], + metadatas=[ + { + "source_id": chunk.source_id, + "chunk_index": chunk.chunk_index, + "payload": chunk.model_dump_json(), + } + for chunk in chunks + ], + embeddings=[chunk.embedding for chunk in chunks], + ) + collection.refresh_index() + + def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: + if not chunk_ids: + return + try: + collection = self._collection(source_id) + collection.delete(ids=chunk_ids) + except Exception as exc: + logger.warning("SeekDB native chunk delete skipped for %s: %s", source_id, exc) + + def search( + self, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("SeekDB native search requires a query embedding") + + merged: list[dict[str, Any]] = [] + per_source_k = max(top_k, 1) + for source_id in source_ids: + try: + collection = self._collection(source_id) + response = collection.query( + query_embeddings=query_embedding, + n_results=per_source_k, + include=["documents", "metadatas", "distances"], + ) + except Exception as exc: + logger.warning("SeekDB native search skipped source %s: %s", source_id, exc) + continue + + ids = (response.get("ids") or [[]])[0] + distances = (response.get("distances") or [[]])[0] + metadatas = (response.get("metadatas") or [[]])[0] + for index, chunk_id in enumerate(ids): + metadata = metadatas[index] if index < len(metadatas) else {} + payload = metadata.get("payload") + if not payload: + continue + distance = distances[index] if index < len(distances) else 1.0 + merged.append( + { + "chunk": KnowledgeChunk.model_validate_json(payload), + "score": 1.0 / (1.0 + max(float(distance), 0.0)), + "backend": "seekdb", + "id": chunk_id, + } + ) + + return sorted(merged, key=lambda item: item["score"], reverse=True)[:top_k] + + def status(self) -> dict[str, Any]: + return { + "vector_backend": "seekdb", + "seekdb_path": str(self.path), + "native_available": True, + } +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_seekdb_chunk_index.py -q +``` + +Expected: `4 passed`. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_chunk_index.py tests/test_seekdb_chunk_index.py +git commit -m "feat(seekdb): add native chunk vector index" +``` + +--- + +### Task 2: Wire Repository Chunk Writes to Native SeekDB Primary + +**Files:** +- Modify: `backend/config.py` +- Modify: `config_example.yaml` +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_source_knowledge_base.py` +- Test: `tests/test_config_settings.py` + +- [ ] **Step 1: Write failing repository tests** + +Append to `tests/test_source_knowledge_base.py`: + +```python +import logging + + +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + self.deletes = [] + self.searches = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + + def delete_source_chunks(self, source_id, chunk_ids): + self.deletes.append((source_id, chunk_ids)) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_chunk", + source_id=source_ids[0], + content="native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + +@pytest.mark.asyncio +async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp_path: Path, caplog): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native", kind=SourceKind.TEXT, title="Native") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native", + source_id=source.id, + content="native indexed content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with caplog.at_level(logging.WARNING): + results = await repo.search_chunks( + "native indexed", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts[0][0] == source.id + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "native_chunk" + assert "Skipping optional pyseekdb chunk mirror" not in caplog.text + + +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native", kind=SourceKind.TEXT, title="No Native") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native", + source_id=source.id, + content="content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) +``` + +Modify `tests/test_config_settings.py` to assert fallback is disabled by default: + +```python +def test_seekdb_native_vector_search_is_default(sample_config_file): + settings = get_settings() + + assert settings.vector_store_type == "seekdb" + assert settings.seekdb_allow_sqlite_fallback is False +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_uses_seekdb_native_index_for_chunk_save_and_search tests/test_source_knowledge_base.py::test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled tests/test_config_settings.py::test_seekdb_native_vector_search_is_default -q +``` + +Expected: FAIL because `SeekDBRepository` does not accept `native_chunk_index` or `allow_sqlite_vector_fallback`, and `Settings` lacks `seekdb_allow_sqlite_fallback`. + +- [ ] **Step 3: Add explicit config** + +Modify `backend/config.py` in `Settings`: + +```python + # SeekDB settings + seekdb_path: str = "./data/seekdb.db" + seekdb_allow_sqlite_fallback: bool = False +``` + +Modify the public dict construction in `Settings.model_config` related code if the project currently enumerates storage keys. Include: + +```python + for key in ( + "vector_store_type", + "chroma_persist_dir", + "seekdb_path", + "seekdb_allow_sqlite_fallback", + ): +``` + +Modify `config_example.yaml`: + +```yaml +vector_store: + vector_store_type: "seekdb" + seekdb_path: "./data/seekdb.db" + # SeekDB is the primary vector/hybrid retrieval backend. + # Set this true only for local debugging when pyseekdb is unavailable. + seekdb_allow_sqlite_fallback: false +``` + +If the file uses a flat `vector_store_type` block instead of `vector_store:`, preserve the existing shape and add: + +```yaml + seekdb_allow_sqlite_fallback: false +``` + +- [ ] **Step 4: Wire native chunk index into repository** + +Modify `backend/infrastructure/repositories/seekdb_repository.py`: + +```python +from ...infrastructure.vector_stores.seekdb_chunk_index import SeekDBChunkIndex, SeekDBUnavailableError +``` + +Change `__init__`: + +```python + def __init__( + self, + db_path: str | Path, + native_chunk_index: object | None = None, + allow_sqlite_vector_fallback: bool = False, + ) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback + self.native_chunk_index = native_chunk_index + if self.native_chunk_index is None: + try: + self.native_chunk_index = SeekDBChunkIndex(self.db_path.parent / f"{self.db_path.stem}.seekdb") + except SeekDBUnavailableError as exc: + if not allow_sqlite_vector_fallback: + logger.warning("Native SeekDB vector index unavailable: %s", exc) + self.native_chunk_index = None + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() +``` + +Remove `_try_pyseekdb` and `self.seekdb_client`. + +Replace the end of `save_chunks` after SQLite commit: + +```python + if self.native_chunk_index is not None: + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + return + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB vector index is unavailable; enable seekdb_allow_sqlite_fallback for SQLite fallback") +``` + +Replace the top of `search_chunks`: + +```python + if self.native_chunk_index is not None and query_embedding: + selected_source_ids = source_ids or [source.id for source in await self.list_sources()] + results = self.native_chunk_index.search( + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + if rerank_provider and results: + try: + return await rerank_provider.rerank(query, results, top_k=top_k) + except Exception: + return results + return results + if self.native_chunk_index is not None and not query_embedding: + logger.warning("SeekDB native vector search skipped because query embedding is missing") + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB vector search requires query embeddings") +``` + +Keep the existing SQLite BM25/cosine code below as fallback only. + +Modify `delete_source` before SQLite deletion: + +```python + chunks = await self.get_chunks(source_id) + if self.native_chunk_index is not None: + self.native_chunk_index.delete_source_chunks(source_id, [chunk.id for chunk in chunks]) +``` + +Add: + +```python + def storage_status(self) -> dict[str, Any]: + if self.native_chunk_index is not None: + return self.native_chunk_index.status() + return { + "vector_backend": "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable", + "seekdb_path": str(self.db_path.parent / f"{self.db_path.stem}.seekdb"), + "native_available": False, + } +``` + +- [ ] **Step 5: Update dependency injection** + +Modify `backend/dependencies.py`: + +```python + cls._knowledge_repository = SeekDBRepository( + settings.seekdb_path, + allow_sqlite_vector_fallback=settings.seekdb_allow_sqlite_fallback, + ) +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_uses_seekdb_native_index_for_chunk_save_and_search tests/test_source_knowledge_base.py::test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled tests/test_config_settings.py::test_seekdb_native_vector_search_is_default -q +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add backend/config.py config_example.yaml backend/dependencies.py backend/infrastructure/repositories/seekdb_repository.py tests/test_source_knowledge_base.py tests/test_config_settings.py +git commit -m "feat(seekdb): make native vector index primary" +``` + +--- + +### Task 3: Preserve Lexical Recall Without Making SQLite Primary + +**Files:** +- Modify: `backend/infrastructure/vector_stores/seekdb_chunk_index.py` +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_chunking_and_rerank.py` + +- [ ] **Step 1: Write failing test for native hybrid call** + +Append to `tests/test_chunking_and_rerank.py`: + +```python +class HybridRecordingIndex: + def __init__(self) -> None: + self.calls = [] + + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid_chunk", + source_id=source_ids[0], + content="hybrid TLS result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.88, + "backend": "seekdb", + } + ] + + def search(self, query_embedding, source_ids, top_k): + raise AssertionError("hybrid_search should be preferred when query text is available") + + +@pytest.mark.asyncio +async def test_seekdb_repository_prefers_native_hybrid_search(tmp_path: Path): + native_index = HybridRecordingIndex() + repo = SeekDBRepository( + tmp_path / "hybrid.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid", kind=SourceKind.TEXT, title="Hybrid") + await repo.save_source(source) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.calls == [("TLS handshake", [0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "hybrid_chunk" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_chunking_and_rerank.py::test_seekdb_repository_prefers_native_hybrid_search -q +``` + +Expected: FAIL because `hybrid_search` is not called. + +- [ ] **Step 3: Implement `hybrid_search` in `SeekDBChunkIndex`** + +Add to `backend/infrastructure/vector_stores/seekdb_chunk_index.py`: + +```python + def hybrid_search( + self, + query_text: str, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_text.strip(): + return self.search(query_embedding=query_embedding, source_ids=source_ids, top_k=top_k) + + merged: list[dict[str, Any]] = [] + for source_id in source_ids: + try: + collection = self._collection(source_id) + response = collection.hybrid_search( + query={"match": {"document": query_text}}, + knn={"query_embeddings": query_embedding, "k": max(top_k, 1)}, + n_results=max(top_k, 1), + include=["documents", "metadatas", "distances"], + ) + except AttributeError: + return self.search(query_embedding=query_embedding, source_ids=source_ids, top_k=top_k) + except Exception as exc: + logger.warning("SeekDB hybrid search skipped source %s: %s", source_id, exc) + continue + + ids = (response.get("ids") or [[]])[0] + distances = (response.get("distances") or [[]])[0] + metadatas = (response.get("metadatas") or [[]])[0] + for index, chunk_id in enumerate(ids): + metadata = metadatas[index] if index < len(metadatas) else {} + payload = metadata.get("payload") + if not payload: + continue + distance = distances[index] if index < len(distances) else 1.0 + merged.append( + { + "chunk": KnowledgeChunk.model_validate_json(payload), + "score": 1.0 / (1.0 + max(float(distance), 0.0)), + "backend": "seekdb", + "id": chunk_id, + } + ) + + return sorted(merged, key=lambda item: item["score"], reverse=True)[:top_k] +``` + +Modify `search_chunks` in `backend/infrastructure/repositories/seekdb_repository.py`: + +```python + if self.native_chunk_index is not None and query_embedding: + selected_source_ids = source_ids or [source.id for source in await self.list_sources()] + if hasattr(self.native_chunk_index, "hybrid_search"): + results = self.native_chunk_index.hybrid_search( + query_text=query, + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + else: + results = self.native_chunk_index.search( + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_chunking_and_rerank.py::test_seekdb_repository_prefers_native_hybrid_search -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_chunk_index.py backend/infrastructure/repositories/seekdb_repository.py tests/test_chunking_and_rerank.py +git commit -m "feat(seekdb): use native hybrid retrieval" +``` + +--- + +### Task 4: Expose Actual Backend Status + +**Files:** +- Modify: `backend/infrastructure/vector_stores/seekdb_vector_store.py` +- Modify: `backend/api/schemas/config.py` +- Modify: `backend/api/routes/config.py` +- Modify: `backend/main.py` +- Test: `tests/test_runtime_config_api.py` + +- [ ] **Step 1: Write failing API status tests** + +Append to `tests/test_runtime_config_api.py`: + +```python +def test_runtime_config_exposes_actual_vector_backend(client): + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert "actual_vector_backend" in storage + assert storage["configured_vector_store_type"] == "seekdb" + + +def test_health_exposes_actual_vector_backend(client): + response = client.get("/health") + + assert response.status_code == 200 + assert "storage" in response.json() + assert "actual_vector_backend" in response.json()["storage"] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_runtime_config_api.py::test_runtime_config_exposes_actual_vector_backend tests/test_runtime_config_api.py::test_health_exposes_actual_vector_backend -q +``` + +Expected: FAIL because the response does not include actual backend status. + +- [ ] **Step 3: Add status helpers** + +Modify `backend/infrastructure/vector_stores/seekdb_vector_store.py`: + +```python + def storage_status(self) -> dict[str, Any]: + if hasattr(self.repository, "storage_status"): + return self.repository.storage_status() + return {"vector_backend": "unknown", "native_available": False} + + async def get_stats(self) -> dict[str, Any]: + sources = await self.repository.list_sources() + chunk_count = sum(source.chunk_count for source in sources) + status = self.storage_status() + return { + "total_documents": len(sources), + "total_chunks": chunk_count, + "backend": status["vector_backend"], + "storage": status, + } +``` + +- [ ] **Step 4: Add config schema/status fields** + +Modify `backend/api/schemas/config.py` so `RuntimeConfigResponse.storage` allows untyped status values: + +```python +class RuntimeConfigResponse(BaseModel): + models: dict[str, PublicModelProfile] + chunking: dict[str, Any] + storage: dict[str, Any] + message: str = "" +``` + +Modify `_response` in `backend/api/routes/config.py`: + +```python + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() if hasattr(vector_store, "get_stats") else {} +``` + +Because `_response` is currently synchronous, change it to: + +```python +async def _response(message: str = "") -> RuntimeConfigResponse: + settings = get_settings() + models = settings.api.models + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + storage_status = stats.get("storage", {}) + return RuntimeConfigResponse( + models={name: _public_profile(getattr(models, name)) for name in ModelProfiles.model_fields}, + chunking={ + "provider": settings.chunking.provider, + "tokenizer": settings.chunking.tokenizer, + "chunk_size": settings.chunk_size, + "chunk_overlap": settings.chunk_overlap, + }, + storage={ + "configured_vector_store_type": settings.vector_store_type, + "seekdb_path": settings.seekdb_path, + "seekdb_allow_sqlite_fallback": settings.seekdb_allow_sqlite_fallback, + "actual_vector_backend": storage_status.get("vector_backend", "unknown"), + "native_available": storage_status.get("native_available", False), + }, + message=message, + ) +``` + +Then update route returns: + +```python +@router.get("", response_model=RuntimeConfigResponse) +async def get_runtime_config() -> RuntimeConfigResponse: + return await _response() + + +@router.post("", response_model=RuntimeConfigResponse) +async def update_runtime_config(request: RuntimeConfigUpdate) -> RuntimeConfigResponse: + profile_data = { + name: profile.model_dump(exclude_none=True) + for name, profile in request.models.items() + } + update_runtime_model_profiles(profile_data) + DependencyContainer.reset_runtime_caches() + return await _response("Runtime model configuration updated.") +``` + +Modify `backend/main.py`: + +```python +@app.get("/health") +async def health_check(): + from .dependencies import get_vector_store + + vector_store = get_vector_store() + stats = await vector_store.get_stats() + return { + "status": "healthy", + "version": settings.app_version, + "storage": { + "actual_vector_backend": stats.get("storage", {}).get("vector_backend", stats.get("backend")), + "native_available": stats.get("storage", {}).get("native_available", False), + }, + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_runtime_config_api.py::test_runtime_config_exposes_actual_vector_backend tests/test_runtime_config_api.py::test_health_exposes_actual_vector_backend -q +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_vector_store.py backend/api/schemas/config.py backend/api/routes/config.py backend/main.py tests/test_runtime_config_api.py +git commit -m "feat(config): expose actual vector backend status" +``` + +--- + +### Task 5: Backfill Existing SQLite Chunk Embeddings Into SeekDB + +**Files:** +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_source_knowledge_base.py` + +- [ ] **Step 1: Write failing test for lazy backfill** + +Append to `tests/test_source_knowledge_base.py`: + +```python +@pytest.mark.asyncio +async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-backfill", kind=SourceKind.TEXT, title="Backfill") + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-backfill", + source_id=source.id, + content="backfill content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + await repo.backfill_native_chunks() + + assert native_index.upserts[0][0] == source.id + assert native_index.upserts[0][1][0].id == "chunk-backfill" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_backfills_existing_sqlite_chunks_to_native_index -q +``` + +Expected: FAIL because `backfill_native_chunks` does not exist. + +- [ ] **Step 3: Implement backfill** + +Add to `backend/infrastructure/repositories/seekdb_repository.py`: + +```python + async def backfill_native_chunks(self) -> int: + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + return 0 + raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + + sources = await self.list_sources() + written = 0 + for source in sources: + chunks = await self.get_chunks(source.id) + chunks_with_embeddings = [chunk for chunk in chunks if chunk.embedding] + if chunks_with_embeddings: + self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + written += len(chunks_with_embeddings) + return written +``` + +Call it in `DependencyContainer.get_knowledge_repository` is not safe because it is async. Instead call it opportunistically from `SourceService` only after new writes and expose a manual helper for migration. Add a script in Task 6 for explicit migration. + +- [ ] **Step 4: Run test to verify it passes** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_backfills_existing_sqlite_chunks_to_native_index -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/repositories/seekdb_repository.py tests/test_source_knowledge_base.py +git commit -m "feat(seekdb): backfill existing chunks into native index" +``` + +--- + +### Task 6: Add a Local Verification Script for the Warning Regression + +**Files:** +- Create: `scripts/verify_seekdb_native.py` +- Test: manual command in this task + +- [ ] **Step 1: Create verification script** + +Create `scripts/verify_seekdb_native.py`: + +```python +from __future__ import annotations + +import asyncio +import tempfile +from pathlib import Path + +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind +from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = SeekDBRepository(Path(tmp) / "knowledge.db", allow_sqlite_vector_fallback=False) + source = KnowledgeSource(id="verify-source", kind=SourceKind.TEXT, title="Verify") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="verify-chunk", + source_id=source.id, + content="SeekDB native vector retrieval verification", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + results = await repo.search_chunks( + "SeekDB native retrieval", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + assert results, "expected at least one native SeekDB result" + assert results[0]["chunk"].id == "verify-chunk" + assert repo.storage_status()["vector_backend"] == "seekdb" + await repo.close() + print("SeekDB native vector verification passed") + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +- [ ] **Step 2: Run script** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 scripts/verify_seekdb_native.py +``` + +Expected: + +```text +SeekDB native vector verification passed +``` + +There must be no log line containing: + +```text +Skipping optional pyseekdb chunk mirror +``` + +- [ ] **Step 3: Commit** + +```bash +git add scripts/verify_seekdb_native.py +git commit -m "test(seekdb): add native vector verification script" +``` + +--- + +### Task 7: Full Regression and Real App Smoke + +**Files:** +- No required source edits if all prior tasks pass. +- Use existing local `config.yaml`; do not commit it. + +- [ ] **Step 1: Run backend unit tests** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest -q +``` + +Expected: all tests pass. + +- [ ] **Step 2: Run backend compile check** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m compileall -q backend +``` + +Expected: exit code `0`. + +- [ ] **Step 3: Run frontend build and tests** + +Run: + +```bash +cd frontend +npm test -- --reporter=dot +npm run build +npm run lint +npm run test:e2e -- --project=chromium +``` + +Expected: +- Vitest passes. +- Build passes. +- Lint has no errors. +- Playwright passes. + +- [ ] **Step 4: Start services** + +Run backend: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m uvicorn backend.main:app --host 127.0.0.1 --port 8000 +``` + +Run frontend in another shell: + +```bash +cd frontend +npm run dev -- --host 127.0.0.1 --port 5173 +``` + +Expected: +- Backend logs show startup without SeekDB mirror warning. +- Frontend is available at `http://127.0.0.1:5173`. + +- [ ] **Step 5: Verify health and config report native SeekDB** + +Run: + +```bash +curl -s http://127.0.0.1:8000/health +curl -s http://127.0.0.1:8000/api/config +``` + +Expected JSON contains: + +```json +{ + "actual_vector_backend": "seekdb", + "native_available": true +} +``` + +- [ ] **Step 6: Browser smoke upload and RAG** + +Use Playwright or the browser manually: + +1. Open `http://127.0.0.1:5173`. +2. Upload `/Users/lzj/proj/notebook/NotebookLM-Lite/doc/L9.md`. +3. Ask: `理想 L9 的智能座舱有哪些核心能力？` +4. Confirm the answer references uploaded content and sources. +5. Check backend logs for absence of: + +```text +Skipping optional pyseekdb chunk mirror +``` + +- [ ] **Step 7: Commit if a smoke-only doc update is added** + +If this task adds a small note to README or docs, commit: + +```bash +git add README.md README_zn.md +git commit -m "docs: clarify SeekDB native vector retrieval" +``` + +If no doc edit is needed, skip this commit. + +--- + +## Self-Review + +**Spec coverage:** The plan makes SeekDB native chunk vector storage and retrieval primary, keeps SQLite as explicit fallback only, exposes actual backend status, and directly targets the observed `Skipping optional pyseekdb chunk mirror` warning by deleting the optional mirror path. + +**Placeholder scan:** No implementation step relies on TBD, TODO, or undefined behavior. Each code-changing task includes concrete test code, implementation code, commands, and expected results. + +**Type consistency:** The new `SeekDBChunkIndex` methods are used consistently: +- `upsert_source_chunks(source_id, chunks)` +- `delete_source_chunks(source_id, chunk_ids)` +- `search(query_embedding, source_ids, top_k)` +- `hybrid_search(query_text, query_embedding, source_ids, top_k)` +- `status()` + +**Risk boundary:** This plan does not migrate artifacts, notes, jobs, or slide decks from SQLite into SeekDB. That is a separate broader database migration. This plan fixes the vector/RAG path that currently claims SeekDB while using SQLite. diff --git a/scripts/verify_seekdb_native.py b/scripts/verify_seekdb_native.py new file mode 100644 index 0000000..0fb6432 --- /dev/null +++ b/scripts/verify_seekdb_native.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind +from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + warning_messages: list[str] = [] + + class WarningCapture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + if record.levelno >= logging.WARNING: + warning_messages.append(record.getMessage()) + + warning_capture = WarningCapture() + root_logger = logging.getLogger() + root_logger.addHandler(warning_capture) + repo = SeekDBRepository(Path(tmp) / "knowledge.db", allow_sqlite_vector_fallback=False) + try: + source = KnowledgeSource(id="verify-source", kind=SourceKind.TEXT, title="Verify") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="verify-chunk", + source_id=source.id, + content="SeekDB native vector retrieval verification", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + results = await repo.search_chunks( + "SeekDB native retrieval", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + if not results: + raise RuntimeError("expected at least one native SeekDB result") + if results[0]["chunk"].id != "verify-chunk": + raise RuntimeError("native SeekDB returned the wrong chunk") + if results[0].get("backend") != "seekdb": + raise RuntimeError("retrieval did not use the native SeekDB backend") + text_results = await repo.search_chunks( + "native vector retrieval verification", + source_ids=[source.id], + top_k=1, + ) + if not text_results or text_results[0].get("backend") != "seekdb": + raise RuntimeError("full-text retrieval did not use the native SeekDB backend") + if text_results[0]["chunk"].id != "verify-chunk": + raise RuntimeError("native SeekDB full-text search returned the wrong chunk") + if repo.storage_status()["vector_backend"] != "seekdb": + raise RuntimeError("repository did not report the native SeekDB backend") + loaded_chunks = await repo.get_chunks(source.id) + if len(loaded_chunks) != 1 or loaded_chunks[0].embedding != [0.1, 0.2, 0.3]: + raise RuntimeError("repository did not read the stored chunk back from SeekDB") + sqlite_row = repo._conn.execute( + "SELECT embedding, payload, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + if sqlite_row is None: + raise RuntimeError("SQLite chunk metadata row is missing") + if sqlite_row["embedding"] is not None or '"embedding":null' not in sqlite_row["payload"]: + raise RuntimeError("SQLite retained a chunk vector in strict SeekDB mode") + if sqlite_row["vector_state"] != "seekdb": + raise RuntimeError("SQLite chunk metadata does not mark SeekDB as vector authority") + if any("Skipping optional pyseekdb chunk mirror" in message for message in warning_messages): + raise RuntimeError("legacy optional pyseekdb mirror warning was emitted") + finally: + await repo.close() + root_logger.removeHandler(warning_capture) + print("SeekDB native vector verification passed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conftest.py b/tests/conftest.py index a85c033..2457278 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ def sample_config_file(tmp_path: Path) -> Path: config_path = tmp_path / "config.yaml" config_path.write_text( - """ + f""" api: models: text_model: @@ -49,7 +49,7 @@ def sample_config_file(tmp_path: Path) -> Path: adapter: raw_chat_multimodal storage: vector_store_type: seekdb - seekdb_path: ./data/test_seekdb.db + seekdb_path: {tmp_path / "data" / "test_seekdb.db"} documents: chunk_size: 512 chunk_overlap: 64 diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index 4646e1c..5484d89 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -10,6 +10,69 @@ from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore +def sqlite_fallback_repo(path: Path) -> SeekDBRepository: + return SeekDBRepository(path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + + +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + self.chunks_by_source = {} + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + return list(self.chunks_by_source.get(source_id, [])) + + def search(self, query_embedding, source_ids, top_k): + return [] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + +class FailingNativeIndex(RecordingNativeIndex): + def upsert_source_chunks(self, source_id, chunks): + super().upsert_source_chunks(source_id, chunks) + raise RuntimeError("native write failed") + + +class HybridRecordingIndex: + def __init__(self) -> None: + self.calls = [] + self.upserts = [] + self.chunks_by_source = {} + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + return list(self.chunks_by_source.get(source_id, [])) + + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid_chunk", + source_id=source_ids[0], + content="hybrid TLS result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.88, + "backend": "seekdb", + } + ] + + def search(self, query_embedding, source_ids, top_k): + raise AssertionError("hybrid_search should be preferred when query text is available") + + def test_chonkie_chunker_returns_offsets_and_counts(): pytest.importorskip("chonkie") service = ChunkingService( @@ -37,7 +100,7 @@ async def rerank(self, query, results, top_k=5): item["score"] = 10 - index return reranked - repo = SeekDBRepository(tmp_path / "rerank.db") + repo = sqlite_fallback_repo(tmp_path / "rerank.db") source = KnowledgeSource( id="source-1", kind=SourceKind.TEXT, @@ -75,7 +138,7 @@ async def rerank(self, query, results, top_k=5): @pytest.mark.asyncio async def test_seekdb_repository_uses_bm25_for_term_overlap_without_exact_phrase(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "bm25.db") + repo = sqlite_fallback_repo(tmp_path / "bm25.db") source = KnowledgeSource( id="source-bm25", kind=SourceKind.TEXT, @@ -113,3 +176,150 @@ async def test_seekdb_repository_uses_bm25_for_term_overlap_without_exact_phrase assert results assert results[0]["chunk"].id == "chunk-http" assert results[0]["score"] > 0 + + +@pytest.mark.asyncio +async def test_seekdb_repository_prefers_native_hybrid_search(tmp_path: Path): + native_index = HybridRecordingIndex() + repo = SeekDBRepository( + tmp_path / "hybrid.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid", kind=SourceKind.TEXT, title="Hybrid") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="seed-hybrid", + source_id=source.id, + content="seed hybrid content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.calls == [("TLS handshake", [0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "hybrid_chunk" + + +@pytest.mark.asyncio +async def test_seekdb_repository_reranks_native_hybrid_results(tmp_path: Path): + class TwoResultHybridIndex(HybridRecordingIndex): + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid-a", + source_id=source_ids[0], + content="first hybrid result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.9, + "backend": "seekdb", + }, + { + "chunk": KnowledgeChunk( + id="hybrid-b", + source_id=source_ids[0], + content="second hybrid result", + chunk_index=1, + embedding=[0.2, 0.3, 0.4], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.8, + "backend": "seekdb", + }, + ] + + class ReverseReranker: + async def rerank(self, query, results, top_k=5): + return list(reversed(results))[:top_k] + + native_index = TwoResultHybridIndex() + repo = SeekDBRepository( + tmp_path / "hybrid-rerank.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid-rerank", kind=SourceKind.TEXT, title="Hybrid rerank") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="seed-hybrid-rerank", + source_id=source.id, + content="seed hybrid rerank content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=2, + query_embedding=[0.1, 0.2, 0.3], + rerank_provider=ReverseReranker(), + ) + + assert [item["chunk"].id for item in results] == ["hybrid-b", "hybrid-a"] + + +@pytest.mark.asyncio +async def test_seekdb_vector_store_add_chunks_does_not_leave_ready_source_on_strict_native_failure( + tmp_path: Path, +): + repo = SeekDBRepository( + tmp_path / "strict-add.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + store = SeekDBVectorStore(repository=repo) + + with pytest.raises(RuntimeError, match="embeddings|native SeekDB"): + await store.add_chunks( + "doc-strict", + [{"content": "strict native content", "metadata": {"filename": "strict.txt"}}], + ) + + assert await repo.get_source("doc-strict") is None + assert await repo.get_chunks("doc-strict") == [] + + +@pytest.mark.asyncio +async def test_seekdb_vector_store_add_chunks_rolls_back_new_source_when_native_save_fails( + tmp_path: Path, +): + repo = SeekDBRepository( + tmp_path / "strict-valid-embeddings.db", + native_chunk_index=FailingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + store = SeekDBVectorStore(repository=repo) + + with pytest.raises(RuntimeError, match="native write failed"): + await store.add_chunks( + "doc-native-fail", + [{"content": "strict native content", "metadata": {"filename": "strict.txt"}}], + embeddings=[[0.1, 0.2, 0.3]], + ) + + assert await repo.get_source("doc-native-fail") is None + assert await repo.get_chunks("doc-native-fail") == [] diff --git a/tests/test_config_settings.py b/tests/test_config_settings.py index 1adedfd..ab44dad 100644 --- a/tests/test_config_settings.py +++ b/tests/test_config_settings.py @@ -15,7 +15,7 @@ def test_loads_yaml_model_profiles_without_using_local_config(sample_config_file assert settings.api.models.audio_model.response_format == "mp3" assert settings.api.models.audio_model.stream is True assert settings.vector_store_type == "seekdb" - assert settings.seekdb_path == "./data/test_seekdb.db" + assert settings.seekdb_path.endswith("/data/test_seekdb.db") assert settings.chunk_size == 512 assert settings.chunk_overlap == 64 assert settings.chunking.provider == "chonkie" @@ -24,6 +24,20 @@ def test_loads_yaml_model_profiles_without_using_local_config(sample_config_file assert settings.output_dir == "./output-test" +def test_seekdb_native_vector_search_is_default(monkeypatch, sample_config_file): + from backend.config import get_settings + + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + get_settings.cache_clear() + try: + settings = get_settings() + finally: + get_settings.cache_clear() + + assert settings.vector_store_type == "seekdb" + assert settings.seekdb_allow_sqlite_fallback is False + + def test_default_settings_do_not_require_local_secrets(monkeypatch): from backend.config import get_settings diff --git a/tests/test_rag_chat.py b/tests/test_rag_chat.py index 91b50ed..31fe767 100644 --- a/tests/test_rag_chat.py +++ b/tests/test_rag_chat.py @@ -15,6 +15,10 @@ from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore +def sqlite_fallback_repo(path) -> SeekDBRepository: + return SeekDBRepository(path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + + class EchoLLM: async def generate_with_context(self, query, context, system_prompt=None, **kwargs): return f"answer: {query} :: {context[0]}" @@ -60,7 +64,7 @@ async def stream_generate_with_context(self, query, context, system_prompt=None, @pytest.mark.asyncio async def test_rag_retrieval_is_restricted_to_selected_sources(tmp_path): - repo = SeekDBRepository(tmp_path / "rag.db") + repo = sqlite_fallback_repo(tmp_path / "rag.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) first = await service.create_text_source("First", "Alpha selected material.") second = await service.create_text_source("Second", "Alpha forbidden material.") @@ -76,7 +80,7 @@ async def test_rag_retrieval_is_restricted_to_selected_sources(tmp_path): def test_chat_api_rejects_empty_sources_and_can_save_answer(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) @@ -92,7 +96,7 @@ def test_chat_api_rejects_empty_sources_and_can_save_answer(tmp_path): def test_chat_api_returns_citations_and_save_answer_as_source(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-save.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-save.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -118,7 +122,7 @@ def test_chat_api_returns_citations_and_save_answer_as_source(tmp_path): def test_chat_api_streams_answer_deltas_and_final_citations(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-stream.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-stream.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -145,7 +149,7 @@ def test_chat_api_streams_answer_deltas_and_final_citations(tmp_path): def test_chat_api_stream_falls_back_to_original_query_when_contextualized_retrieval_misses(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-stream-fallback.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-stream-fallback.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -176,7 +180,7 @@ def test_chat_api_stream_falls_back_to_original_query_when_contextualized_retrie @pytest.mark.asyncio async def test_rag_no_results_and_follow_up_stay_source_scoped(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-follow-up.db") + repo = sqlite_fallback_repo(tmp_path / "rag-follow-up.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) source = await service.create_text_source("Follow", "Beta follow up material.") await service.create_text_source("Other", "Beta forbidden follow up material.") @@ -199,7 +203,7 @@ async def test_rag_no_results_and_follow_up_stay_source_scoped(tmp_path): @pytest.mark.asyncio async def test_rag_multi_turn_uses_contextualized_query_only_for_retrieval(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-follow-up-original.db") + repo = sqlite_fallback_repo(tmp_path / "rag-follow-up-original.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) source = await service.create_text_source("Follow", "Alpha standalone retrieval material.") store = SeekDBVectorStore(repo) @@ -220,7 +224,7 @@ async def test_rag_multi_turn_uses_contextualized_query_only_for_retrieval(tmp_p @pytest.mark.asyncio async def test_rag_context_labels_sources_for_normal_user_file_comparison(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-source-labels.db") + repo = sqlite_fallback_repo(tmp_path / "rag-source-labels.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -247,7 +251,7 @@ async def test_rag_context_labels_sources_for_normal_user_file_comparison(tmp_pa @pytest.mark.asyncio async def test_rag_file_overview_includes_every_selected_source_even_when_bm25_matches_one_source(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-overview-balanced.db") + repo = sqlite_fallback_repo(tmp_path / "rag-overview-balanced.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -274,7 +278,7 @@ async def test_rag_file_overview_includes_every_selected_source_even_when_bm25_m @pytest.mark.asyncio async def test_rag_file_overview_includes_every_selected_source_when_top_k_is_smaller(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-overview-all-sources.db") + repo = sqlite_fallback_repo(tmp_path / "rag-overview-all-sources.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) sources = [ await service.create_text_source(f"Source {index}", f"Source {index} overview material.") @@ -297,7 +301,7 @@ async def test_rag_file_overview_includes_every_selected_source_when_top_k_is_sm @pytest.mark.asyncio async def test_rag_balances_context_across_selected_sources_for_cross_source_questions(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-cross-source-balanced.db") + repo = sqlite_fallback_repo(tmp_path / "rag-cross-source-balanced.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -324,7 +328,7 @@ async def test_rag_balances_context_across_selected_sources_for_cross_source_que @pytest.mark.asyncio async def test_rag_targeted_summary_uses_retrieval_instead_of_first_chunks(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-targeted-summary.db") + repo = sqlite_fallback_repo(tmp_path / "rag-targeted-summary.db") source = KnowledgeSource( kind=SourceKind.FILE, title="HTTPS.pdf", diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index 047f8fc..ecb11aa 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -4,12 +4,49 @@ from fastapi.testclient import TestClient from backend.api.routes.config import router as config_router -from backend.config import _RUNTIME_MODEL_OVERRIDES, get_settings +from backend.config import Settings, _RUNTIME_MODEL_OVERRIDES, get_settings from backend.dependencies import DependencyContainer +class FakeVectorStore: + def __init__( + self, + status: dict, + total_chunks: int = 0, + indexed_vector_chunks: int | None = None, + ) -> None: + self.status = status + self.total_chunks = total_chunks + self.indexed_vector_chunks = ( + total_chunks if indexed_vector_chunks is None else indexed_vector_chunks + ) + + async def get_stats(self) -> dict: + return { + "total_documents": 0, + "total_chunks": self.total_chunks, + "indexed_vector_chunks": self.indexed_vector_chunks, + "backend": self.status["vector_backend"], + "storage": self.status, + } + + +class FailingStatsVectorStore: + async def get_stats(self) -> dict: + raise RuntimeError("storage unavailable") + + def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample_config_file): monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True} + ) + ), + ) _RUNTIME_MODEL_OVERRIDES.clear() get_settings.cache_clear() DependencyContainer.reset_runtime_caches() @@ -49,3 +86,262 @@ def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample _RUNTIME_MODEL_OVERRIDES.clear() get_settings.cache_clear() DependencyContainer.reset_runtime_caches() + + +def test_runtime_cache_reset_recreates_knowledge_repository_with_new_storage_settings(tmp_path): + DependencyContainer.reset_runtime_caches() + first_settings = Settings( + seekdb_path=str(tmp_path / "first.db"), + seekdb_allow_sqlite_fallback=True, + ) + first_repo = DependencyContainer.get_knowledge_repository(settings=first_settings) + + DependencyContainer.reset_runtime_caches() + second_settings = Settings( + seekdb_path=str(tmp_path / "second.db"), + seekdb_allow_sqlite_fallback=False, + ) + second_repo = DependencyContainer.get_knowledge_repository(settings=second_settings) + + try: + assert second_repo is not first_repo + assert second_repo.db_path == tmp_path / "second.db" + assert second_repo.allow_sqlite_vector_fallback is False + finally: + DependencyContainer.reset_runtime_caches() + + +def test_runtime_cache_reset_closes_cached_repository(): + class ClosableRepository: + def __init__(self) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + repository = ClosableRepository() + DependencyContainer._knowledge_repository = repository + + DependencyContainer.reset_runtime_caches() + + assert repository.close_calls == 1 + assert DependencyContainer._knowledge_repository is None + + +def test_runtime_cache_reset_can_preserve_open_repository(): + class ClosableRepository: + def __init__(self) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + repository = ClosableRepository() + DependencyContainer._knowledge_repository = repository + DependencyContainer._vector_store = object() + DependencyContainer._slide_deck_service = object() + + DependencyContainer.reset_runtime_caches(preserve_repository=True) + + assert repository.close_calls == 0 + assert DependencyContainer._knowledge_repository is repository + assert DependencyContainer._vector_store is None + assert DependencyContainer._slide_deck_service is None + DependencyContainer.reset_runtime_caches() + + +def test_force_new_repository_closes_old_repository_and_invalidates_dependents(monkeypatch): + class ClosableRepository: + def __init__(self, *_args, **_kwargs) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + old_repository = ClosableRepository() + DependencyContainer._knowledge_repository = old_repository + DependencyContainer._vector_store = object() + DependencyContainer._slide_deck_service = object() + monkeypatch.setattr( + "backend.infrastructure.repositories.seekdb_repository.SeekDBRepository", + ClosableRepository, + ) + + new_repository = DependencyContainer.get_knowledge_repository( + settings=Settings(seekdb_allow_sqlite_fallback=True), + force_new=True, + ) + + assert new_repository is not old_repository + assert old_repository.close_calls == 1 + assert DependencyContainer._vector_store is None + assert DependencyContainer._slide_deck_service is None + DependencyContainer.reset_runtime_caches() + + +def test_runtime_config_exposes_actual_vector_backend(monkeypatch, sample_config_file): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True} + ) + ), + ) + get_settings.cache_clear() + DependencyContainer.reset_runtime_caches() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert storage["configured_vector_store_type"] == "seekdb" + assert storage["vector_store_type"] == "seekdb" + assert storage["actual_vector_backend"] == "seekdb" + assert storage["native_available"] is True + + get_settings.cache_clear() + DependencyContainer.reset_runtime_caches() + + +def test_runtime_config_exposes_sqlite_fallback_status(monkeypatch, sample_config_file): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "sqlite_fallback", "native_available": False} + ) + ), + ) + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert storage["configured_vector_store_type"] == "seekdb" + assert storage["actual_vector_backend"] == "sqlite_fallback" + assert storage["native_available"] is False + + +def test_runtime_config_rejects_embedding_identity_change_with_indexed_chunks( + monkeypatch, + sample_config_file, +): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True}, + total_chunks=3, + ) + ), + ) + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.post( + "/api/config", + json={"models": {"embedding_model": {"model": "different-embedding-model"}}}, + ) + + assert response.status_code == 409 + assert "re-index" in response.json()["detail"] + assert get_settings().api.models.embedding_model.model == "test-embedding-model" + + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + +def test_runtime_config_allows_embedding_identity_change_without_indexed_vectors( + monkeypatch, + sample_config_file, +): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True}, + total_chunks=3, + indexed_vector_chunks=0, + ) + ), + ) + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + response = TestClient(app).post( + "/api/config", + json={"models": {"embedding_model": {"model": "different-embedding-model"}}}, + ) + + assert response.status_code == 200 + assert response.json()["models"]["embedding_model"]["model"] == "different-embedding-model" + + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + +def test_health_exposes_actual_vector_backend(monkeypatch): + from backend import main as backend_main + + def fake_get_vector_store(): + return FakeVectorStore({"vector_backend": "unavailable", "native_available": False}) + + monkeypatch.setattr("backend.dependencies.get_vector_store", fake_get_vector_store) + client = TestClient(backend_main.app) + + response = client.get("/health") + + assert response.status_code == 503 + assert response.json()["status"] == "unhealthy" + storage = response.json()["storage"] + assert storage["actual_vector_backend"] == "unavailable" + assert storage["native_available"] is False + + +def test_health_returns_structured_503_when_storage_stats_fail(monkeypatch): + from backend import main as backend_main + + monkeypatch.setattr( + "backend.dependencies.get_vector_store", + lambda: FailingStatsVectorStore(), + ) + + response = TestClient(backend_main.app).get("/health") + + assert response.status_code == 503 + assert response.json() == { + "status": "unhealthy", + "version": get_settings().app_version, + "storage": { + "actual_vector_backend": "unavailable", + "native_available": False, + "embedding_configured": bool( + get_settings().api.models.embedding_model.model + and get_settings().api.models.embedding_model.api_key + ), + "error": "storage unavailable", + }, + } diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py new file mode 100644 index 0000000..29b4232 --- /dev/null +++ b/tests/test_seekdb_chunk_index.py @@ -0,0 +1,577 @@ +import sys +import types +from pathlib import Path + +import pytest + +from backend.domain.source import KnowledgeChunk +from backend.infrastructure.vector_stores.seekdb_chunk_index import ( + SeekDBChunkIndex, + SeekDBUnavailableError, +) + + +class FakeHNSWConfiguration: + def __init__(self, dimension: int, distance: str = "cosine") -> None: + self.dimension = dimension + self.distance = distance + + +class FakeCollection: + def __init__(self, name: str, dimension: int) -> None: + self.name = name + self.dimension = dimension + self.upsert_calls = [] + self.delete_calls = [] + self.query_calls = [] + self.hybrid_calls = [] + self.events = [] + self.refresh_calls = 0 + self.rows = {} + self.query_error = None + self.hybrid_error = None + + def upsert(self, ids, documents, metadatas, embeddings): + self.events.append({"op": "upsert", "ids": ids}) + self.upsert_calls.append( + { + "ids": ids, + "documents": documents, + "metadatas": metadatas, + "embeddings": embeddings, + } + ) + for index, chunk_id in enumerate(ids): + self.rows[chunk_id] = { + "document": documents[index], + "metadata": metadatas[index], + "embedding": embeddings[index], + } + + def delete(self, ids=None, **kwargs): + self.events.append({"op": "delete", "ids": ids, **kwargs}) + self.delete_calls.append({"ids": ids, **kwargs}) + if ids: + for chunk_id in ids: + self.rows.pop(chunk_id, None) + where = kwargs.get("where") + if where: + for chunk_id, row in list(self.rows.items()): + if all(row["metadata"].get(key) == value for key, value in where.items()): + self.rows.pop(chunk_id, None) + + def query(self, query_embeddings, n_results, include): + if self.query_error is not None: + raise self.query_error + self.query_calls.append( + { + "query_embeddings": query_embeddings, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[0.0 + index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + + def hybrid_search(self, query, knn, rank, n_results, include): + if self.hybrid_error is not None: + raise self.hybrid_error + self.hybrid_calls.append( + { + "query": query, + "knn": knn, + "rank": rank, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[10.0 - index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + + def count(self): + return len(self.rows) + + def get(self, include=None, **kwargs): + ids = list(self.rows) + result = {"ids": ids} + if include and "documents" in include: + result["documents"] = [self.rows[chunk_id]["document"] for chunk_id in ids] + if include and "metadatas" in include: + result["metadatas"] = [self.rows[chunk_id]["metadata"] for chunk_id in ids] + return result + + def refresh_index(self): + self.refresh_calls += 1 + + +class FakeClient: + instances = [] + + def __init__(self, path: str) -> None: + self.path = path + self.collections = {} + self.created = [] + self.exit_calls = 0 + self.list_calls = 0 + FakeClient.instances.append(self) + + def __exit__(self, exc_type, exc_value, traceback): + self.exit_calls += 1 + + def get_or_create_collection(self, name, configuration=None, embedding_function=None): + if embedding_function is not None: + raise AssertionError("explicit LiteLLM embeddings require embedding_function=None") + if configuration is None: + raise AssertionError("native SeekDB collection must declare vector dimension") + if name not in self.collections: + self.created.append((name, configuration.dimension)) + self.collections[name] = FakeCollection(name, configuration.dimension) + return self.collections[name] + + def get_collection(self, name, embedding_function=None): + if name not in self.collections: + raise KeyError(name) + return self.collections[name] + + def list_collections(self): + self.list_calls += 1 + return list(self.collections.values()) + + +@pytest.fixture(autouse=True) +def fake_pyseekdb(monkeypatch): + FakeClient.instances.clear() + module = types.SimpleNamespace(Client=FakeClient, HNSWConfiguration=FakeHNSWConfiguration) + monkeypatch.setitem(sys.modules, "pyseekdb", module) + yield + + +def chunk( + chunk_id: str, + source_id: str, + embedding: list[float], + metadata: dict | None = None, +) -> KnowledgeChunk: + return KnowledgeChunk( + id=chunk_id, + source_id=source_id, + content=f"{chunk_id} content", + chunk_index=0, + embedding=embedding, + metadata=metadata if metadata is not None else {"source_id": source_id}, + ) + + +def test_upsert_creates_dimensioned_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + client = FakeClient.instances[0] + assert client.created == [(index.collection_name("source-a"), 3)] + collection = client.collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["ids"] == ["chunk-a"] + assert collection.upsert_calls[0]["embeddings"] == [[0.1, 0.2, 0.3]] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert "payload" in collection.upsert_calls[0]["metadatas"][0] + assert collection.refresh_calls == 1 + + +def test_close_releases_native_client_once(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + client = FakeClient.instances[0] + + index.close() + index.close() + + assert client.exit_calls == 1 + + +def test_probe_eagerly_opens_native_client(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + client = FakeClient.instances[0] + + index.probe() + + assert client.list_calls == 1 + + +def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("source-b", [chunk("chunk-b", "source-b", [0.9, 0.1, 0.1])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a", "source-b"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert all(item["backend"] == "seekdb" for item in results) + client = FakeClient.instances[0] + source_a_query = client.collections[index.collection_name("source-a")].query_calls[0] + source_b_query = client.collections[index.collection_name("source-b")].query_calls[0] + assert source_a_query["query_embeddings"] == [[0.1, 0.2, 0.3]] + assert source_a_query["n_results"] == 2 + assert source_a_query["include"] == ["documents", "metadatas", "distances"] + assert source_b_query["query_embeddings"] == [[0.1, 0.2, 0.3]] + assert source_b_query["n_results"] == 2 + assert source_b_query["include"] == ["documents", "metadatas", "distances"] + + +def test_get_source_chunks_reads_chunk_payloads_from_seekdb(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + stored = chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]) + index.upsert_source_chunks("source-a", [stored]) + + loaded = index.get_source_chunks("source-a") + + assert [item.id for item in loaded] == ["chunk-a"] + assert loaded[0].embedding == [0.1, 0.2, 0.3] + assert index.get_source_chunks("missing-source") == [] + + +def test_hybrid_search_calls_seekdb_hybrid_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + assert results[0]["backend"] == "seekdb" + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls == [ + { + "query": {"where_document": {"$contains": "TLS handshake"}, "n_results": 1}, + "knn": {"query_embeddings": [0.1, 0.2, 0.3], "n_results": 1}, + "rank": {"rrf": {}}, + "n_results": 1, + "include": ["documents", "metadatas", "distances"], + } + ] + + +def test_text_search_uses_seekdb_fulltext_without_knn(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.text_search(query_text="TLS handshake", source_ids=["source-a"], top_k=1) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + assert results[0]["backend"] == "seekdb" + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls[-1] == { + "query": {"where_document": {"$contains": "TLS handshake"}, "n_results": 1}, + "knn": None, + "rank": None, + "n_results": 1, + "include": ["documents", "metadatas", "distances"], + } + + +def test_hybrid_search_preserves_native_relevance_score_order(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]), + chunk("chunk-b", "source-a", [0.9, 0.1, 0.1]), + ], + ) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert [item["score"] for item in results] == [10.0, 9.0] + + +def test_hybrid_search_raises_collection_hybrid_errors(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.hybrid_error = RuntimeError("hybrid exploded") + + with pytest.raises(RuntimeError, match="hybrid exploded"): + index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_hybrid_search_falls_back_to_vector_search_when_collection_lacks_hybrid_api( + tmp_path: Path, + monkeypatch, +): + monkeypatch.delattr(FakeCollection, "hybrid_search") + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.query_calls[0]["query_embeddings"] == [[0.1, 0.2, 0.3]] + + +def test_hybrid_search_falls_back_to_vector_search_for_blank_query_text(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text=" ", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls == [] + assert collection.query_calls[0]["query_embeddings"] == [[0.1, 0.2, 0.3]] + + +def test_hybrid_search_raises_malformed_payload_results(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.rows["chunk-a"]["metadata"]["payload"] = "{not-json" + + with pytest.raises(ValueError, match="Malformed SeekDB chunk result payload"): + index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_search_raises_collection_query_errors(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.query_error = RuntimeError("query exploded") + + with pytest.raises(RuntimeError, match="query exploded"): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_search_raises_malformed_payload_results(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.rows["chunk-a"]["metadata"]["payload"] = "{not-json" + + with pytest.raises(ValueError, match="Malformed SeekDB chunk result payload"): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_search_raises_missing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + with pytest.raises(KeyError): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-missing"], + top_k=1, + ) + + +def test_delete_source_chunks_refreshes_collection_after_successful_delete(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.delete_source_chunks("source-a", ["chunk-a"]) + + assert collection.delete_calls[-1]["ids"] == ["chunk-a"] + assert collection.rows == {} + assert collection.refresh_calls == 2 + + +def test_upsert_source_chunks_replaces_existing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]), + chunk("chunk-b", "source-a", [0.3, 0.2, 0.1]), + ], + ) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + assert collection.delete_calls[-1]["ids"] == ["chunk-a", "chunk-b"] + assert [event["op"] for event in collection.events[-2:]] == ["upsert", "delete"] + + +def test_upsert_source_chunks_removes_stale_bad_metadata_by_collection_membership(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + stale = chunk( + "stale-chunk", + "source-a", + [0.2, 0.2, 0.2], + metadata={"source_id": "wrong-source"}, + ) + collection.upsert( + ids=[stale.id], + documents=[stale.content], + metadatas=[{"source_id": "wrong-source", "payload": stale.model_dump_json()}], + embeddings=[stale.embedding], + ) + collection.refresh_index() + + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + assert collection.delete_calls[-1]["ids"] == ["chunk-a", "stale-chunk"] + + +def test_upsert_source_chunks_dimension_mismatch_preserves_existing_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + with pytest.raises(ValueError, match="dimension"): + index.upsert_source_chunks("source-a", [chunk("chunk-b", "source-a", [0.1, 0.2])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + + +def test_upsert_source_chunks_overrides_conflicting_metadata_source_id_for_replacement(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk( + "chunk-a", + "source-a", + [0.1, 0.2, 0.3], + metadata={"source_id": "wrong-source", "user_label": "kept"}, + ), + chunk( + "chunk-b", + "source-a", + [0.3, 0.2, 0.1], + metadata={"source_id": "wrong-source"}, + ), + ], + ) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert collection.upsert_calls[0]["metadatas"][0]["user_label"] == "kept" + + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + + +def test_upsert_source_chunks_empty_list_clears_existing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.upsert_source_chunks("source-a", []) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=10, + ) + + assert results == [] + assert collection.delete_calls[-1]["ids"] == ["chunk-a"] + assert collection.refresh_calls == 2 + + +def test_upsert_source_chunks_empty_list_missing_collection_is_noop(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", []) + + client = FakeClient.instances[0] + assert index.collection_name("source-a") not in client.collections + + +def test_delete_source_chunks_missing_collection_is_noop(tmp_path: Path, caplog): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.delete_source_chunks("source-a", ["chunk-a"]) + + client = FakeClient.instances[0] + assert index.collection_name("source-a") not in client.collections + assert not [record for record in caplog.records if record.levelname == "WARNING"] + + +def test_upsert_source_chunks_rejects_mismatched_source_id(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + with pytest.raises(ValueError, match="source_id"): + index.upsert_source_chunks("source-a", [chunk("chunk-b", "source-b", [0.1, 0.2, 0.3])]) + + +def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: Path, monkeypatch): + monkeypatch.setitem(sys.modules, "pyseekdb", None) + + with pytest.raises(SeekDBUnavailableError): + SeekDBChunkIndex(tmp_path / "native.seekdb") diff --git a/tests/test_slide_deck_domain_repository.py b/tests/test_slide_deck_domain_repository.py index 8b85837..450eee3 100644 --- a/tests/test_slide_deck_domain_repository.py +++ b/tests/test_slide_deck_domain_repository.py @@ -244,7 +244,7 @@ async def test_seekdb_repository_persists_slide_deck_state_after_restart(tmp_pat assert (await reopened.get_slide_deck_job(job.id)).status == JobStatus.FAILED -def test_seekdb_repository_uses_separate_embedded_path_for_pyseekdb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): +def test_seekdb_repository_uses_separate_embedded_path_for_native_seekdb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): calls: list[str] = [] class FakeClient: @@ -252,6 +252,9 @@ def __init__(self, path: str) -> None: calls.append(path) Path(path).mkdir(parents=True, exist_ok=True) + def list_collections(self): + return [] + monkeypatch.setitem(sys.modules, "pyseekdb", types.SimpleNamespace(Client=FakeClient)) db_path = tmp_path / "knowledge.db" @@ -259,26 +262,18 @@ def __init__(self, path: str) -> None: assert calls == [str(tmp_path / "knowledge.seekdb")] assert db_path.is_file() - assert repo.seekdb_client is not None + assert repo.native_chunk_index is not None + assert repo.storage_status()["seekdb_path"] == str(tmp_path / "knowledge.seekdb") + assert repo.storage_status()["native_available"] is True @pytest.mark.asyncio -async def test_seekdb_repository_keeps_sqlite_chunks_when_optional_pyseekdb_mirror_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - class FailingCollection: - def upsert(self, **kwargs) -> None: - raise RuntimeError("default embedding model is unavailable") - - class FakeClient: - def __init__(self, path: str) -> None: - Path(path).mkdir(parents=True, exist_ok=True) - - def get_or_create_collection(self, name: str): - return FailingCollection() - - monkeypatch.setitem(sys.modules, "pyseekdb", types.SimpleNamespace(Client=FakeClient)) - repo = SeekDBRepository(tmp_path / "knowledge.db") +async def test_seekdb_repository_uses_sqlite_chunk_search_only_when_fallback_is_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) chunk = KnowledgeChunk( id="chunk_1", source_id="src_1", @@ -293,6 +288,7 @@ def get_or_create_collection(self, name: str): results = await repo.search_chunks("hybrid retrieval", top_k=1) assert [item.id for item in loaded] == ["chunk_1"] assert results[0]["chunk"].id == "chunk_1" + assert repo.storage_status()["vector_backend"] == "sqlite_fallback" def test_slide_deck_file_store_writes_ignored_files_and_returns_metadata(tmp_path: Path): diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index a90a29e..e1eaf65 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -1,6 +1,8 @@ from __future__ import annotations +import logging from pathlib import Path +import sqlite3 from types import ModuleType, SimpleNamespace import sys import threading @@ -12,14 +14,1983 @@ from backend.api.routes.sources import router as sources_router from backend.core.services.source_service import SourceService from backend.config import Settings, get_settings +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind from backend.dependencies import get_source_service from backend.infrastructure.parsers.docling_parser import DoclingParser from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository +def test_repository_migrates_legacy_chunk_schema_with_vector_state(tmp_path: Path): + db_path = tmp_path / "legacy.db" + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE sources ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, payload TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, source_id TEXT NOT NULL, content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, embedding TEXT, payload TEXT NOT NULL + ); + """ + ) + conn.close() + + repo = SeekDBRepository( + db_path, + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + columns = {row["name"] for row in repo._conn.execute("PRAGMA table_info(chunks)").fetchall()} + assert "vector_state" in columns + state_columns = { + row["name"] for row in repo._conn.execute("PRAGMA table_info(chunk_index_state)").fetchall() + } + assert "embedding_profile" in state_columns + + +def test_repository_enables_sqlite_wal_mode(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "wal.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + journal_mode = repo._conn.execute("PRAGMA journal_mode").fetchone()[0] + + assert journal_mode.lower() == "wal" + + +@pytest.mark.asyncio +async def test_backfill_recognizes_pre_vector_state_strict_native_rows(tmp_path: Path): + db_path = tmp_path / "legacy-strict.db" + source = KnowledgeSource(id="src-legacy-strict", kind=SourceKind.TEXT, title="Legacy strict") + native_chunk = KnowledgeChunk( + id="chunk-legacy-strict", + source_id=source.id, + content="already stored natively", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + sqlite_chunk = native_chunk.model_copy(update={"embedding": None}) + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE sources ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, payload TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, source_id TEXT NOT NULL, content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, embedding TEXT, payload TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO sources VALUES (?, ?, ?, ?, ?)", + ( + source.id, + source.status.value, + source.model_dump_json(), + source.created_at.isoformat(), + source.updated_at.isoformat(), + ), + ) + conn.execute( + "INSERT INTO chunks VALUES (?, ?, ?, ?, ?, ?)", + ( + sqlite_chunk.id, + sqlite_chunk.source_id, + sqlite_chunk.content, + sqlite_chunk.chunk_index, + None, + sqlite_chunk.model_dump_json(), + ), + ) + conn.commit() + conn.close() + native_index = RecordingNativeIndex() + native_index.chunks_by_source[source.id] = [native_chunk] + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + + assert written == 0 + assert native_index.upserts == [] + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [native_chunk.id] + state = repo._conn.execute( + "SELECT vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone()["vector_state"] + assert state == "seekdb" + + +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + self.deletes = [] + self.searches = [] + self.text_searches = [] + self.chunks_by_source = {} + self.get_source_calls = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + self.get_source_calls.append(source_id) + return list(self.chunks_by_source.get(source_id, [])) + + def delete_source_chunks(self, source_id, chunk_ids): + self.deletes.append((source_id, chunk_ids)) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_chunk", + source_id=source_ids[0], + content="native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + def text_search(self, query_text, source_ids, top_k): + self.text_searches.append((query_text, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_text_chunk", + source_id=source_ids[0], + content="native fulltext result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.97, + "backend": "seekdb", + } + ] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + +class StaleAwareNativeIndex(RecordingNativeIndex): + def __init__(self) -> None: + super().__init__() + self.cleared_sources: set[str] = set() + + def upsert_source_chunks(self, source_id, chunks): + super().upsert_source_chunks(source_id, chunks) + if chunks == []: + self.cleared_sources.add(source_id) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + if source_ids[0] in self.cleared_sources: + return [] + return [ + { + "chunk": KnowledgeChunk( + id="stale_native_chunk", + source_id=source_ids[0], + content="stale native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + +class FailingNativeIndex(RecordingNativeIndex): + def __init__(self, message: str = "native write failed") -> None: + super().__init__() + self.message = message + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + raise RuntimeError(self.message) + + +class ProbeFailingNativeIndex(RecordingNativeIndex): + def probe(self): + raise RuntimeError("native probe failed") + + +class FailOnceNativeIndex(RecordingNativeIndex): + def __init__(self, source_id: str, existing_chunks: list[KnowledgeChunk]) -> None: + super().__init__() + self.chunks_by_source[source_id] = list(existing_chunks) + self.fail_next_upsert = True + + def upsert_source_chunks(self, source_id, chunks): + if self.fail_next_upsert: + self.fail_next_upsert = False + self.chunks_by_source[source_id] = list(chunks) + raise RuntimeError("native write failed") + super().upsert_source_chunks(source_id, chunks) + + +class PartialWriteAndRestoreFailingNativeIndex(RecordingNativeIndex): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 0 + + def upsert_source_chunks(self, source_id, chunks): + if self.failures_remaining == 2: + self.failures_remaining -= 1 + self.chunks_by_source[source_id] = list(chunks) + raise RuntimeError("partial native write") + if self.failures_remaining == 1: + self.failures_remaining -= 1 + raise RuntimeError("native restore failed") + super().upsert_source_chunks(source_id, chunks) + + +class DimensionCheckingNativeIndex(RecordingNativeIndex): + def search(self, query_embedding, source_ids, top_k): + bad_source_ids = [ + source_id + for source_id, chunks in self.upserts + if source_id in source_ids + and chunks + and len(chunks[0].embedding or []) != len(query_embedding) + ] + if bad_source_ids: + raise ValueError(f"dimension mismatch for {bad_source_ids}") + return super().search(query_embedding, source_ids, top_k) + + +class QueryFailingNativeIndex(RecordingNativeIndex): + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + raise KeyError("collection not found") + + +class DeterministicEmbeddingProvider: + def __init__(self) -> None: + self.calls = [] + + async def embed(self, content): + self.calls.append(content) + if isinstance(content, str): + return [0.1, 0.2, 0.3] + return [[0.1, 0.2, 0.3] for _ in content] + + +@pytest.mark.asyncio +async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp_path: Path, caplog): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native", kind=SourceKind.TEXT, title="Native") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native", + source_id=source.id, + content="native indexed content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with caplog.at_level(logging.WARNING): + results = await repo.search_chunks( + "native indexed", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts[0][0] == source.id + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "native_chunk" + assert "Skipping optional pyseekdb chunk mirror" not in caplog.text + + +@pytest.mark.asyncio +async def test_repository_search_automatically_migrates_legacy_sqlite_vectors(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-auto-migrate", kind=SourceKind.TEXT, title="Auto migrate") + chunk = KnowledgeChunk( + id="chunk-auto-migrate", + source_id=source.id, + content="legacy vector should migrate before search", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + legacy_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await legacy_repo.save_source(source) + await legacy_repo.save_chunks(source.id, [chunk]) + await legacy_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + results = await repo.search_chunks( + "legacy vector", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [chunk])] + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["backend"] == "seekdb" + row = repo._conn.execute( + "SELECT embedding, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert row["embedding"] is None + assert row["vector_state"] == "seekdb" + + +@pytest.mark.asyncio +async def test_pending_seekdb_replace_is_recovered_after_process_interruption( + tmp_path: Path, + monkeypatch, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-crash-recovery", kind=SourceKind.TEXT, title="Crash recovery") + chunk = KnowledgeChunk( + id="chunk-crash-recovery", + source_id=source.id, + content="durable pending replacement", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + + def interrupt_after_native_write(*_args, **_kwargs): + raise SystemExit("simulated process interruption") + + monkeypatch.setattr(repo, "_finalize_chunk_replace", interrupt_after_native_write) + with pytest.raises(SystemExit, match="process interruption"): + await repo.save_chunks(source.id, [chunk]) + + pending = repo._conn.execute( + "SELECT operation FROM chunk_sync_operations WHERE source_id = ?", + (source.id,), + ).fetchone() + assert pending["operation"] == "replace" + assert repo._get_sqlite_chunks_sync(source.id) == [] + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert [item.id for item in await recovered.get_chunks(source.id)] == [chunk.id] + assert [item.id for item in recovered._get_sqlite_chunks_sync(source.id)] == [chunk.id] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_failed_native_compensation_keeps_a_durable_restore_operation(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-restore-retry", kind=SourceKind.TEXT, title="Restore retry") + old_chunk = KnowledgeChunk( + id="chunk-old-restore-retry", + source_id=source.id, + content="old canonical content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + new_chunk = KnowledgeChunk( + id="chunk-new-restore-retry", + source_id=source.id, + content="partially written content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = PartialWriteAndRestoreFailingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + await repo.save_chunks(source.id, [old_chunk]) + native_index.failures_remaining = 2 + + with pytest.raises(RuntimeError, match="partial native write"): + await repo.save_chunks(source.id, [new_chunk]) + + pending = repo._conn.execute( + "SELECT operation FROM chunk_sync_operations WHERE source_id = ?", + (source.id,), + ).fetchone() + assert pending["operation"] == "native_restore" + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [new_chunk.id] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == [old_chunk.id] + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [old_chunk.id] + assert [chunk.id for chunk in recovered._get_sqlite_chunks_sync(source.id)] == [old_chunk.id] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_pending_seekdb_delete_is_recovered_after_process_interruption( + tmp_path: Path, + monkeypatch, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-delete-crash", kind=SourceKind.TEXT, title="Delete crash") + chunk = KnowledgeChunk( + id="chunk-delete-crash", + source_id=source.id, + content="delete must finish after restart", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + + def interrupt_after_native_delete(*_args, **_kwargs): + raise SystemExit("simulated delete interruption") + + monkeypatch.setattr(repo, "_finalize_source_delete", interrupt_after_native_delete) + with pytest.raises(SystemExit, match="delete interruption"): + await repo.delete_source(source.id) + assert await repo.get_source(source.id) is not None + assert repo._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 1 + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert await recovered.get_source(source.id) is None + assert native_index.get_source_chunks(source.id) == [] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_native_search_uses_source_state_without_loading_all_chunks(tmp_path: Path): + source = KnowledgeSource(id="src-lightweight-search", kind=SourceKind.TEXT, title="Lightweight") + chunk = KnowledgeChunk( + id="chunk-lightweight-search", + source_id=source.id, + content="search should not deserialize the whole collection", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + native_index.get_source_calls.clear() + + await repo.search_chunks( + "deserialize", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + await repo.search_chunks("deserialize", source_ids=[source.id], top_k=1) + + assert native_index.get_source_calls == [] + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert native_index.text_searches == [("deserialize", [source.id], 1)] + + +@pytest.mark.asyncio +async def test_strict_mode_scrubs_vectors_after_fallback_backfill(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-mode-switch", kind=SourceKind.TEXT, title="Mode switch") + chunk = KnowledgeChunk( + id="chunk-mode-switch", + source_id=source.id, + content="fallback vectors must be scrubbed in strict mode", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks(source.id, [chunk]) + await fallback_repo.backfill_native_chunks() + await fallback_repo.close() + + strict_repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await strict_repo.initialize_storage() + + row = strict_repo._conn.execute( + "SELECT embedding, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert row["embedding"] is None + assert row["vector_state"] == "seekdb" + + +@pytest.mark.asyncio +async def test_strict_native_mode_keeps_vectors_out_of_sqlite(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-only", kind=SourceKind.TEXT, title="Native only") + chunk = KnowledgeChunk( + id="chunk-native-only", + source_id=source.id, + content="stored in SeekDB", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + await repo.save_source(source) + + await repo.save_chunks(source.id, [chunk]) + + sqlite_row = repo._conn.execute( + "SELECT embedding, payload, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert sqlite_row is not None + assert sqlite_row["embedding"] is None + assert '"embedding":null' in sqlite_row["payload"] + assert sqlite_row["vector_state"] == "seekdb" + loaded = await repo.get_chunks(source.id) + assert [item.id for item in loaded] == [chunk.id] + assert loaded[0].embedding == [0.1, 0.2, 0.3] + + +@pytest.mark.asyncio +async def test_repository_requires_embeddings_when_native_seekdb_is_primary(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-no-embedding", kind=SourceKind.TEXT, title="Native No Embedding") + await repo.save_source(source) + + with pytest.raises((RuntimeError, ValueError), match="embedding|native SeekDB"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native-no-embedding", + source_id=source.id, + content="native primary requires an embedding", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native", kind=SourceKind.TEXT, title="No Native") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native", + source_id=source.id, + content="content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert repo._get_sqlite_chunks_sync(source.id) == [] + + +@pytest.mark.asyncio +async def test_strict_empty_save_with_native_unavailable_keeps_existing_sqlite_chunks(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-strict-empty", kind=SourceKind.TEXT, title="Strict Empty") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="existing sqlite content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=False) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks(source.id, []) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_non_vector_chunk_save_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native-text", kind=SourceKind.TEXT, title="No Native Text") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native-text", + source_id=source.id, + content="content without an embedding", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + +@pytest.mark.asyncio +async def test_repository_uses_native_fulltext_when_query_embedding_is_unavailable(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-query", kind=SourceKind.TEXT, title="Native Query") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native-query", + source_id=source.id, + content="native query content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks("native query", source_ids=[source.id], top_k=1) + + assert native_index.text_searches == [("native query", [source.id], 1)] + assert results[0]["chunk"].id == "native_text_chunk" + assert results[0]["backend"] == "seekdb" + + +@pytest.mark.asyncio +async def test_strict_native_get_chunks_rejects_unavailable_seekdb(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks("src-missing-native") + + +@pytest.mark.asyncio +async def test_repository_does_not_search_sqlite_bm25_without_explicit_fallback(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-bm25-disabled", kind=SourceKind.TEXT, title="BM25 Disabled") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-bm25-disabled", + source_id=source.id, + content="Needle lexical content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + repo.allow_sqlite_vector_fallback = False + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.search_chunks("Needle", source_ids=[source.id], top_k=1) + + +@pytest.mark.asyncio +async def test_fallback_no_embedding_save_clears_native_chunks_and_uses_sqlite_search(tmp_path: Path): + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-fallback-clear", kind=SourceKind.TEXT, title="Fallback Clear") + await repo.save_source(source) + + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-fallback-clear", + source_id=source.id, + content="Fresh fallback lexical content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + sqlite_results = await repo.search_chunks("Fresh fallback", source_ids=[source.id], top_k=1) + vector_results = await repo.search_chunks( + "stale native", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [])] + assert sqlite_results[0]["chunk"].id == "chunk-fallback-clear" + assert vector_results == [] + + +@pytest.mark.asyncio +async def test_delete_source_clears_native_chunks_even_when_sqlite_ids_are_stale(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-delete-clear", kind=SourceKind.TEXT, title="Delete Clear") + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="sqlite-only-chunk", + source_id=source.id, + content="sqlite-only content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await seed_repo.close() + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=True) + + await repo.delete_source(source.id) + stale_results = await repo.search_chunks( + "stale native", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert (source.id, []) in native_index.upserts + assert stale_results == [] + + +@pytest.mark.asyncio +async def test_delete_source_requires_native_seekdb_before_sqlite_mutation(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-strict-delete", kind=SourceKind.TEXT, title="Strict Delete") + old_chunk = KnowledgeChunk( + id="chunk-strict-delete", + source_id=source.id, + content="existing sqlite metadata", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=False) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.delete_source(source.id) + + assert await repo.get_source(source.id) is not None + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-strict-delete"] + + +@pytest.mark.asyncio +async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-native-rollback", kind=SourceKind.TEXT, title="Native Rollback") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old sqlite content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + native_index = FailOnceNativeIndex(source.id, [old_chunk]) + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="native write failed"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new", + source_id=source.id, + content="new sqlite content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_delete_source_restores_native_chunks_when_sqlite_delete_fails(tmp_path: Path): + source = KnowledgeSource(id="src-delete-compensation", kind=SourceKind.TEXT, title="Delete compensation") + old_chunk = KnowledgeChunk( + id="chunk-delete-compensation", + source_id=source.id, + content="must survive a failed delete", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [old_chunk]) + repo._conn.execute( + """ + CREATE TRIGGER fail_source_delete BEFORE DELETE ON sources + BEGIN SELECT RAISE(ABORT, 'delete failed'); END + """ + ) + + with pytest.raises(Exception, match="delete failed"): + await repo.delete_source(source.id) + + assert await repo.get_source(source.id) is not None + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == [old_chunk.id] + + +@pytest.mark.asyncio +async def test_strict_native_rejects_mixed_dimension_embeddings_before_sqlite_mutation( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-mixed-dim", kind=SourceKind.TEXT, title="Mixed Dimension") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old sqlite content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="same-dimension embeddings"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new-a", + source_id=source.id, + content="new sqlite content a", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-new-b", + source_id=source.id, + content="new sqlite content b", + chunk_index=1, + embedding=[0.1, 0.2], + metadata={"source_id": source.id}, + ), + ], + ) + + assert native_index.upserts == [] + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_native_clear_failure_rolls_back_fallback_sqlite_chunk_replacement(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-clear-rollback", kind=SourceKind.TEXT, title="Clear Rollback") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + repo = SeekDBRepository( + db_path, + native_chunk_index=FailingNativeIndex("native clear failed"), + allow_sqlite_vector_fallback=True, + ) + + with pytest.raises(RuntimeError, match="native clear failed"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new", + source_id=source.id, + content="new fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-backfill", kind=SourceKind.TEXT, title="Backfill") + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-backfill", + source_id=source.id, + content="backfill content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + written = await repo.backfill_native_chunks() + second_written = await repo.backfill_native_chunks() + + assert written == 1 + assert second_written == 0 + assert len(native_index.upserts) == 1 + assert native_index.upserts[0][0] == source.id + assert native_index.upserts[0][1][0].id == "chunk-backfill" + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == ["chunk-backfill"] + sqlite_row = repo._conn.execute( + "SELECT embedding, payload FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert sqlite_row["embedding"] is None + assert '"embedding":null' in sqlite_row["payload"] + + +@pytest.mark.asyncio +async def test_backfill_skips_payload_deserialization_for_reconciled_sources(tmp_path: Path): + db_path = tmp_path / "reconciled.db" + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-reconciled", kind=SourceKind.TEXT, title="Reconciled") + chunk = KnowledgeChunk( + id="chunk-reconciled", + source_id=source.id, + content="already reconciled", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + repo._conn.execute( + "UPDATE chunks SET payload = 'not-json' WHERE source_id = ?", + (source.id,), + ) + repo._conn.commit() + + written = repo._backfill_native_chunks_sync() + + assert written == 0 + + +@pytest.mark.asyncio +async def test_backfill_restores_native_chunks_when_sqlite_scrub_fails(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-backfill-compensation", kind=SourceKind.TEXT, title="Backfill compensation") + legacy_chunk = KnowledgeChunk( + id="chunk-legacy", + source_id=source.id, + content="legacy vector", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + previous_native_chunk = KnowledgeChunk( + id="chunk-native-old", + source_id=source.id, + content="previous native vector", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks(source.id, [legacy_chunk]) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + native_index.chunks_by_source[source.id] = [previous_native_chunk] + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + repo._conn.execute( + """ + CREATE TRIGGER fail_chunk_scrub BEFORE UPDATE ON chunks + BEGIN SELECT RAISE(ABORT, 'scrub failed'); END + """ + ) + + with pytest.raises(Exception, match="scrub failed"): + await repo.backfill_native_chunks() + + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [previous_native_chunk.id] + sqlite_chunk = repo._get_sqlite_chunks_sync(source.id)[0] + assert sqlite_chunk.embedding == [0.1, 0.2, 0.3] + + +@pytest.mark.asyncio +async def test_repository_backfill_requires_native_index_without_fallback(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="cannot backfill chunks"): + await repo.backfill_native_chunks() + + +def test_repository_storage_status_reports_failed_native_probe(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "probe-status.db", + native_chunk_index=ProbeFailingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + status = repo.storage_status() + + assert status["vector_backend"] == "unavailable" + assert status["native_available"] is False + assert "native probe failed" in status["error"] + + +@pytest.mark.asyncio +async def test_repository_backfill_clears_stale_native_for_fallback_rows_without_embeddings(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-backfill-clear", kind=SourceKind.TEXT, title="Backfill Clear") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain backfill content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + native_index.chunks_by_source[source.id] = [ + KnowledgeChunk( + id="stale-native", + source_id=source.id, + content="stale native content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ] + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + second_written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "plain backfill", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert second_written == 0 + assert native_index.upserts == [(source.id, [])] + assert native_index.get_source_chunks(source.id) == [] + state = repo._conn.execute( + "SELECT vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone()["vector_state"] + assert state == "legacy_needs_embedding" + assert results[0]["chunk"].id == "chunk-plain" + assert results[0]["backend"] == "sqlite_legacy_text" + + +@pytest.mark.asyncio +async def test_strict_backfill_keeps_unembedded_legacy_sources_lexically_searchable_and_retryable( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-legacy-lexical", kind=SourceKind.TEXT, title="Legacy lexical") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-legacy-lexical", + source_id=source.id, + content="Asterion protocol preserves lexical retrieval", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + repo = SeekDBRepository( + db_path, + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + await repo.initialize_storage() + results = await repo.search_chunks( + "Asterion protocol", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert results[0]["chunk"].id == "chunk-legacy-lexical" + assert results[0]["backend"] == "sqlite_legacy_text" + state = repo._conn.execute( + "SELECT storage_mode FROM chunk_index_state WHERE source_id = ?", + (source.id,), + ).fetchone() + assert state["storage_mode"] == "sqlite_legacy" + assert repo._get_repository_meta("native_chunk_migration_v2") is None + chunks = await repo.get_chunks(source.id) + assert [chunk.id for chunk in chunks] == ["chunk-legacy-lexical"] + + +@pytest.mark.asyncio +async def test_repository_native_search_skips_legacy_sources_without_embeddings_after_backfill( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + embedded_source = KnowledgeSource(id="src-embedded", kind=SourceKind.TEXT, title="Embedded") + plain_source = KnowledgeSource(id="src-plain", kind=SourceKind.TEXT, title="Plain") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(embedded_source) + await fallback_repo.save_chunks( + embedded_source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=embedded_source.id, + content="embedded legacy content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": embedded_source.id}, + ) + ], + ) + await fallback_repo.save_source(plain_source) + await fallback_repo.save_chunks( + plain_source.id, + [ + KnowledgeChunk( + id="chunk-plain", + source_id=plain_source.id, + content="plain legacy content", + chunk_index=0, + metadata={"source_id": plain_source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + await repo.backfill_native_chunks() + await repo.search_chunks("legacy", source_ids=None, top_k=1, query_embedding=[0.1, 0.2, 0.3]) + + assert {source_id for source_id, _chunks in native_index.upserts} == { + embedded_source.id, + plain_source.id, + } + assert native_index.searches == [([0.1, 0.2, 0.3], [embedded_source.id], 1)] + + +@pytest.mark.asyncio +async def test_repository_backfill_skips_partially_embedded_sources_and_native_search( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-partial", kind=SourceKind.TEXT, title="Partial") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=source.id, + content="embedded partial content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain partial content", + chunk_index=1, + metadata={"source_id": source.id}, + ), + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "partial", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results[0]["chunk"].source_id == source.id + assert results[0]["backend"] == "sqlite_legacy_text" + + +@pytest.mark.asyncio +async def test_repository_native_search_uses_sqlite_fallback_for_partial_writes(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-partial-fallback", kind=SourceKind.TEXT, title="Partial Fallback") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=source.id, + content="embedded fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain fallback content", + chunk_index=1, + metadata={"source_id": source.id}, + ), + ], + ) + + results = await repo.search_chunks( + "fallback", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results + assert results[0]["chunk"].source_id == source.id + + +@pytest.mark.asyncio +async def test_repository_backfill_treats_empty_embedding_as_native_ineligible(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-empty-embedding", kind=SourceKind.TEXT, title="Empty Embedding") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-embedding", + source_id=source.id, + content="empty embedding content", + chunk_index=0, + embedding=[], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "empty embedding", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results[0]["chunk"].id == "chunk-empty-embedding" + assert results[0]["backend"] == "sqlite_legacy_text" + + +@pytest.mark.asyncio +async def test_repository_native_search_rejects_dimension_mismatched_sources( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source_3d = KnowledgeSource(id="src-3d", kind=SourceKind.TEXT, title="3D") + source_2d = KnowledgeSource(id="src-2d", kind=SourceKind.TEXT, title="2D") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source_3d) + await fallback_repo.save_chunks( + source_3d.id, + [ + KnowledgeChunk( + id="chunk-3d", + source_id=source_3d.id, + content="three dimension content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_3d.id}, + ) + ], + ) + await fallback_repo.save_source(source_2d) + await fallback_repo.save_chunks( + source_2d.id, + [ + KnowledgeChunk( + id="chunk-2d", + source_id=source_2d.id, + content="two dimension content", + chunk_index=0, + embedding=[0.1, 0.2], + metadata={"source_id": source_2d.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = DimensionCheckingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.backfill_native_chunks() + + with pytest.raises(RuntimeError, match="embedding dimension"): + await repo.search_chunks( + "dimension", + source_ids=None, + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [] + + +@pytest.mark.asyncio +async def test_repository_native_search_rejects_changed_embedding_profile(tmp_path: Path): + native_index = RecordingNativeIndex() + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-profile", kind=SourceKind.TEXT, title="Profile") + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + embedding_profile_id="profile-a", + ) + await writer.save_source(source) + await writer.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-profile", + source_id=source.id, + content="embedding profile content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await writer.close() + + reader = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + embedding_profile_id="profile-b", + ) + + with pytest.raises(RuntimeError, match="embedding profile"): + await reader.search_chunks( + "profile", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + +@pytest.mark.asyncio +async def test_sqlite_fallback_does_not_compare_vectors_from_an_old_embedding_profile( + tmp_path: Path, +): + native_index = RecordingNativeIndex() + db_path = tmp_path / "fallback-profile.db" + source = KnowledgeSource(id="src-fallback-profile", kind=SourceKind.TEXT, title="Fallback") + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + embedding_profile_id="profile-a", + ) + await writer.save_source(source) + await writer.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-fallback-profile", + source_id=source.id, + content="unrelated content", + chunk_index=0, + embedding=[1.0, 0.0], + metadata={"source_id": source.id}, + ) + ], + ) + await writer.close() + + reader = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + embedding_profile_id="profile-b", + ) + results = await reader.search_chunks( + "needle", + source_ids=[source.id], + top_k=1, + query_embedding=[1.0, 0.0], + ) + + assert results == [] + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_handles_dimension_mismatched_sources( + tmp_path: Path, +): + native_index = DimensionCheckingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source_3d = KnowledgeSource(id="src-3d-fallback", kind=SourceKind.TEXT, title="3D Fallback") + source_2d = KnowledgeSource(id="src-2d-fallback", kind=SourceKind.TEXT, title="2D Fallback") + await repo.save_source(source_3d) + await repo.save_chunks( + source_3d.id, + [ + KnowledgeChunk( + id="chunk-3d-fallback", + source_id=source_3d.id, + content="three dimension fallback", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_3d.id}, + ) + ], + ) + await repo.save_source(source_2d) + await repo.save_chunks( + source_2d.id, + [ + KnowledgeChunk( + id="chunk-2d-fallback", + source_id=source_2d.id, + content="two dimension fallback", + chunk_index=0, + embedding=[0.1, 0.2], + metadata={"source_id": source_2d.id}, + ) + ], + ) + + results = await repo.search_chunks( + "two dimension", + source_ids=[source_2d.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [] + assert results + assert results[0]["chunk"].id == "chunk-2d-fallback" + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_handles_missing_native_collection_for_eligible_chunks( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-legacy-eligible", kind=SourceKind.TEXT, title="Legacy Eligible") + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-legacy-eligible", + source_id=source.id, + content="legacy eligible fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await seed_repo.close() + + native_index = QueryFailingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=True) + + results = await repo.search_chunks( + "legacy eligible", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [] + assert results + assert results[0]["chunk"].id == "chunk-legacy-eligible" + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_keeps_vector_only_zero_cosine_results( + tmp_path: Path, +): + source = KnowledgeSource(id="src-zero-cosine", kind=SourceKind.TEXT, title="Zero Cosine") + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=QueryFailingNativeIndex(), + allow_sqlite_vector_fallback=True, + ) + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-zero-cosine", + source_id=source.id, + content="orthogonal vector content without lexical query overlap", + chunk_index=0, + embedding=[0.0, 1.0], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "unrelated", + source_ids=[source.id], + top_k=1, + query_embedding=[1.0, 0.0], + ) + + assert results + assert results[0]["chunk"].id == "chunk-zero-cosine" + assert results[0]["score"] == 0.0 + + +@pytest.mark.asyncio +async def test_strict_native_search_rejects_empty_query_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-empty-query", kind=SourceKind.TEXT, title="Empty Query") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-query", + source_id=source.id, + content="empty query content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with pytest.raises(RuntimeError, match="non-empty query embedding"): + await repo.search_chunks( + "empty query", + source_ids=[source.id], + top_k=1, + query_embedding=[], + ) + + assert native_index.searches == [] + + +@pytest.mark.asyncio +async def test_sqlite_fallback_handles_empty_query_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-empty-query-fallback", kind=SourceKind.TEXT, title="Empty Query Fallback") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-query-fallback", + source_id=source.id, + content="empty query fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "empty query fallback", + source_ids=[source.id], + top_k=1, + query_embedding=[], + ) + + assert native_index.searches == [] + assert results + assert results[0]["chunk"].id == "chunk-empty-query-fallback" + + +@pytest.mark.asyncio +async def test_sqlite_fallback_respects_empty_source_selection(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-empty-selection", kind=SourceKind.TEXT, title="Empty Selection") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-selection", + source_id=source.id, + content="empty selection fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + no_embedding_results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=None, + ) + empty_embedding_results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=[], + ) + + assert no_embedding_results == [] + assert empty_embedding_results == [] + + +@pytest.mark.asyncio +async def test_strict_native_search_respects_empty_source_selection_without_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=None, + ) + + assert results == [] + assert native_index.searches == [] + + +@pytest.mark.asyncio +async def test_source_service_ingestion_and_vector_store_search_use_native_seekdb(tmp_path: Path): + native_index = RecordingNativeIndex() + embedding_provider = DeterministicEmbeddingProvider() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + service = SourceService( + repository=repo, + parser=DoclingParser(), + embedding_provider=embedding_provider, + chunk_size=128, + chunk_overlap=0, + ) + + source = await service.create_text_source("Native E2E", "Alpha native retrieval content.") + from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore + + store = SeekDBVectorStore(repo, embedding_provider=embedding_provider) + results = await store.search("Alpha native", top_k=1, doc_ids=[source.id]) + + assert source.status == "ready" + assert native_index.upserts[0][0] == source.id + assert all(chunk.embedding == [0.1, 0.2, 0.3] for chunk in native_index.upserts[0][1]) + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["id"] == "native_chunk" + + +@pytest.mark.asyncio +async def test_source_service_marks_ready_only_after_chunk_commit(tmp_path: Path, monkeypatch): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "ingestion-order.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + events: list[str] = [] + original_save_source = repo.save_source + original_save_chunks = repo.save_chunks + + async def record_source(source): + events.append(f"source:{source.status.value}") + return await original_save_source(source) + + async def record_chunks(source_id, chunks): + events.append("chunks") + return await original_save_chunks(source_id, chunks) + + monkeypatch.setattr(repo, "save_source", record_source) + monkeypatch.setattr(repo, "save_chunks", record_chunks) + service = SourceService( + repository=repo, + parser=DoclingParser(), + embedding_provider=DeterministicEmbeddingProvider(), + chunk_size=128, + chunk_overlap=0, + ) + + source = await service.create_text_source("Ordered", "Commit chunks before ready.") + + assert source.status.value == "ready" + assert events == ["source:processing", "source:processing", "chunks", "source:ready"] + + +@pytest.mark.asyncio +async def test_repository_recovers_processing_source_after_committed_chunks(tmp_path: Path): + db_path = tmp_path / "processing-recovery.db" + native_index = RecordingNativeIndex() + source = KnowledgeSource( + id="src-processing-recovery", + kind=SourceKind.TEXT, + title="Recovery", + text="Parsed content survived the interruption.", + chunk_count=1, + ) + chunk = KnowledgeChunk( + id="chunk-processing-recovery", + source_id=source.id, + content=source.text, + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await writer.save_source(source) + await writer.save_chunks(source.id, [chunk]) + await writer.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + loaded = await recovered.get_source(source.id) + assert loaded is not None + assert loaded.status.value == "ready" + + @pytest.mark.asyncio async def test_text_source_is_chunked_with_citation_metadata(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService( repository=repo, parser=DoclingParser(), @@ -53,7 +2024,7 @@ async def parse_file(self, file_path: str | Path, filename: str | None = None): async def parse_text(self, text: str, title: str = "Pasted text"): raise AssertionError("not used") - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=BrokenParser()) file_path = tmp_path / "broken.pdf" file_path.write_bytes(b"%PDF-1.4 broken") @@ -68,13 +2039,13 @@ async def parse_text(self, text: str, title: str = "Pasted text"): @pytest.mark.asyncio async def test_seekdb_repository_persists_sources_after_restart(tmp_path: Path): db_path = tmp_path / "knowledge.db" - repo = SeekDBRepository(db_path) + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=40, chunk_overlap=0) created = await service.create_text_source("Durable", "Persistent source text for restart.") await repo.close() - reopened = SeekDBRepository(db_path) + reopened = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) loaded = await reopened.get_source(created.id) chunks = await reopened.get_chunks(created.id) @@ -86,7 +2057,7 @@ async def test_seekdb_repository_persists_sources_after_restart(tmp_path: Path): @pytest.mark.asyncio async def test_delete_source_removes_chunks_from_retrieval(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) source = await service.create_text_source("Delete me", "Needle phrase should disappear.") @@ -100,7 +2071,7 @@ async def test_delete_source_removes_chunks_from_retrieval(tmp_path: Path): def test_sources_api_text_upload_list_get_delete(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "api.db") + repo = SeekDBRepository(tmp_path / "api.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) app = FastAPI() app.include_router(sources_router, prefix="/api") @@ -131,8 +2102,28 @@ def test_sources_api_text_upload_list_get_delete(tmp_path: Path): assert missing.status_code == 404 +def test_sources_api_rejects_strict_seekdb_ingestion_without_embeddings(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "strict-api.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) + app = FastAPI() + app.include_router(sources_router, prefix="/api") + app.dependency_overrides[get_source_service] = lambda: service + + response = TestClient(app).post( + "/api/sources/text", + json={"title": "Missing embeddings", "text": "This source requires vectors."}, + ) + + assert response.status_code == 422 + assert "require embeddings" in response.json()["detail"] + + def test_sources_api_file_upload_text_fallback(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "upload.db") + repo = SeekDBRepository(tmp_path / "upload.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) app = FastAPI() app.include_router(sources_router, prefix="/api")