From 50b2171396805d8809b8c6eb48de4f3b7b5efa75 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 19:56:05 +0800 Subject: [PATCH 01/22] feat(seekdb): add native chunk vector index --- .../vector_stores/seekdb_chunk_index.py | 139 ++++++++++++++ tests/test_seekdb_chunk_index.py | 174 ++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 backend/infrastructure/vector_stores/seekdb_chunk_index.py create mode 100644 tests/test_seekdb_chunk_index.py diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py new file mode 100644 index 0000000..d534fed --- /dev/null +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -0,0 +1,139 @@ +"""Native SeekDB vector index for knowledge chunks.""" + +from __future__ import annotations + +import hashlib +import importlib +import logging +from pathlib import Path +from typing import Any + +from ...domain.source import KnowledgeChunk + +logger = logging.getLogger(__name__) + + +class SeekDBUnavailableError(RuntimeError): + """Raised when the native pyseekdb chunk index cannot be initialized.""" + + +class SeekDBChunkIndex: + """Source-scoped native pyseekdb chunk vector index.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.path.mkdir(parents=True, exist_ok=True) + try: + self.pyseekdb = importlib.import_module("pyseekdb") + self.client = self.pyseekdb.Client(path=str(self.path)) + except Exception as exc: + raise SeekDBUnavailableError("Native SeekDB chunk index is unavailable") from exc + + def collection_name(self, source_id: str) -> str: + digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] + return f"chunks_{digest}" + + def _first_embedding_dimension(self, chunks: list[KnowledgeChunk]) -> int: + if not chunks: + raise ValueError("At least one chunk with an embedding is required") + embedding = chunks[0].embedding + if not embedding: + raise ValueError("Chunk embeddings must be non-empty") + return len(embedding) + + def _collection(self, source_id: str, dimension: int | None = None) -> Any: + name = self.collection_name(source_id) + if dimension is None: + return self.client.get_collection(name=name, embedding_function=None) + configuration = self.pyseekdb.HNSWConfiguration(dimension=dimension) + return self.client.get_or_create_collection( + name=name, + configuration=configuration, + embedding_function=None, + ) + + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + dimension = self._first_embedding_dimension(chunks) + embeddings: list[list[float]] = [] + metadatas: list[dict[str, Any]] = [] + for chunk in chunks: + if not chunk.embedding or len(chunk.embedding) != dimension: + raise ValueError("All chunks must have same-dimension non-empty embeddings") + embeddings.append(chunk.embedding) + metadata = dict(chunk.metadata) + metadata.setdefault("source_id", chunk.source_id) + metadata.setdefault("chunk_index", chunk.chunk_index) + metadata["payload"] = chunk.model_dump_json() + metadatas.append(metadata) + + collection = self._collection(source_id, dimension=dimension) + collection.upsert( + ids=[chunk.id for chunk in chunks], + documents=[chunk.content for chunk in chunks], + metadatas=metadatas, + embeddings=embeddings, + ) + collection.refresh_index() + + def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: + if not chunk_ids: + return + try: + collection = self._collection(source_id) + collection.delete(ids=chunk_ids) + except Exception as exc: + logger.warning("Failed to delete SeekDB chunks for source %s: %s", source_id, exc) + + def search( + self, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("Query embedding is required") + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + for source_id in source_ids: + try: + collection = self._collection(source_id) + query_result = collection.query( + query_embeddings=[query_embedding], + n_results=top_k, + include=include, + ) + except Exception as exc: + logger.warning("Failed to query SeekDB chunks for source %s: %s", source_id, exc) + continue + + ids = self._result_group(query_result.get("ids", [])) + distances = self._result_group(query_result.get("distances", [])) + metadatas = self._result_group(query_result.get("metadatas", [])) + for index, _chunk_id in enumerate(ids): + try: + metadata = metadatas[index] + chunk = KnowledgeChunk.model_validate_json(metadata["payload"]) + distance = distances[index] if index < len(distances) else 0.0 + score = 1.0 / (1.0 + max(float(distance), 0.0)) + results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) + except Exception as exc: + logger.warning("Skipping malformed SeekDB chunk result: %s", exc) + + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + + @staticmethod + def _result_group(value: Any) -> list[Any]: + if not value: + return [] + first = value[0] + return first if isinstance(first, list) else value + + def status(self) -> dict[str, Any]: + return { + "vector_backend": "seekdb", + "seekdb_path": str(self.path), + "native_available": True, + } diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py new file mode 100644 index 0000000..f0f0b59 --- /dev/null +++ b/tests/test_seekdb_chunk_index.py @@ -0,0 +1,174 @@ +import sys +import types +from pathlib import Path + +import pytest + +from backend.domain.source import KnowledgeChunk +from backend.infrastructure.vector_stores.seekdb_chunk_index import ( + SeekDBChunkIndex, + SeekDBUnavailableError, +) + + +class FakeHNSWConfiguration: + def __init__(self, dimension: int, distance: str = "cosine") -> None: + self.dimension = dimension + self.distance = distance + + +class FakeCollection: + def __init__(self, name: str, dimension: int) -> None: + self.name = name + self.dimension = dimension + self.upsert_calls = [] + self.delete_calls = [] + self.query_calls = [] + self.refresh_calls = 0 + self.rows = {} + + def upsert(self, ids, documents, metadatas, embeddings): + self.upsert_calls.append( + { + "ids": ids, + "documents": documents, + "metadatas": metadatas, + "embeddings": embeddings, + } + ) + for index, chunk_id in enumerate(ids): + self.rows[chunk_id] = { + "document": documents[index], + "metadata": metadatas[index], + "embedding": embeddings[index], + } + + def delete(self, ids=None, **kwargs): + self.delete_calls.append({"ids": ids, **kwargs}) + if ids: + for chunk_id in ids: + self.rows.pop(chunk_id, None) + + def query(self, query_embeddings, n_results, include): + self.query_calls.append( + { + "query_embeddings": query_embeddings, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[0.0 + index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + + def count(self): + return len(self.rows) + + def refresh_index(self): + self.refresh_calls += 1 + + +class FakeClient: + instances = [] + + def __init__(self, path: str) -> None: + self.path = path + self.collections = {} + self.created = [] + FakeClient.instances.append(self) + + def get_or_create_collection(self, name, configuration=None, embedding_function=None): + if embedding_function is not None: + raise AssertionError("explicit LiteLLM embeddings require embedding_function=None") + if configuration is None: + raise AssertionError("native SeekDB collection must declare vector dimension") + if name not in self.collections: + self.created.append((name, configuration.dimension)) + self.collections[name] = FakeCollection(name, configuration.dimension) + return self.collections[name] + + def get_collection(self, name, embedding_function=None): + if name not in self.collections: + raise KeyError(name) + return self.collections[name] + + +@pytest.fixture(autouse=True) +def fake_pyseekdb(monkeypatch, request): + if request.node.name == "test_real_pyseekdb_upsert_refresh_and_query_round_trip": + yield + return + FakeClient.instances.clear() + module = types.SimpleNamespace(Client=FakeClient, HNSWConfiguration=FakeHNSWConfiguration) + monkeypatch.setitem(sys.modules, "pyseekdb", module) + yield + + +def chunk(chunk_id: str, source_id: str, embedding: list[float]) -> KnowledgeChunk: + return KnowledgeChunk( + id=chunk_id, + source_id=source_id, + content=f"{chunk_id} content", + chunk_index=0, + embedding=embedding, + metadata={"source_id": source_id}, + ) + + +def test_upsert_creates_dimensioned_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + client = FakeClient.instances[0] + assert client.created == [(index.collection_name("source-a"), 3)] + collection = client.collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["ids"] == ["chunk-a"] + assert collection.upsert_calls[0]["embeddings"] == [[0.1, 0.2, 0.3]] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert "payload" in collection.upsert_calls[0]["metadatas"][0] + assert collection.refresh_calls == 1 + + +def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("source-b", [chunk("chunk-b", "source-b", [0.9, 0.1, 0.1])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a", "source-b"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert all(item["backend"] == "seekdb" for item in results) + client = FakeClient.instances[0] + assert client.collections[index.collection_name("source-a")].query_calls + assert client.collections[index.collection_name("source-b")].query_calls + + +def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: Path, monkeypatch): + monkeypatch.setitem(sys.modules, "pyseekdb", None) + + with pytest.raises(SeekDBUnavailableError): + SeekDBChunkIndex(tmp_path / "native.seekdb") + + +def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path, monkeypatch): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=1, + ) + + assert results[0]["chunk"].id == "real-chunk" From 8f1d7a3da18004bdf9515f439ae3fd92feb53b9d Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 20:05:38 +0800 Subject: [PATCH 02/22] fix(seekdb): refresh and replace chunk index writes --- .../vector_stores/seekdb_chunk_index.py | 4 + tests/test_seekdb_chunk_index.py | 84 ++++++++++++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index d534fed..673714c 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -57,6 +57,8 @@ def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> embeddings: list[list[float]] = [] metadatas: list[dict[str, Any]] = [] for chunk in chunks: + if chunk.source_id != source_id: + raise ValueError("chunk.source_id must match source_id") if not chunk.embedding or len(chunk.embedding) != dimension: raise ValueError("All chunks must have same-dimension non-empty embeddings") embeddings.append(chunk.embedding) @@ -67,6 +69,7 @@ def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> metadatas.append(metadata) collection = self._collection(source_id, dimension=dimension) + collection.delete(where={"source_id": source_id}) collection.upsert( ids=[chunk.id for chunk in chunks], documents=[chunk.content for chunk in chunks], @@ -81,6 +84,7 @@ def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: try: collection = self._collection(source_id) collection.delete(ids=chunk_ids) + collection.refresh_index() except Exception as exc: logger.warning("Failed to delete SeekDB chunks for source %s: %s", source_id, exc) diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index f0f0b59..f5c77d9 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -24,10 +24,12 @@ def __init__(self, name: str, dimension: int) -> None: self.upsert_calls = [] self.delete_calls = [] self.query_calls = [] + self.events = [] self.refresh_calls = 0 self.rows = {} def upsert(self, ids, documents, metadatas, embeddings): + self.events.append({"op": "upsert", "ids": ids}) self.upsert_calls.append( { "ids": ids, @@ -44,10 +46,16 @@ def upsert(self, ids, documents, metadatas, embeddings): } def delete(self, ids=None, **kwargs): + self.events.append({"op": "delete", "ids": ids, **kwargs}) self.delete_calls.append({"ids": ids, **kwargs}) if ids: for chunk_id in ids: self.rows.pop(chunk_id, None) + where = kwargs.get("where") + if where: + for chunk_id, row in list(self.rows.items()): + if all(row["metadata"].get(key) == value for key, value in where.items()): + self.rows.pop(chunk_id, None) def query(self, query_embeddings, n_results, include): self.query_calls.append( @@ -99,7 +107,7 @@ def get_collection(self, name, embedding_function=None): @pytest.fixture(autouse=True) def fake_pyseekdb(monkeypatch, request): - if request.node.name == "test_real_pyseekdb_upsert_refresh_and_query_round_trip": + if request.node.name.startswith("test_real_pyseekdb"): yield return FakeClient.instances.clear() @@ -148,8 +156,56 @@ def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Pat assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] assert all(item["backend"] == "seekdb" for item in results) client = FakeClient.instances[0] - assert client.collections[index.collection_name("source-a")].query_calls - assert client.collections[index.collection_name("source-b")].query_calls + source_a_query = client.collections[index.collection_name("source-a")].query_calls[0] + source_b_query = client.collections[index.collection_name("source-b")].query_calls[0] + assert source_a_query["query_embeddings"] == [[0.1, 0.2, 0.3]] + assert source_a_query["n_results"] == 2 + assert source_a_query["include"] == ["documents", "metadatas", "distances"] + assert source_b_query["query_embeddings"] == [[0.1, 0.2, 0.3]] + assert source_b_query["n_results"] == 2 + assert source_b_query["include"] == ["documents", "metadatas", "distances"] + + +def test_delete_source_chunks_refreshes_collection_after_successful_delete(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.delete_source_chunks("source-a", ["chunk-a"]) + + assert collection.delete_calls[-1]["ids"] == ["chunk-a"] + assert collection.rows == {} + assert collection.refresh_calls == 2 + + +def test_upsert_source_chunks_replaces_existing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]), + chunk("chunk-b", "source-a", [0.3, 0.2, 0.1]), + ], + ) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + assert collection.delete_calls[-1]["where"] == {"source_id": "source-a"} + assert [event["op"] for event in collection.events[-2:]] == ["delete", "upsert"] + + +def test_upsert_source_chunks_rejects_mismatched_source_id(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + with pytest.raises(ValueError, match="source_id"): + index.upsert_source_chunks("source-a", [chunk("chunk-b", "source-b", [0.1, 0.2, 0.3])]) def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: Path, monkeypatch): @@ -172,3 +228,25 @@ def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path, monke ) assert results[0]["chunk"].id == "real-chunk" + + +def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path, monkeypatch): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks( + "real-source", + [ + chunk("real-a", "real-source", [0.1, 0.2, 0.3]), + chunk("real-b", "real-source", [0.3, 0.2, 0.1]), + ], + ) + index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["real-source"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["real-c"] From 67ddfea7cc4a72f62ab6226a4c0cf10db0fdad34 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 20:18:45 +0800 Subject: [PATCH 03/22] fix(seekdb): make source replacement authoritative --- .../vector_stores/seekdb_chunk_index.py | 10 +- tests/test_seekdb_chunk_index.py | 111 +++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index 673714c..1b2fc1a 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -53,6 +53,12 @@ def _collection(self, source_id: str, dimension: int | None = None) -> Any: ) def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if not chunks: + collection = self._collection(source_id) + collection.delete(where={"source_id": source_id}) + collection.refresh_index() + return + dimension = self._first_embedding_dimension(chunks) embeddings: list[list[float]] = [] metadatas: list[dict[str, Any]] = [] @@ -63,8 +69,8 @@ def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> raise ValueError("All chunks must have same-dimension non-empty embeddings") embeddings.append(chunk.embedding) metadata = dict(chunk.metadata) - metadata.setdefault("source_id", chunk.source_id) - metadata.setdefault("chunk_index", chunk.chunk_index) + metadata["source_id"] = chunk.source_id + metadata["chunk_index"] = chunk.chunk_index metadata["payload"] = chunk.model_dump_json() metadatas.append(metadata) diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index f5c77d9..073afc4 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -116,14 +116,19 @@ def fake_pyseekdb(monkeypatch, request): yield -def chunk(chunk_id: str, source_id: str, embedding: list[float]) -> KnowledgeChunk: +def chunk( + chunk_id: str, + source_id: str, + embedding: list[float], + metadata: dict | None = None, +) -> KnowledgeChunk: return KnowledgeChunk( id=chunk_id, source_id=source_id, content=f"{chunk_id} content", chunk_index=0, embedding=embedding, - metadata={"source_id": source_id}, + metadata=metadata if metadata is not None else {"source_id": source_id}, ) @@ -201,6 +206,57 @@ def test_upsert_source_chunks_replaces_existing_source_collection(tmp_path: Path assert [event["op"] for event in collection.events[-2:]] == ["delete", "upsert"] +def test_upsert_source_chunks_overrides_conflicting_metadata_source_id_for_replacement(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk( + "chunk-a", + "source-a", + [0.1, 0.2, 0.3], + metadata={"source_id": "wrong-source", "user_label": "kept"}, + ), + chunk( + "chunk-b", + "source-a", + [0.3, 0.2, 0.1], + metadata={"source_id": "wrong-source"}, + ), + ], + ) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert collection.upsert_calls[0]["metadatas"][0]["user_label"] == "kept" + + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + + +def test_upsert_source_chunks_empty_list_clears_existing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + index.upsert_source_chunks("source-a", []) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=10, + ) + + assert results == [] + assert collection.delete_calls[-1]["where"] == {"source_id": "source-a"} + assert collection.refresh_calls == 2 + + def test_upsert_source_chunks_rejects_mismatched_source_id(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") @@ -250,3 +306,54 @@ def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path, monkeypatch ) assert [item["chunk"].id for item in results] == ["real-c"] + + +def test_real_pyseekdb_conflicting_metadata_source_id_does_not_leave_stale_chunks( + tmp_path: Path, + monkeypatch, +): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks( + "real-source", + [ + chunk( + "real-a", + "real-source", + [0.1, 0.2, 0.3], + metadata={"source_id": "wrong-source"}, + ), + chunk( + "real-b", + "real-source", + [0.3, 0.2, 0.1], + metadata={"source_id": "wrong-source"}, + ), + ], + ) + index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["real-source"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["real-c"] + + +def test_real_pyseekdb_empty_reindex_clears_existing_chunks(tmp_path: Path, monkeypatch): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("real-source", []) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=10, + ) + + assert results == [] From 857c5a5f6f086f66fcc9eb2e399c4f58f6f4f6d0 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 20:34:49 +0800 Subject: [PATCH 04/22] fix(seekdb): make empty replacement idempotent --- .../vector_stores/seekdb_chunk_index.py | 29 ++- tests/test_seekdb_chunk_index.py | 213 ++++++++++++------ 2 files changed, 167 insertions(+), 75 deletions(-) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index 1b2fc1a..530cd41 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -52,9 +52,32 @@ def _collection(self, source_id: str, dimension: int | None = None) -> Any: embedding_function=None, ) + def _existing_collection(self, source_id: str) -> Any | None: + try: + return self._collection(source_id) + except Exception as exc: + if self._is_missing_collection_error(exc): + return None + raise + + @staticmethod + def _is_missing_collection_error(exc: Exception) -> bool: + if isinstance(exc, KeyError): + return True + message = str(exc).lower() + missing_markers = ( + "not found", + "not exist", + "doesn't exist", + "failed to resolve collection metadata", + ) + return "collection" in message and any(marker in message for marker in missing_markers) + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: if not chunks: - collection = self._collection(source_id) + collection = self._existing_collection(source_id) + if collection is None: + return collection.delete(where={"source_id": source_id}) collection.refresh_index() return @@ -88,7 +111,9 @@ def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: if not chunk_ids: return try: - collection = self._collection(source_id) + collection = self._existing_collection(source_id) + if collection is None: + return collection.delete(ids=chunk_ids) collection.refresh_index() except Exception as exc: diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 073afc4..e70a214 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -1,4 +1,7 @@ +import os +import subprocess import sys +import textwrap import types from pathlib import Path @@ -257,6 +260,25 @@ def test_upsert_source_chunks_empty_list_clears_existing_source_collection(tmp_p assert collection.refresh_calls == 2 +def test_upsert_source_chunks_empty_list_missing_collection_is_noop(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", []) + + client = FakeClient.instances[0] + assert index.collection_name("source-a") not in client.collections + + +def test_delete_source_chunks_missing_collection_is_noop(tmp_path: Path, caplog): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.delete_source_chunks("source-a", ["chunk-a"]) + + client = FakeClient.instances[0] + assert index.collection_name("source-a") not in client.collections + assert not [record for record in caplog.records if record.levelname == "WARNING"] + + def test_upsert_source_chunks_rejects_mismatched_source_id(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") @@ -271,89 +293,134 @@ def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: SeekDBChunkIndex(tmp_path / "native.seekdb") -def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path, monkeypatch): - pyseekdb = pytest.importorskip("pyseekdb") - monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) - index = SeekDBChunkIndex(tmp_path / "real.seekdb") - - index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) - results = index.search( - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=1, +def run_real_pyseekdb_subprocess(script: str) -> None: + repo_root = Path(__file__).resolve().parents[1] + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + value for value in [str(repo_root), env.get("PYTHONPATH", "")] if value + ) + subprocess_script = textwrap.dedent( + """ + try: + import pyseekdb # noqa: F401 + except Exception: + print("PYSEEKDB_UNAVAILABLE") + raise SystemExit(0) + + from backend.domain.source import KnowledgeChunk + from backend.infrastructure.vector_stores.seekdb_chunk_index import SeekDBChunkIndex + + + def chunk(chunk_id: str, source_id: str, embedding: list[float], metadata: dict | None = None): + return KnowledgeChunk( + id=chunk_id, + source_id=source_id, + content=f"{chunk_id} content", + chunk_index=0, + embedding=embedding, + metadata=metadata if metadata is not None else {"source_id": source_id}, + ) + """ + ) + "\n" + textwrap.dedent(script) + try: + completed = subprocess.run( + [sys.executable, "-c", subprocess_script], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + except subprocess.TimeoutExpired: + pytest.skip("real pyseekdb subprocess timed out after 60 seconds") + if "PYSEEKDB_UNAVAILABLE" in completed.stdout: + pytest.skip("pyseekdb is unavailable in subprocess") + assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=1, + ) + assert results[0]["chunk"].id == "real-chunk" + """ ) - assert results[0]["chunk"].id == "real-chunk" - - -def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path, monkeypatch): - pyseekdb = pytest.importorskip("pyseekdb") - monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) - index = SeekDBChunkIndex(tmp_path / "real.seekdb") - index.upsert_source_chunks( - "real-source", - [ - chunk("real-a", "real-source", [0.1, 0.2, 0.3]), - chunk("real-b", "real-source", [0.3, 0.2, 0.1]), - ], - ) - index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) - results = index.search( - query_embedding=[0.9, 0.1, 0.1], - source_ids=["real-source"], - top_k=10, +def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks( + "real-source", + [ + chunk("real-a", "real-source", [0.1, 0.2, 0.3]), + chunk("real-b", "real-source", [0.3, 0.2, 0.1]), + ], + ) + index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["real-source"], + top_k=10, + ) + assert [item["chunk"].id for item in results] == ["real-c"] + """ ) - assert [item["chunk"].id for item in results] == ["real-c"] - def test_real_pyseekdb_conflicting_metadata_source_id_does_not_leave_stale_chunks( tmp_path: Path, - monkeypatch, ): - pyseekdb = pytest.importorskip("pyseekdb") - monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) - index = SeekDBChunkIndex(tmp_path / "real.seekdb") - - index.upsert_source_chunks( - "real-source", - [ - chunk( - "real-a", - "real-source", - [0.1, 0.2, 0.3], - metadata={"source_id": "wrong-source"}, - ), - chunk( - "real-b", - "real-source", - [0.3, 0.2, 0.1], - metadata={"source_id": "wrong-source"}, - ), - ], - ) - index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) - results = index.search( - query_embedding=[0.9, 0.1, 0.1], - source_ids=["real-source"], - top_k=10, + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks( + "real-source", + [ + chunk( + "real-a", + "real-source", + [0.1, 0.2, 0.3], + metadata={{"source_id": "wrong-source"}}, + ), + chunk( + "real-b", + "real-source", + [0.3, 0.2, 0.1], + metadata={{"source_id": "wrong-source"}}, + ), + ], + ) + index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["real-source"], + top_k=10, + ) + assert [item["chunk"].id for item in results] == ["real-c"] + """ ) - assert [item["chunk"].id for item in results] == ["real-c"] - -def test_real_pyseekdb_empty_reindex_clears_existing_chunks(tmp_path: Path, monkeypatch): - pyseekdb = pytest.importorskip("pyseekdb") - monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) - index = SeekDBChunkIndex(tmp_path / "real.seekdb") - - index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) - index.upsert_source_chunks("real-source", []) - results = index.search( - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=10, +def test_real_pyseekdb_empty_reindex_clears_existing_chunks(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("real-source", []) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=10, + ) + assert results == [] + """ ) - - assert results == [] From e5dbf328a25e1915c33cf6ac416737aea48157a2 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 20:43:46 +0800 Subject: [PATCH 05/22] fix(seekdb): preserve replacement semantics --- .../vector_stores/seekdb_chunk_index.py | 41 +++++-- tests/test_seekdb_chunk_index.py | 108 +++++++++++++++++- 2 files changed, 139 insertions(+), 10 deletions(-) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index 530cd41..47c2a59 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -73,13 +73,23 @@ def _is_missing_collection_error(exc: Exception) -> bool: ) return "collection" in message and any(marker in message for marker in missing_markers) + @staticmethod + def _collection_ids(collection: Any) -> list[str]: + result = collection.get(include=[]) + ids = result.get("ids", []) + if isinstance(ids, str): + return [ids] + return list(ids or []) + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + existing_collection = self._existing_collection(source_id) if not chunks: - collection = self._existing_collection(source_id) - if collection is None: + if existing_collection is None: return - collection.delete(where={"source_id": source_id}) - collection.refresh_index() + existing_ids = self._collection_ids(existing_collection) + if existing_ids: + existing_collection.delete(ids=existing_ids) + existing_collection.refresh_index() return dimension = self._first_embedding_dimension(chunks) @@ -97,15 +107,32 @@ def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> metadata["payload"] = chunk.model_dump_json() metadatas.append(metadata) - collection = self._collection(source_id, dimension=dimension) - collection.delete(where={"source_id": source_id}) + if existing_collection is not None: + existing_dimension = getattr(existing_collection, "dimension", None) + if existing_dimension is not None and existing_dimension != dimension: + raise ValueError( + f"Existing SeekDB collection dimension {existing_dimension} " + f"does not match chunk embedding dimension {dimension}" + ) + collection = existing_collection + existing_ids = self._collection_ids(collection) + else: + collection = self._collection(source_id, dimension=dimension) + existing_ids = [] + + new_ids = [chunk.id for chunk in chunks] collection.upsert( - ids=[chunk.id for chunk in chunks], + ids=new_ids, documents=[chunk.content for chunk in chunks], metadatas=metadatas, embeddings=embeddings, ) collection.refresh_index() + new_id_set = set(new_ids) + stale_ids = [chunk_id for chunk_id in existing_ids if chunk_id not in new_id_set] + if stale_ids: + collection.delete(ids=stale_ids) + collection.refresh_index() def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: if not chunk_ids: diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index e70a214..70a338c 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -79,6 +79,9 @@ def query(self, query_embeddings, n_results, include): def count(self): return len(self.rows) + def get(self, include=None, **kwargs): + return {"ids": list(self.rows)} + def refresh_index(self): self.refresh_calls += 1 @@ -205,8 +208,54 @@ def test_upsert_source_chunks_replaces_existing_source_collection(tmp_path: Path ) assert [item["chunk"].id for item in results] == ["chunk-c"] - assert collection.delete_calls[-1]["where"] == {"source_id": "source-a"} - assert [event["op"] for event in collection.events[-2:]] == ["delete", "upsert"] + assert collection.delete_calls[-1]["ids"] == ["chunk-a", "chunk-b"] + assert [event["op"] for event in collection.events[-2:]] == ["upsert", "delete"] + + +def test_upsert_source_chunks_removes_stale_bad_metadata_by_collection_membership(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + stale = chunk( + "stale-chunk", + "source-a", + [0.2, 0.2, 0.2], + metadata={"source_id": "wrong-source"}, + ) + collection.upsert( + ids=[stale.id], + documents=[stale.content], + metadatas=[{"source_id": "wrong-source", "payload": stale.model_dump_json()}], + embeddings=[stale.embedding], + ) + collection.refresh_index() + + index.upsert_source_chunks("source-a", [chunk("chunk-c", "source-a", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-c"] + assert collection.delete_calls[-1]["ids"] == ["chunk-a", "stale-chunk"] + + +def test_upsert_source_chunks_dimension_mismatch_preserves_existing_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + with pytest.raises(ValueError, match="dimension"): + index.upsert_source_chunks("source-a", [chunk("chunk-b", "source-a", [0.1, 0.2])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=10, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] def test_upsert_source_chunks_overrides_conflicting_metadata_source_id_for_replacement(tmp_path: Path): @@ -256,7 +305,7 @@ def test_upsert_source_chunks_empty_list_clears_existing_source_collection(tmp_p ) assert results == [] - assert collection.delete_calls[-1]["where"] == {"source_id": "source-a"} + assert collection.delete_calls[-1]["ids"] == ["chunk-a"] assert collection.refresh_calls == 2 @@ -424,3 +473,56 @@ def test_real_pyseekdb_empty_reindex_clears_existing_chunks(tmp_path: Path): assert results == [] """ ) + + +def test_real_pyseekdb_stale_bad_metadata_row_is_removed_by_collection_membership(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) + stale = chunk( + "real-stale", + "real-source", + [0.2, 0.2, 0.2], + metadata={{"source_id": "wrong-source"}}, + ) + collection = index._collection("real-source", dimension=3) + collection.upsert( + ids=[stale.id], + documents=[stale.content], + metadatas=[{{"source_id": "wrong-source", "payload": stale.model_dump_json()}}], + embeddings=[stale.embedding], + ) + collection.refresh_index() + + index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) + results = index.search( + query_embedding=[0.9, 0.1, 0.1], + source_ids=["real-source"], + top_k=10, + ) + assert [item["chunk"].id for item in results] == ["real-c"] + """ + ) + + +def test_real_pyseekdb_dimension_mismatch_preserves_existing_chunks(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) + try: + index.upsert_source_chunks("real-source", [chunk("real-b", "real-source", [0.1, 0.2])]) + except ValueError: + pass + else: + raise AssertionError("dimension mismatch did not raise") + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=10, + ) + assert [item["chunk"].id for item in results] == ["real-a"] + """ + ) From cf83148a57137a2b22a1333f01044698030cdc7f Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 20:56:04 +0800 Subject: [PATCH 06/22] feat(seekdb): make native vector index primary --- backend/config.py | 8 +- backend/dependencies.py | 5 +- .../repositories/seekdb_repository.py | 127 ++++++++++++++---- config_example.yaml | 1 + tests/test_config_settings.py | 14 ++ tests/test_source_knowledge_base.py | 99 ++++++++++++++ 6 files changed, 223 insertions(+), 31 deletions(-) diff --git a/backend/config.py b/backend/config.py index 273f16d..d06deeb 100644 --- a/backend/config.py +++ b/backend/config.py @@ -84,6 +84,7 @@ class Settings(BaseSettings): vector_store_type: Literal["chroma", "faiss", "seekdb"] = "seekdb" chroma_persist_dir: str = "./data/chroma" seekdb_path: str = "./data/seekdb.db" + seekdb_allow_sqlite_fallback: bool = False # 嵌入模型设置 (OpenAI API) embedding_model: str = "text-embedding-3-small" @@ -138,7 +139,12 @@ def _flatten_config(data: dict[str, Any]) -> dict[str, Any]: storage = data.get("storage") or {} if isinstance(storage, dict): - for key in ("vector_store_type", "chroma_persist_dir", "seekdb_path"): + for key in ( + "vector_store_type", + "chroma_persist_dir", + "seekdb_path", + "seekdb_allow_sqlite_fallback", + ): if key in storage: flattened[key] = storage[key] diff --git a/backend/dependencies.py b/backend/dependencies.py index e073f71..1d433fb 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -88,7 +88,10 @@ def get_knowledge_repository( if cls._knowledge_repository is None or force_new: settings = settings or get_settings() from .infrastructure.repositories.seekdb_repository import SeekDBRepository - cls._knowledge_repository = SeekDBRepository(settings.seekdb_path) + cls._knowledge_repository = SeekDBRepository( + settings.seekdb_path, + allow_sqlite_vector_fallback=settings.seekdb_allow_sqlite_fallback, + ) return cls._knowledge_repository @classmethod diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 6396f7b..6a964aa 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -1,9 +1,8 @@ -"""SeekDB repository wrapper with embedded pyseekdb preference. +"""SeekDB repository wrapper. -The pyseekdb SDK is loaded lazily. When it is unavailable, the repository keeps -the same durable interface on top of SQLite so development and tests remain -deterministic while deployment can use pyseekdb embedded mode by installing the -SDK. +SQLite stores source and chunk payload metadata. Native SeekDB owns chunk vector +indexing whenever embeddings are present; SQLite vector scoring is retained only +as an explicit fallback path. """ from __future__ import annotations @@ -19,9 +18,15 @@ from ...core.interfaces.knowledge_repository import KnowledgeRepositoryInterface from ...domain.slide_deck import SlideAsset, SlideDeckExport, SlideDeckJob, SlideDeckProject from ...domain.source import Artifact, Job, KnowledgeChunk, KnowledgeSource, Note, SourceStatus, utc_now +from ..vector_stores.seekdb_chunk_index import SeekDBChunkIndex logger = logging.getLogger(__name__) +_AUTO_NATIVE_CHUNK_INDEX: Any = object() +_NATIVE_UNAVAILABLE_MESSAGE = ( + "native SeekDB vector index is unavailable; enable seekdb_allow_sqlite_fallback for SQLite fallback" +) + def _dump_model(model: Any) -> str: return model.model_dump_json() @@ -80,23 +85,31 @@ def _bm25_scores(query: str, documents: list[str]) -> list[float]: class SeekDBRepository(KnowledgeRepositoryInterface): - def __init__(self, db_path: str | Path) -> None: + def __init__( + self, + db_path: str | Path, + native_chunk_index: object | None = _AUTO_NATIVE_CHUNK_INDEX, + allow_sqlite_vector_fallback: bool = False, + ) -> None: self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) - self.seekdb_client = self._try_pyseekdb(self.db_path.parent / f"{self.db_path.stem}.seekdb") + self.seekdb_path = self.db_path.parent / f"{self.db_path.stem}.seekdb" + self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback + self.native_chunk_index = ( + self._try_native_chunk_index(self.seekdb_path) + if native_chunk_index is _AUTO_NATIVE_CHUNK_INDEX + else native_chunk_index + ) self._conn = sqlite3.connect(self.db_path, check_same_thread=False) self._conn.row_factory = sqlite3.Row self._init_schema() - @staticmethod - def _try_pyseekdb(path: Path) -> Any | None: - try: - import pyseekdb # type: ignore - except Exception: - return None + def _try_native_chunk_index(self, path: Path) -> Any | None: try: - return pyseekdb.Client(path=str(path)) - except Exception: + return SeekDBChunkIndex(path) + except Exception as exc: + if not self.allow_sqlite_vector_fallback: + logger.warning("Native SeekDB vector index is unavailable: %s", exc) return None def _init_schema(self) -> None: @@ -174,6 +187,18 @@ def _init_schema(self) -> None: async def close(self) -> None: self._conn.close() + def storage_status(self) -> dict[str, Any]: + if self.native_chunk_index is not None: + status = dict(self.native_chunk_index.status()) + status.setdefault("seekdb_path", str(self.seekdb_path)) + status.setdefault("native_available", True) + return status + return { + "vector_backend": "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable", + "seekdb_path": str(self.seekdb_path), + "native_available": False, + } + async def save_source(self, source: KnowledgeSource) -> KnowledgeSource: self._conn.execute( """ @@ -207,6 +232,10 @@ async def list_sources(self) -> list[KnowledgeSource]: return [KnowledgeSource.model_validate_json(row["payload"]) for row in rows] async def delete_source(self, source_id: str) -> bool: + chunk_rows = self._conn.execute("SELECT id FROM chunks WHERE source_id = ?", (source_id,)).fetchall() + chunk_ids = [row["id"] for row in chunk_rows] + if self.native_chunk_index is not None: + self.native_chunk_index.delete_source_chunks(source_id, chunk_ids) self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) self._conn.commit() @@ -233,19 +262,16 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non ) self._conn.commit() - if self.seekdb_client is not None and chunks: # pragma: no cover - requires SDK/runtime - try: - collection = self.seekdb_client.get_or_create_collection(name="chunks") - payload: dict[str, Any] = { - "ids": [chunk.id for chunk in chunks], - "documents": [chunk.content for chunk in chunks], - "metadatas": [chunk.metadata for chunk in chunks], - } - if all(chunk.embedding is not None for chunk in chunks): - payload["embeddings"] = [chunk.embedding for chunk in chunks] - collection.upsert(**payload) - except Exception as exc: - logger.warning("Skipping optional pyseekdb chunk mirror after save failure: %s", exc) + has_vector_chunks = any(chunk.embedding is not None for chunk in chunks) + if self.native_chunk_index is not None: + if chunks and has_vector_chunks: + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + elif not chunks: + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + return + + if has_vector_chunks and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( @@ -262,14 +288,48 @@ async def search_chunks( query_embedding: list[float] | None = None, rerank_provider: object | None = None, ) -> list[dict]: + if self.native_chunk_index is not None and query_embedding is not None: + selected_source_ids = source_ids or self._current_source_ids() + results = self.native_chunk_index.search(query_embedding, selected_source_ids, top_k) + return await self._maybe_rerank(query, results, top_k, rerank_provider) + + rows = self._chunk_rows(source_ids) + has_sqlite_vectors = any(row["embedding"] for row in rows) + if self.native_chunk_index is not None and has_sqlite_vectors and not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB vector search requires query embeddings") + if ( + self.native_chunk_index is None + and (query_embedding is not None or has_sqlite_vectors) + and not self.allow_sqlite_vector_fallback + ): + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + + return await self._search_sqlite_chunk_rows(query, rows, top_k, query_embedding, rerank_provider) + + def _current_source_ids(self) -> list[str]: + rows = self._conn.execute( + "SELECT id FROM sources WHERE status != ? ORDER BY created_at DESC", + (SourceStatus.DELETED.value,), + ).fetchall() + return [row["id"] for row in rows] + + def _chunk_rows(self, source_ids: list[str] | None = None) -> list[sqlite3.Row]: params: list[Any] = [] sql = "SELECT payload, content, embedding FROM chunks" if source_ids: placeholders = ",".join("?" for _ in source_ids) sql += f" WHERE source_id IN ({placeholders})" params.extend(source_ids) - rows = self._conn.execute(sql, params).fetchall() + return self._conn.execute(sql, params).fetchall() + async def _search_sqlite_chunk_rows( + self, + query: str, + rows: list[sqlite3.Row], + top_k: int, + query_embedding: list[float] | None, + rerank_provider: object | None, + ) -> list[dict]: bm25_scores = _bm25_scores(query, [row["content"] for row in rows]) results = [] for row, bm25_score in zip(rows, bm25_scores): @@ -282,6 +342,15 @@ async def search_chunks( results.append({"chunk": chunk, "score": score}) results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + return await self._maybe_rerank(query, results, top_k, rerank_provider) + + async def _maybe_rerank( + self, + query: str, + results: list[dict], + top_k: int, + rerank_provider: object | None, + ) -> list[dict]: if rerank_provider and results: try: return await rerank_provider.rerank(query, results, top_k=top_k) diff --git a/config_example.yaml b/config_example.yaml index f322550..5aecbb6 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -48,6 +48,7 @@ api: storage: vector_store_type: "seekdb" seekdb_path: "./data/seekdb.db" + seekdb_allow_sqlite_fallback: false chroma_persist_dir: "./data/chroma" documents: diff --git a/tests/test_config_settings.py b/tests/test_config_settings.py index 1adedfd..7564911 100644 --- a/tests/test_config_settings.py +++ b/tests/test_config_settings.py @@ -24,6 +24,20 @@ def test_loads_yaml_model_profiles_without_using_local_config(sample_config_file assert settings.output_dir == "./output-test" +def test_seekdb_native_vector_search_is_default(monkeypatch, sample_config_file): + from backend.config import get_settings + + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + get_settings.cache_clear() + try: + settings = get_settings() + finally: + get_settings.cache_clear() + + assert settings.vector_store_type == "seekdb" + assert settings.seekdb_allow_sqlite_fallback is False + + def test_default_settings_do_not_require_local_secrets(monkeypatch): from backend.config import get_settings diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index a90a29e..8981cf8 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from pathlib import Path from types import ModuleType, SimpleNamespace import sys @@ -12,11 +13,109 @@ from backend.api.routes.sources import router as sources_router from backend.core.services.source_service import SourceService from backend.config import Settings, get_settings +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind from backend.dependencies import get_source_service from backend.infrastructure.parsers.docling_parser import DoclingParser from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + self.deletes = [] + self.searches = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + + def delete_source_chunks(self, source_id, chunk_ids): + self.deletes.append((source_id, chunk_ids)) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_chunk", + source_id=source_ids[0], + content="native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + +@pytest.mark.asyncio +async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp_path: Path, caplog): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native", kind=SourceKind.TEXT, title="Native") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native", + source_id=source.id, + content="native indexed content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with caplog.at_level(logging.WARNING): + results = await repo.search_chunks( + "native indexed", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts[0][0] == source.id + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "native_chunk" + assert "Skipping optional pyseekdb chunk mirror" not in caplog.text + + +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native", kind=SourceKind.TEXT, title="No Native") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native", + source_id=source.id, + content="content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + @pytest.mark.asyncio async def test_text_source_is_chunked_with_citation_metadata(tmp_path: Path): repo = SeekDBRepository(tmp_path / "knowledge.db") From 4485f943a889d75fe042cf45fa81856a560db617 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 21:03:53 +0800 Subject: [PATCH 07/22] fix(seekdb): require explicit sqlite vector fallback --- .../repositories/seekdb_repository.py | 19 ++-- tests/test_chunking_and_rerank.py | 8 +- tests/test_rag_chat.py | 28 +++--- tests/test_slide_deck_domain_repository.py | 29 +++--- tests/test_source_knowledge_base.py | 93 +++++++++++++++++-- 5 files changed, 126 insertions(+), 51 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 6a964aa..fb397da 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -_AUTO_NATIVE_CHUNK_INDEX: Any = object() +_AUTO_NATIVE_INDEX: Any = object() _NATIVE_UNAVAILABLE_MESSAGE = ( "native SeekDB vector index is unavailable; enable seekdb_allow_sqlite_fallback for SQLite fallback" ) @@ -88,7 +88,7 @@ class SeekDBRepository(KnowledgeRepositoryInterface): def __init__( self, db_path: str | Path, - native_chunk_index: object | None = _AUTO_NATIVE_CHUNK_INDEX, + native_chunk_index: object = _AUTO_NATIVE_INDEX, allow_sqlite_vector_fallback: bool = False, ) -> None: self.db_path = Path(db_path) @@ -97,7 +97,7 @@ def __init__( self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback self.native_chunk_index = ( self._try_native_chunk_index(self.seekdb_path) - if native_chunk_index is _AUTO_NATIVE_CHUNK_INDEX + if native_chunk_index is _AUTO_NATIVE_INDEX else native_chunk_index ) self._conn = sqlite3.connect(self.db_path, check_same_thread=False) @@ -270,7 +270,7 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non self.native_chunk_index.upsert_source_chunks(source_id, chunks) return - if has_vector_chunks and not self.allow_sqlite_vector_fallback: + if chunks and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: @@ -293,17 +293,12 @@ async def search_chunks( results = self.native_chunk_index.search(query_embedding, selected_source_ids, top_k) return await self._maybe_rerank(query, results, top_k, rerank_provider) - rows = self._chunk_rows(source_ids) - has_sqlite_vectors = any(row["embedding"] for row in rows) - if self.native_chunk_index is not None and has_sqlite_vectors and not self.allow_sqlite_vector_fallback: + if self.native_chunk_index is not None and not self.allow_sqlite_vector_fallback: raise RuntimeError("native SeekDB vector search requires query embeddings") - if ( - self.native_chunk_index is None - and (query_embedding is not None or has_sqlite_vectors) - and not self.allow_sqlite_vector_fallback - ): + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + rows = self._chunk_rows(source_ids) return await self._search_sqlite_chunk_rows(query, rows, top_k, query_embedding, rerank_provider) def _current_source_ids(self) -> list[str]: diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index 4646e1c..e401f24 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -10,6 +10,10 @@ from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore +def sqlite_fallback_repo(path: Path) -> SeekDBRepository: + return SeekDBRepository(path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + + def test_chonkie_chunker_returns_offsets_and_counts(): pytest.importorskip("chonkie") service = ChunkingService( @@ -37,7 +41,7 @@ async def rerank(self, query, results, top_k=5): item["score"] = 10 - index return reranked - repo = SeekDBRepository(tmp_path / "rerank.db") + repo = sqlite_fallback_repo(tmp_path / "rerank.db") source = KnowledgeSource( id="source-1", kind=SourceKind.TEXT, @@ -75,7 +79,7 @@ async def rerank(self, query, results, top_k=5): @pytest.mark.asyncio async def test_seekdb_repository_uses_bm25_for_term_overlap_without_exact_phrase(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "bm25.db") + repo = sqlite_fallback_repo(tmp_path / "bm25.db") source = KnowledgeSource( id="source-bm25", kind=SourceKind.TEXT, diff --git a/tests/test_rag_chat.py b/tests/test_rag_chat.py index 91b50ed..31fe767 100644 --- a/tests/test_rag_chat.py +++ b/tests/test_rag_chat.py @@ -15,6 +15,10 @@ from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore +def sqlite_fallback_repo(path) -> SeekDBRepository: + return SeekDBRepository(path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + + class EchoLLM: async def generate_with_context(self, query, context, system_prompt=None, **kwargs): return f"answer: {query} :: {context[0]}" @@ -60,7 +64,7 @@ async def stream_generate_with_context(self, query, context, system_prompt=None, @pytest.mark.asyncio async def test_rag_retrieval_is_restricted_to_selected_sources(tmp_path): - repo = SeekDBRepository(tmp_path / "rag.db") + repo = sqlite_fallback_repo(tmp_path / "rag.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) first = await service.create_text_source("First", "Alpha selected material.") second = await service.create_text_source("Second", "Alpha forbidden material.") @@ -76,7 +80,7 @@ async def test_rag_retrieval_is_restricted_to_selected_sources(tmp_path): def test_chat_api_rejects_empty_sources_and_can_save_answer(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) @@ -92,7 +96,7 @@ def test_chat_api_rejects_empty_sources_and_can_save_answer(tmp_path): def test_chat_api_returns_citations_and_save_answer_as_source(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-save.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-save.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -118,7 +122,7 @@ def test_chat_api_returns_citations_and_save_answer_as_source(tmp_path): def test_chat_api_streams_answer_deltas_and_final_citations(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-stream.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-stream.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -145,7 +149,7 @@ def test_chat_api_streams_answer_deltas_and_final_citations(tmp_path): def test_chat_api_stream_falls_back_to_original_query_when_contextualized_retrieval_misses(tmp_path): - repo = SeekDBRepository(tmp_path / "api-rag-stream-fallback.db") + repo = sqlite_fallback_repo(tmp_path / "api-rag-stream-fallback.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) store = SeekDBVectorStore(repo) source = asyncio.run(asyncio_create_source(service)) @@ -176,7 +180,7 @@ def test_chat_api_stream_falls_back_to_original_query_when_contextualized_retrie @pytest.mark.asyncio async def test_rag_no_results_and_follow_up_stay_source_scoped(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-follow-up.db") + repo = sqlite_fallback_repo(tmp_path / "rag-follow-up.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) source = await service.create_text_source("Follow", "Beta follow up material.") await service.create_text_source("Other", "Beta forbidden follow up material.") @@ -199,7 +203,7 @@ async def test_rag_no_results_and_follow_up_stay_source_scoped(tmp_path): @pytest.mark.asyncio async def test_rag_multi_turn_uses_contextualized_query_only_for_retrieval(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-follow-up-original.db") + repo = sqlite_fallback_repo(tmp_path / "rag-follow-up-original.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) source = await service.create_text_source("Follow", "Alpha standalone retrieval material.") store = SeekDBVectorStore(repo) @@ -220,7 +224,7 @@ async def test_rag_multi_turn_uses_contextualized_query_only_for_retrieval(tmp_p @pytest.mark.asyncio async def test_rag_context_labels_sources_for_normal_user_file_comparison(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-source-labels.db") + repo = sqlite_fallback_repo(tmp_path / "rag-source-labels.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -247,7 +251,7 @@ async def test_rag_context_labels_sources_for_normal_user_file_comparison(tmp_pa @pytest.mark.asyncio async def test_rag_file_overview_includes_every_selected_source_even_when_bm25_matches_one_source(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-overview-balanced.db") + repo = sqlite_fallback_repo(tmp_path / "rag-overview-balanced.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -274,7 +278,7 @@ async def test_rag_file_overview_includes_every_selected_source_even_when_bm25_m @pytest.mark.asyncio async def test_rag_file_overview_includes_every_selected_source_when_top_k_is_smaller(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-overview-all-sources.db") + repo = sqlite_fallback_repo(tmp_path / "rag-overview-all-sources.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) sources = [ await service.create_text_source(f"Source {index}", f"Source {index} overview material.") @@ -297,7 +301,7 @@ async def test_rag_file_overview_includes_every_selected_source_when_top_k_is_sm @pytest.mark.asyncio async def test_rag_balances_context_across_selected_sources_for_cross_source_questions(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-cross-source-balanced.db") + repo = sqlite_fallback_repo(tmp_path / "rag-cross-source-balanced.db") service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=128, chunk_overlap=0) pdf_source = await service.create_text_source( "HTTPS.pdf", @@ -324,7 +328,7 @@ async def test_rag_balances_context_across_selected_sources_for_cross_source_que @pytest.mark.asyncio async def test_rag_targeted_summary_uses_retrieval_instead_of_first_chunks(tmp_path): - repo = SeekDBRepository(tmp_path / "rag-targeted-summary.db") + repo = sqlite_fallback_repo(tmp_path / "rag-targeted-summary.db") source = KnowledgeSource( kind=SourceKind.FILE, title="HTTPS.pdf", diff --git a/tests/test_slide_deck_domain_repository.py b/tests/test_slide_deck_domain_repository.py index 8b85837..28c708b 100644 --- a/tests/test_slide_deck_domain_repository.py +++ b/tests/test_slide_deck_domain_repository.py @@ -244,7 +244,7 @@ async def test_seekdb_repository_persists_slide_deck_state_after_restart(tmp_pat assert (await reopened.get_slide_deck_job(job.id)).status == JobStatus.FAILED -def test_seekdb_repository_uses_separate_embedded_path_for_pyseekdb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): +def test_seekdb_repository_uses_separate_embedded_path_for_native_seekdb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): calls: list[str] = [] class FakeClient: @@ -259,26 +259,18 @@ def __init__(self, path: str) -> None: assert calls == [str(tmp_path / "knowledge.seekdb")] assert db_path.is_file() - assert repo.seekdb_client is not None + assert repo.native_chunk_index is not None + assert repo.storage_status()["seekdb_path"] == str(tmp_path / "knowledge.seekdb") + assert repo.storage_status()["native_available"] is True @pytest.mark.asyncio -async def test_seekdb_repository_keeps_sqlite_chunks_when_optional_pyseekdb_mirror_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -): - class FailingCollection: - def upsert(self, **kwargs) -> None: - raise RuntimeError("default embedding model is unavailable") - - class FakeClient: - def __init__(self, path: str) -> None: - Path(path).mkdir(parents=True, exist_ok=True) - - def get_or_create_collection(self, name: str): - return FailingCollection() - - monkeypatch.setitem(sys.modules, "pyseekdb", types.SimpleNamespace(Client=FakeClient)) - repo = SeekDBRepository(tmp_path / "knowledge.db") +async def test_seekdb_repository_uses_sqlite_chunk_search_only_when_fallback_is_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) chunk = KnowledgeChunk( id="chunk_1", source_id="src_1", @@ -293,6 +285,7 @@ def get_or_create_collection(self, name: str): results = await repo.search_chunks("hybrid retrieval", top_k=1) assert [item.id for item in loaded] == ["chunk_1"] assert results[0]["chunk"].id == "chunk_1" + assert repo.storage_status()["vector_backend"] == "sqlite_fallback" def test_slide_deck_file_store_writes_ignored_files_and_returns_metadata(tmp_path: Path): diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 8981cf8..5006f97 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -116,9 +116,88 @@ async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback ) +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_non_vector_chunk_save_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native-text", kind=SourceKind.TEXT, title="No Native Text") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native-text", + source_id=source.id, + content="content without an embedding", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + +@pytest.mark.asyncio +async def test_repository_requires_query_embedding_when_native_seekdb_is_primary(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-query", kind=SourceKind.TEXT, title="Native Query") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native-query", + source_id=source.id, + content="native query content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + with pytest.raises(RuntimeError, match="native SeekDB vector search requires query embeddings"): + await repo.search_chunks("native query", source_ids=[source.id], top_k=1) + + +@pytest.mark.asyncio +async def test_repository_does_not_search_sqlite_bm25_without_explicit_fallback(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-bm25-disabled", kind=SourceKind.TEXT, title="BM25 Disabled") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-bm25-disabled", + source_id=source.id, + content="Needle lexical content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + repo.allow_sqlite_vector_fallback = False + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.search_chunks("Needle", source_ids=[source.id], top_k=1) + + @pytest.mark.asyncio async def test_text_source_is_chunked_with_citation_metadata(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService( repository=repo, parser=DoclingParser(), @@ -152,7 +231,7 @@ async def parse_file(self, file_path: str | Path, filename: str | None = None): async def parse_text(self, text: str, title: str = "Pasted text"): raise AssertionError("not used") - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=BrokenParser()) file_path = tmp_path / "broken.pdf" file_path.write_bytes(b"%PDF-1.4 broken") @@ -167,13 +246,13 @@ async def parse_text(self, text: str, title: str = "Pasted text"): @pytest.mark.asyncio async def test_seekdb_repository_persists_sources_after_restart(tmp_path: Path): db_path = tmp_path / "knowledge.db" - repo = SeekDBRepository(db_path) + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=40, chunk_overlap=0) created = await service.create_text_source("Durable", "Persistent source text for restart.") await repo.close() - reopened = SeekDBRepository(db_path) + reopened = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) loaded = await reopened.get_source(created.id) chunks = await reopened.get_chunks(created.id) @@ -185,7 +264,7 @@ async def test_seekdb_repository_persists_sources_after_restart(tmp_path: Path): @pytest.mark.asyncio async def test_delete_source_removes_chunks_from_retrieval(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "knowledge.db") + repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) source = await service.create_text_source("Delete me", "Needle phrase should disappear.") @@ -199,7 +278,7 @@ async def test_delete_source_removes_chunks_from_retrieval(tmp_path: Path): def test_sources_api_text_upload_list_get_delete(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "api.db") + repo = SeekDBRepository(tmp_path / "api.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) app = FastAPI() app.include_router(sources_router, prefix="/api") @@ -231,7 +310,7 @@ def test_sources_api_text_upload_list_get_delete(tmp_path: Path): def test_sources_api_file_upload_text_fallback(tmp_path: Path): - repo = SeekDBRepository(tmp_path / "upload.db") + repo = SeekDBRepository(tmp_path / "upload.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) app = FastAPI() app.include_router(sources_router, prefix="/api") From 3d103613c189b2c4bf529fe34149e4fd94cc4de3 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 21:16:04 +0800 Subject: [PATCH 08/22] fix(seekdb): enforce native embedding writes --- backend/dependencies.py | 1 + .../repositories/seekdb_repository.py | 8 +++ tests/test_runtime_config_api.py | 25 ++++++- tests/test_source_knowledge_base.py | 67 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/backend/dependencies.py b/backend/dependencies.py index 1d433fb..7c039a8 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -26,6 +26,7 @@ def reset_runtime_caches(cls) -> None: """Drop cached services that depend on model settings.""" cls._vector_store = None + cls._knowledge_repository = None cls._slide_deck_service = None @classmethod diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index fb397da..3702e07 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -242,6 +242,14 @@ async def delete_source(self, source_id: str) -> bool: return True async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if self.native_chunk_index is not None and chunks and not self.allow_sqlite_vector_fallback: + missing_embeddings = [chunk.id for chunk in chunks if not chunk.embedding] + if missing_embeddings: + raise RuntimeError( + "native SeekDB chunk writes require embeddings for all chunks; " + f"missing embeddings for: {', '.join(missing_embeddings)}" + ) + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) self._conn.executemany( """ diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index 047f8fc..d9d64f6 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -4,7 +4,7 @@ from fastapi.testclient import TestClient from backend.api.routes.config import router as config_router -from backend.config import _RUNTIME_MODEL_OVERRIDES, get_settings +from backend.config import Settings, _RUNTIME_MODEL_OVERRIDES, get_settings from backend.dependencies import DependencyContainer @@ -49,3 +49,26 @@ def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample _RUNTIME_MODEL_OVERRIDES.clear() get_settings.cache_clear() DependencyContainer.reset_runtime_caches() + + +def test_runtime_cache_reset_recreates_knowledge_repository_with_new_storage_settings(tmp_path): + DependencyContainer.reset_runtime_caches() + first_settings = Settings( + seekdb_path=str(tmp_path / "first.db"), + seekdb_allow_sqlite_fallback=True, + ) + first_repo = DependencyContainer.get_knowledge_repository(settings=first_settings) + + DependencyContainer.reset_runtime_caches() + second_settings = Settings( + seekdb_path=str(tmp_path / "second.db"), + seekdb_allow_sqlite_fallback=False, + ) + second_repo = DependencyContainer.get_knowledge_repository(settings=second_settings) + + try: + assert second_repo is not first_repo + assert second_repo.db_path == tmp_path / "second.db" + assert second_repo.allow_sqlite_vector_fallback is False + finally: + DependencyContainer.reset_runtime_caches() diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 5006f97..1c38205 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -52,6 +52,17 @@ def status(self): return {"vector_backend": "seekdb", "native_available": True} +class DeterministicEmbeddingProvider: + def __init__(self) -> None: + self.calls = [] + + async def embed(self, content): + self.calls.append(content) + if isinstance(content, str): + return [0.1, 0.2, 0.3] + return [[0.1, 0.2, 0.3] for _ in content] + + @pytest.mark.asyncio async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp_path: Path, caplog): native_index = RecordingNativeIndex() @@ -90,6 +101,31 @@ async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp assert "Skipping optional pyseekdb chunk mirror" not in caplog.text +@pytest.mark.asyncio +async def test_repository_requires_embeddings_when_native_seekdb_is_primary(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-no-embedding", kind=SourceKind.TEXT, title="Native No Embedding") + await repo.save_source(source) + + with pytest.raises((RuntimeError, ValueError), match="embedding|native SeekDB"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native-no-embedding", + source_id=source.id, + content="native primary requires an embedding", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + @pytest.mark.asyncio async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback_enabled(tmp_path: Path): repo = SeekDBRepository( @@ -159,6 +195,7 @@ async def test_repository_requires_query_embedding_when_native_seekdb_is_primary source_id=source.id, content="native query content", chunk_index=0, + embedding=[0.1, 0.2, 0.3], metadata={"source_id": source.id}, ) ], @@ -195,6 +232,36 @@ async def test_repository_does_not_search_sqlite_bm25_without_explicit_fallback( await repo.search_chunks("Needle", source_ids=[source.id], top_k=1) +@pytest.mark.asyncio +async def test_source_service_ingestion_and_vector_store_search_use_native_seekdb(tmp_path: Path): + native_index = RecordingNativeIndex() + embedding_provider = DeterministicEmbeddingProvider() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + service = SourceService( + repository=repo, + parser=DoclingParser(), + embedding_provider=embedding_provider, + chunk_size=128, + chunk_overlap=0, + ) + + source = await service.create_text_source("Native E2E", "Alpha native retrieval content.") + from backend.infrastructure.vector_stores.seekdb_vector_store import SeekDBVectorStore + + store = SeekDBVectorStore(repo, embedding_provider=embedding_provider) + results = await store.search("Alpha native", top_k=1, doc_ids=[source.id]) + + assert source.status == "ready" + assert native_index.upserts[0][0] == source.id + assert all(chunk.embedding == [0.1, 0.2, 0.3] for chunk in native_index.upserts[0][1]) + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["id"] == "native_chunk" + + @pytest.mark.asyncio async def test_text_source_is_chunked_with_citation_metadata(tmp_path: Path): repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) From 7aad0896146c58c9aec9c0303ddb6439a6c84028 Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 21:23:47 +0800 Subject: [PATCH 09/22] fix(seekdb): avoid stale native chunk state --- .../repositories/seekdb_repository.py | 14 ++-- tests/test_source_knowledge_base.py | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 3702e07..65f3107 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -242,6 +242,9 @@ async def delete_source(self, source_id: str) -> bool: return True async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if self.native_chunk_index is None and chunks and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + if self.native_chunk_index is not None and chunks and not self.allow_sqlite_vector_fallback: missing_embeddings = [chunk.id for chunk in chunks if not chunk.embedding] if missing_embeddings: @@ -270,17 +273,14 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non ) self._conn.commit() - has_vector_chunks = any(chunk.embedding is not None for chunk in chunks) if self.native_chunk_index is not None: - if chunks and has_vector_chunks: - self.native_chunk_index.upsert_source_chunks(source_id, chunks) - elif not chunks: + has_all_embeddings = bool(chunks) and all(chunk.embedding is not None for chunk in chunks) + if has_all_embeddings: self.native_chunk_index.upsert_source_chunks(source_id, chunks) + else: + self.native_chunk_index.upsert_source_chunks(source_id, []) return - if chunks and not self.allow_sqlite_vector_fallback: - raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) - async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( "SELECT payload FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 1c38205..2df3bab 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -52,6 +52,36 @@ def status(self): return {"vector_backend": "seekdb", "native_available": True} +class StaleAwareNativeIndex(RecordingNativeIndex): + def __init__(self) -> None: + super().__init__() + self.cleared_sources: set[str] = set() + + def upsert_source_chunks(self, source_id, chunks): + super().upsert_source_chunks(source_id, chunks) + if chunks == []: + self.cleared_sources.add(source_id) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + if source_ids[0] in self.cleared_sources: + return [] + return [ + { + "chunk": KnowledgeChunk( + id="stale_native_chunk", + source_id=source_ids[0], + content="stale native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + class DeterministicEmbeddingProvider: def __init__(self) -> None: self.calls = [] @@ -150,6 +180,7 @@ async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback ) ], ) + assert await repo.get_chunks(source.id) == [] @pytest.mark.asyncio @@ -232,6 +263,43 @@ async def test_repository_does_not_search_sqlite_bm25_without_explicit_fallback( await repo.search_chunks("Needle", source_ids=[source.id], top_k=1) +@pytest.mark.asyncio +async def test_fallback_no_embedding_save_clears_native_chunks_and_uses_sqlite_search(tmp_path: Path): + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-fallback-clear", kind=SourceKind.TEXT, title="Fallback Clear") + await repo.save_source(source) + + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-fallback-clear", + source_id=source.id, + content="Fresh fallback lexical content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + sqlite_results = await repo.search_chunks("Fresh fallback", source_ids=[source.id], top_k=1) + vector_results = await repo.search_chunks( + "stale native", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [])] + assert sqlite_results[0]["chunk"].id == "chunk-fallback-clear" + assert vector_results == [] + + @pytest.mark.asyncio async def test_source_service_ingestion_and_vector_store_search_use_native_seekdb(tmp_path: Path): native_index = RecordingNativeIndex() From 05f157998e922066aed1d3a71f96b05a4a6905db Mon Sep 17 00:00:00 2001 From: Lriver Date: Thu, 9 Jul 2026 21:31:18 +0800 Subject: [PATCH 10/22] fix(seekdb): keep chunk writes atomic with native index --- .../repositories/seekdb_repository.py | 63 ++++---- tests/test_source_knowledge_base.py | 146 ++++++++++++++++++ 2 files changed, 176 insertions(+), 33 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 65f3107..f86aaa6 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -232,17 +232,15 @@ async def list_sources(self) -> list[KnowledgeSource]: return [KnowledgeSource.model_validate_json(row["payload"]) for row in rows] async def delete_source(self, source_id: str) -> bool: - chunk_rows = self._conn.execute("SELECT id FROM chunks WHERE source_id = ?", (source_id,)).fetchall() - chunk_ids = [row["id"] for row in chunk_rows] - if self.native_chunk_index is not None: - self.native_chunk_index.delete_source_chunks(source_id, chunk_ids) - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) - self._conn.commit() + with self._conn: + if self.native_chunk_index is not None: + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) return True async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: - if self.native_chunk_index is None and chunks and not self.allow_sqlite_vector_fallback: + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) if self.native_chunk_index is not None and chunks and not self.allow_sqlite_vector_fallback: @@ -253,33 +251,32 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non f"missing embeddings for: {', '.join(missing_embeddings)}" ) - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.executemany( - """ - INSERT INTO chunks(id, source_id, content, chunk_index, embedding, payload) - VALUES (?, ?, ?, ?, ?, ?) - """, - [ - ( - chunk.id, - chunk.source_id, - chunk.content, - chunk.chunk_index, - json.dumps(chunk.embedding) if chunk.embedding is not None else None, - _dump_model(chunk), - ) - for chunk in chunks - ], - ) - self._conn.commit() - + native_chunks: list[KnowledgeChunk] | None = None if self.native_chunk_index is not None: has_all_embeddings = bool(chunks) and all(chunk.embedding is not None for chunk in chunks) - if has_all_embeddings: - self.native_chunk_index.upsert_source_chunks(source_id, chunks) - else: - self.native_chunk_index.upsert_source_chunks(source_id, []) - return + native_chunks = chunks if has_all_embeddings else [] + + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.executemany( + """ + INSERT INTO chunks(id, source_id, content, chunk_index, embedding, payload) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + ( + chunk.id, + chunk.source_id, + chunk.content, + chunk.chunk_index, + json.dumps(chunk.embedding) if chunk.embedding is not None else None, + _dump_model(chunk), + ) + for chunk in chunks + ], + ) + if native_chunks is not None: + self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 2df3bab..780b2af 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -82,6 +82,16 @@ def search(self, query_embedding, source_ids, top_k): ] +class FailingNativeIndex(RecordingNativeIndex): + def __init__(self, message: str = "native write failed") -> None: + super().__init__() + self.message = message + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + raise RuntimeError(self.message) + + class DeterministicEmbeddingProvider: def __init__(self) -> None: self.calls = [] @@ -183,6 +193,30 @@ async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback assert await repo.get_chunks(source.id) == [] +@pytest.mark.asyncio +async def test_strict_empty_save_with_native_unavailable_keeps_existing_sqlite_chunks(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-strict-empty", kind=SourceKind.TEXT, title="Strict Empty") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="existing sqlite content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=False) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks(source.id, []) + + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + + @pytest.mark.asyncio async def test_repository_requires_native_seekdb_for_non_vector_chunk_save_unless_fallback_enabled(tmp_path: Path): repo = SeekDBRepository( @@ -300,6 +334,118 @@ async def test_fallback_no_embedding_save_clears_native_chunks_and_uses_sqlite_s assert vector_results == [] +@pytest.mark.asyncio +async def test_delete_source_clears_native_chunks_even_when_sqlite_ids_are_stale(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-delete-clear", kind=SourceKind.TEXT, title="Delete Clear") + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="sqlite-only-chunk", + source_id=source.id, + content="sqlite-only content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await seed_repo.close() + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=True) + + await repo.delete_source(source.id) + stale_results = await repo.search_chunks( + "stale native", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert (source.id, []) in native_index.upserts + assert stale_results == [] + + +@pytest.mark.asyncio +async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-native-rollback", kind=SourceKind.TEXT, title="Native Rollback") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old sqlite content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + repo = SeekDBRepository( + db_path, + native_chunk_index=FailingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="native write failed"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new", + source_id=source.id, + content="new sqlite content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_native_clear_failure_rolls_back_fallback_sqlite_chunk_replacement(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-clear-rollback", kind=SourceKind.TEXT, title="Clear Rollback") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + repo = SeekDBRepository( + db_path, + native_chunk_index=FailingNativeIndex("native clear failed"), + allow_sqlite_vector_fallback=True, + ) + + with pytest.raises(RuntimeError, match="native clear failed"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new", + source_id=source.id, + content="new fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + + @pytest.mark.asyncio async def test_source_service_ingestion_and_vector_store_search_use_native_seekdb(tmp_path: Path): native_index = RecordingNativeIndex() From 3f4510b5a45a86913cdc8f17e552deab7d29803b Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 00:44:41 +0800 Subject: [PATCH 11/22] fix(seekdb): reject inconsistent native metadata mutations --- .../repositories/seekdb_repository.py | 3 ++ .../vector_stores/seekdb_vector_store.py | 24 +++++++++++++ tests/test_chunking_and_rerank.py | 35 +++++++++++++++++++ tests/test_source_knowledge_base.py | 25 +++++++++++++ 4 files changed, 87 insertions(+) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index f86aaa6..1fc6a90 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -232,6 +232,9 @@ async def list_sources(self) -> list[KnowledgeSource]: return [KnowledgeSource.model_validate_json(row["payload"]) for row in rows] async def delete_source(self, source_id: str) -> bool: + if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + with self._conn: if self.native_chunk_index is not None: self.native_chunk_index.upsert_source_chunks(source_id, []) diff --git a/backend/infrastructure/vector_stores/seekdb_vector_store.py b/backend/infrastructure/vector_stores/seekdb_vector_store.py index d5ce655..5c62898 100644 --- a/backend/infrastructure/vector_stores/seekdb_vector_store.py +++ b/backend/infrastructure/vector_stores/seekdb_vector_store.py @@ -28,6 +28,8 @@ async def add_chunks( chunks: list[dict[str, Any]], embeddings: Optional[list[list[float]]] = None, ) -> None: + self._validate_strict_native_write(chunks, embeddings) + existing = await self.repository.get_source(doc_id) if existing is None: existing = KnowledgeSource( @@ -53,6 +55,28 @@ async def add_chunks( ) await self.repository.save_chunks(doc_id, knowledge_chunks) + def _validate_strict_native_write( + self, + chunks: list[dict[str, Any]], + embeddings: Optional[list[list[float]]], + ) -> None: + if getattr(self.repository, "allow_sqlite_vector_fallback", True): + return + if getattr(self.repository, "native_chunk_index", None) is None: + raise RuntimeError( + "native SeekDB vector index is unavailable; " + "enable seekdb_allow_sqlite_fallback for SQLite fallback" + ) + if not chunks: + return + + provided_embeddings = embeddings or [] + has_missing_embedding = len(provided_embeddings) < len(chunks) or any( + embedding is None or len(embedding) == 0 for embedding in provided_embeddings[: len(chunks)] + ) + if has_missing_embedding: + raise RuntimeError("native SeekDB chunk writes require embeddings for all chunks") + async def search( self, query: str, diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index e401f24..8ca1c6c 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -14,6 +14,20 @@ def sqlite_fallback_repo(path: Path) -> SeekDBRepository: return SeekDBRepository(path, native_chunk_index=None, allow_sqlite_vector_fallback=True) +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + + def search(self, query_embedding, source_ids, top_k): + return [] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + def test_chonkie_chunker_returns_offsets_and_counts(): pytest.importorskip("chonkie") service = ChunkingService( @@ -117,3 +131,24 @@ async def test_seekdb_repository_uses_bm25_for_term_overlap_without_exact_phrase assert results assert results[0]["chunk"].id == "chunk-http" assert results[0]["score"] > 0 + + +@pytest.mark.asyncio +async def test_seekdb_vector_store_add_chunks_does_not_leave_ready_source_on_strict_native_failure( + tmp_path: Path, +): + repo = SeekDBRepository( + tmp_path / "strict-add.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + store = SeekDBVectorStore(repository=repo) + + with pytest.raises(RuntimeError, match="embeddings|native SeekDB"): + await store.add_chunks( + "doc-strict", + [{"content": "strict native content", "metadata": {"filename": "strict.txt"}}], + ) + + assert await repo.get_source("doc-strict") is None + assert await repo.get_chunks("doc-strict") == [] diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 780b2af..3e27e9d 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -368,6 +368,31 @@ async def test_delete_source_clears_native_chunks_even_when_sqlite_ids_are_stale assert stale_results == [] +@pytest.mark.asyncio +async def test_delete_source_requires_native_seekdb_before_sqlite_mutation(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-strict-delete", kind=SourceKind.TEXT, title="Strict Delete") + old_chunk = KnowledgeChunk( + id="chunk-strict-delete", + source_id=source.id, + content="existing sqlite metadata", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=False) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.delete_source(source.id) + + assert await repo.get_source(source.id) is not None + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-strict-delete"] + + @pytest.mark.asyncio async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_path: Path): db_path = tmp_path / "knowledge.db" From 7328fbedf7cab57b35ea8b92c12bd8cbd4ff9953 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 00:52:50 +0800 Subject: [PATCH 12/22] fix(seekdb): keep native failures strict --- .../repositories/seekdb_repository.py | 6 +++ .../vector_stores/seekdb_chunk_index.py | 24 +++++------ .../vector_stores/seekdb_vector_store.py | 15 ++++++- tests/test_chunking_and_rerank.py | 28 +++++++++++++ tests/test_seekdb_chunk_index.py | 42 +++++++++++++++++++ 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 1fc6a90..0d2708a 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -242,6 +242,12 @@ async def delete_source(self, source_id: str) -> bool: self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) return True + async def delete_source_metadata_only(self, source_id: str) -> None: + """Rollback local metadata without touching native SeekDB.""" + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index 47c2a59..eb3c415 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -160,16 +160,12 @@ def search( results: list[dict[str, Any]] = [] include = ["documents", "metadatas", "distances"] for source_id in source_ids: - try: - collection = self._collection(source_id) - query_result = collection.query( - query_embeddings=[query_embedding], - n_results=top_k, - include=include, - ) - except Exception as exc: - logger.warning("Failed to query SeekDB chunks for source %s: %s", source_id, exc) - continue + collection = self._collection(source_id) + query_result = collection.query( + query_embeddings=[query_embedding], + n_results=top_k, + include=include, + ) ids = self._result_group(query_result.get("ids", [])) distances = self._result_group(query_result.get("distances", [])) @@ -178,11 +174,11 @@ def search( try: metadata = metadatas[index] chunk = KnowledgeChunk.model_validate_json(metadata["payload"]) - distance = distances[index] if index < len(distances) else 0.0 - score = 1.0 / (1.0 + max(float(distance), 0.0)) - results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) except Exception as exc: - logger.warning("Skipping malformed SeekDB chunk result: %s", exc) + raise ValueError("Malformed SeekDB chunk result payload") from exc + distance = distances[index] if index < len(distances) else 0.0 + score = 1.0 / (1.0 + max(float(distance), 0.0)) + results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] diff --git a/backend/infrastructure/vector_stores/seekdb_vector_store.py b/backend/infrastructure/vector_stores/seekdb_vector_store.py index 5c62898..8cb3554 100644 --- a/backend/infrastructure/vector_stores/seekdb_vector_store.py +++ b/backend/infrastructure/vector_stores/seekdb_vector_store.py @@ -31,6 +31,7 @@ async def add_chunks( self._validate_strict_native_write(chunks, embeddings) existing = await self.repository.get_source(doc_id) + created_source = existing is None if existing is None: existing = KnowledgeSource( id=doc_id, @@ -53,7 +54,12 @@ async def add_chunks( metadata={"source_id": doc_id, "chunk_index": index, **metadata}, ) ) - await self.repository.save_chunks(doc_id, knowledge_chunks) + try: + await self.repository.save_chunks(doc_id, knowledge_chunks) + except Exception: + if created_source: + await self._rollback_created_source(doc_id) + raise def _validate_strict_native_write( self, @@ -77,6 +83,13 @@ def _validate_strict_native_write( if has_missing_embedding: raise RuntimeError("native SeekDB chunk writes require embeddings for all chunks") + async def _rollback_created_source(self, doc_id: str) -> None: + rollback_metadata = getattr(self.repository, "delete_source_metadata_only", None) + if rollback_metadata is not None: + await rollback_metadata(doc_id) + return + await self.repository.delete_source(doc_id) + async def search( self, query: str, diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index 8ca1c6c..027f314 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -28,6 +28,12 @@ def status(self): return {"vector_backend": "seekdb", "native_available": True} +class FailingNativeIndex(RecordingNativeIndex): + def upsert_source_chunks(self, source_id, chunks): + super().upsert_source_chunks(source_id, chunks) + raise RuntimeError("native write failed") + + def test_chonkie_chunker_returns_offsets_and_counts(): pytest.importorskip("chonkie") service = ChunkingService( @@ -152,3 +158,25 @@ async def test_seekdb_vector_store_add_chunks_does_not_leave_ready_source_on_str assert await repo.get_source("doc-strict") is None assert await repo.get_chunks("doc-strict") == [] + + +@pytest.mark.asyncio +async def test_seekdb_vector_store_add_chunks_rolls_back_new_source_when_native_save_fails( + tmp_path: Path, +): + repo = SeekDBRepository( + tmp_path / "strict-valid-embeddings.db", + native_chunk_index=FailingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + store = SeekDBVectorStore(repository=repo) + + with pytest.raises(RuntimeError, match="native write failed"): + await store.add_chunks( + "doc-native-fail", + [{"content": "strict native content", "metadata": {"filename": "strict.txt"}}], + embeddings=[[0.1, 0.2, 0.3]], + ) + + assert await repo.get_source("doc-native-fail") is None + assert await repo.get_chunks("doc-native-fail") == [] diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 70a338c..5769509 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -30,6 +30,7 @@ def __init__(self, name: str, dimension: int) -> None: self.events = [] self.refresh_calls = 0 self.rows = {} + self.query_error = None def upsert(self, ids, documents, metadatas, embeddings): self.events.append({"op": "upsert", "ids": ids}) @@ -61,6 +62,8 @@ def delete(self, ids=None, **kwargs): self.rows.pop(chunk_id, None) def query(self, query_embeddings, n_results, include): + if self.query_error is not None: + raise self.query_error self.query_calls.append( { "query_embeddings": query_embeddings, @@ -177,6 +180,45 @@ def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Pat assert source_b_query["include"] == ["documents", "metadatas", "distances"] +def test_search_raises_collection_query_errors(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.query_error = RuntimeError("query exploded") + + with pytest.raises(RuntimeError, match="query exploded"): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_search_raises_malformed_payload_results(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.rows["chunk-a"]["metadata"]["payload"] = "{not-json" + + with pytest.raises(ValueError, match="Malformed SeekDB chunk result payload"): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_search_raises_missing_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + with pytest.raises(KeyError): + index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-missing"], + top_k=1, + ) + + def test_delete_source_chunks_refreshes_collection_after_successful_delete(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) From 243eef0a440cf3dcc0807f561adb86aca67c7ff1 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 01:20:08 +0800 Subject: [PATCH 13/22] feat(seekdb): use native hybrid retrieval --- .../repositories/seekdb_repository.py | 11 +- .../vector_stores/seekdb_chunk_index.py | 66 ++++++-- tests/test_chunking_and_rerank.py | 103 +++++++++++ tests/test_seekdb_chunk_index.py | 160 ++++++++++++++++++ 4 files changed, 327 insertions(+), 13 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 0d2708a..bab9c2b 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -304,7 +304,16 @@ async def search_chunks( ) -> list[dict]: if self.native_chunk_index is not None and query_embedding is not None: selected_source_ids = source_ids or self._current_source_ids() - results = self.native_chunk_index.search(query_embedding, selected_source_ids, top_k) + hybrid_search = getattr(self.native_chunk_index, "hybrid_search", None) + if callable(hybrid_search): + results = hybrid_search( + query_text=query, + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + else: + results = self.native_chunk_index.search(query_embedding, selected_source_ids, top_k) return await self._maybe_rerank(query, results, top_k, rerank_provider) if self.native_chunk_index is not None and not self.allow_sqlite_vector_fallback: diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index eb3c415..968ee71 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -166,19 +166,42 @@ def search( n_results=top_k, include=include, ) + results.extend(self._rows_to_results(query_result, score_mode="distance")) - ids = self._result_group(query_result.get("ids", [])) - distances = self._result_group(query_result.get("distances", [])) - metadatas = self._result_group(query_result.get("metadatas", [])) - for index, _chunk_id in enumerate(ids): - try: - metadata = metadatas[index] - chunk = KnowledgeChunk.model_validate_json(metadata["payload"]) - except Exception as exc: - raise ValueError("Malformed SeekDB chunk result payload") from exc - distance = distances[index] if index < len(distances) else 0.0 - score = 1.0 / (1.0 + max(float(distance), 0.0)) - results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + + def hybrid_search( + self, + query_text: str, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("Query embedding is required") + query_text = query_text.strip() + if not query_text: + return self.search(query_embedding, source_ids, top_k) + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + n_results = max(top_k, 1) + for source_id in source_ids: + collection = self._collection(source_id) + collection_hybrid_search = getattr(collection, "hybrid_search", None) + if not callable(collection_hybrid_search): + return self.search(query_embedding, source_ids, top_k) + + query_result = collection_hybrid_search( + query={"where_document": {"$contains": query_text}, "n_results": n_results}, + knn={"query_embeddings": query_embedding, "n_results": n_results}, + rank={"rrf": {}}, + n_results=n_results, + include=include, + ) + results.extend(self._rows_to_results(query_result, score_mode="similarity")) return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] @@ -189,6 +212,25 @@ def _result_group(value: Any) -> list[Any]: first = value[0] return first if isinstance(first, list) else value + def _rows_to_results(self, query_result: dict[str, Any], score_mode: str) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + ids = self._result_group(query_result.get("ids", [])) + distances = self._result_group(query_result.get("distances", [])) + metadatas = self._result_group(query_result.get("metadatas", [])) + for index, _chunk_id in enumerate(ids): + try: + metadata = metadatas[index] + chunk = KnowledgeChunk.model_validate_json(metadata["payload"]) + except Exception as exc: + raise ValueError("Malformed SeekDB chunk result payload") from exc + raw_score = float(distances[index]) if index < len(distances) else 0.0 + if score_mode == "distance": + score = 1.0 / (1.0 + max(raw_score, 0.0)) + else: + score = raw_score + results.append({"chunk": chunk, "score": score, "backend": "seekdb"}) + return results + def status(self) -> dict[str, Any]: return { "vector_backend": "seekdb", diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index 027f314..c69f2fe 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -34,6 +34,31 @@ def upsert_source_chunks(self, source_id, chunks): raise RuntimeError("native write failed") +class HybridRecordingIndex: + def __init__(self) -> None: + self.calls = [] + + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid_chunk", + source_id=source_ids[0], + content="hybrid TLS result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.88, + "backend": "seekdb", + } + ] + + def search(self, query_embedding, source_ids, top_k): + raise AssertionError("hybrid_search should be preferred when query text is available") + + def test_chonkie_chunker_returns_offsets_and_counts(): pytest.importorskip("chonkie") service = ChunkingService( @@ -139,6 +164,84 @@ async def test_seekdb_repository_uses_bm25_for_term_overlap_without_exact_phrase assert results[0]["score"] > 0 +@pytest.mark.asyncio +async def test_seekdb_repository_prefers_native_hybrid_search(tmp_path: Path): + native_index = HybridRecordingIndex() + repo = SeekDBRepository( + tmp_path / "hybrid.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid", kind=SourceKind.TEXT, title="Hybrid") + await repo.save_source(source) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.calls == [("TLS handshake", [0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "hybrid_chunk" + + +@pytest.mark.asyncio +async def test_seekdb_repository_reranks_native_hybrid_results(tmp_path: Path): + class TwoResultHybridIndex(HybridRecordingIndex): + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid-a", + source_id=source_ids[0], + content="first hybrid result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.9, + "backend": "seekdb", + }, + { + "chunk": KnowledgeChunk( + id="hybrid-b", + source_id=source_ids[0], + content="second hybrid result", + chunk_index=1, + embedding=[0.2, 0.3, 0.4], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.8, + "backend": "seekdb", + }, + ] + + class ReverseReranker: + async def rerank(self, query, results, top_k=5): + return list(reversed(results))[:top_k] + + native_index = TwoResultHybridIndex() + repo = SeekDBRepository( + tmp_path / "hybrid-rerank.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid-rerank", kind=SourceKind.TEXT, title="Hybrid rerank") + await repo.save_source(source) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=2, + query_embedding=[0.1, 0.2, 0.3], + rerank_provider=ReverseReranker(), + ) + + assert [item["chunk"].id for item in results] == ["hybrid-b", "hybrid-a"] + + @pytest.mark.asyncio async def test_seekdb_vector_store_add_chunks_does_not_leave_ready_source_on_strict_native_failure( tmp_path: Path, diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 5769509..44dc749 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -27,10 +27,12 @@ def __init__(self, name: str, dimension: int) -> None: self.upsert_calls = [] self.delete_calls = [] self.query_calls = [] + self.hybrid_calls = [] self.events = [] self.refresh_calls = 0 self.rows = {} self.query_error = None + self.hybrid_error = None def upsert(self, ids, documents, metadatas, embeddings): self.events.append({"op": "upsert", "ids": ids}) @@ -79,6 +81,26 @@ def query(self, query_embeddings, n_results, include): "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], } + def hybrid_search(self, query, knn, rank, n_results, include): + if self.hybrid_error is not None: + raise self.hybrid_error + self.hybrid_calls.append( + { + "query": query, + "knn": knn, + "rank": rank, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[10.0 - index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + def count(self): return len(self.rows) @@ -180,6 +202,119 @@ def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Pat assert source_b_query["include"] == ["documents", "metadatas", "distances"] +def test_hybrid_search_calls_seekdb_hybrid_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + assert results[0]["backend"] == "seekdb" + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls == [ + { + "query": {"where_document": {"$contains": "TLS handshake"}, "n_results": 1}, + "knn": {"query_embeddings": [0.1, 0.2, 0.3], "n_results": 1}, + "rank": {"rrf": {}}, + "n_results": 1, + "include": ["documents", "metadatas", "distances"], + } + ] + + +def test_hybrid_search_preserves_native_relevance_score_order(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks( + "source-a", + [ + chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]), + chunk("chunk-b", "source-a", [0.9, 0.1, 0.1]), + ], + ) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert [item["score"] for item in results] == [10.0, 9.0] + + +def test_hybrid_search_raises_collection_hybrid_errors(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.hybrid_error = RuntimeError("hybrid exploded") + + with pytest.raises(RuntimeError, match="hybrid exploded"): + index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + +def test_hybrid_search_falls_back_to_vector_search_when_collection_lacks_hybrid_api( + tmp_path: Path, + monkeypatch, +): + monkeypatch.delattr(FakeCollection, "hybrid_search") + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.query_calls[0]["query_embeddings"] == [[0.1, 0.2, 0.3]] + + +def test_hybrid_search_falls_back_to_vector_search_for_blank_query_text(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.hybrid_search( + query_text=" ", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls == [] + assert collection.query_calls[0]["query_embeddings"] == [[0.1, 0.2, 0.3]] + + +def test_hybrid_search_raises_malformed_payload_results(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + collection.rows["chunk-a"]["metadata"]["payload"] = "{not-json" + + with pytest.raises(ValueError, match="Malformed SeekDB chunk result payload"): + index.hybrid_search( + query_text="TLS handshake", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a"], + top_k=1, + ) + + def test_search_raises_collection_query_errors(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) @@ -445,6 +580,31 @@ def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path): ) +def test_real_pyseekdb_hybrid_search_round_trip(tmp_path: Path): + run_real_pyseekdb_subprocess( + f""" + index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) + index.upsert_source_chunks( + "real-source", + [ + chunk("real-lexical", "real-source", [0.9, 0.1, 0.1]), + chunk("real-vector", "real-source", [0.1, 0.2, 0.3]), + chunk("real-noise", "real-source", [0.8, 0.8, 0.8]), + ], + ) + results = index.hybrid_search( + query_text="real-lexical", + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=3, + ) + assert results + assert results[0]["chunk"].id == "real-lexical" + assert results[0]["score"] > 0 + """ + ) + + def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path): run_real_pyseekdb_subprocess( f""" From b3f816066aa73ae40c1d470cae91742105f26f68 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 01:32:47 +0800 Subject: [PATCH 14/22] feat(config): expose actual vector backend status --- backend/api/routes/config.py | 13 ++- .../vector_stores/seekdb_vector_store.py | 14 ++- backend/main.py | 14 ++- tests/conftest.py | 4 +- tests/test_config_settings.py | 2 +- tests/test_runtime_config_api.py | 86 +++++++++++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/backend/api/routes/config.py b/backend/api/routes/config.py index 83321fe..eb5eaf6 100644 --- a/backend/api/routes/config.py +++ b/backend/api/routes/config.py @@ -24,9 +24,12 @@ def _public_profile(profile: ModelProfile) -> PublicModelProfile: ) -def _response(message: str = "") -> RuntimeConfigResponse: +async def _response(message: str = "") -> RuntimeConfigResponse: settings = get_settings() models = settings.api.models + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + storage_status = stats.get("storage", {}) return RuntimeConfigResponse( models={ name: _public_profile(getattr(models, name)) @@ -40,7 +43,11 @@ def _response(message: str = "") -> RuntimeConfigResponse: }, storage={ "vector_store_type": settings.vector_store_type, + "configured_vector_store_type": settings.vector_store_type, "seekdb_path": settings.seekdb_path, + "seekdb_allow_sqlite_fallback": settings.seekdb_allow_sqlite_fallback, + "actual_vector_backend": storage_status.get("vector_backend", stats.get("backend", "unknown")), + "native_available": storage_status.get("native_available", False), }, message=message, ) @@ -48,7 +55,7 @@ def _response(message: str = "") -> RuntimeConfigResponse: @router.get("", response_model=RuntimeConfigResponse) async def get_runtime_config() -> RuntimeConfigResponse: - return _response() + return await _response() @router.post("", response_model=RuntimeConfigResponse) @@ -59,4 +66,4 @@ async def update_runtime_config(request: RuntimeConfigUpdate) -> RuntimeConfigRe } update_runtime_model_profiles(profile_data) DependencyContainer.reset_runtime_caches() - return _response("Runtime model configuration updated.") + return await _response("Runtime model configuration updated.") diff --git a/backend/infrastructure/vector_stores/seekdb_vector_store.py b/backend/infrastructure/vector_stores/seekdb_vector_store.py index 8cb3554..7d93374 100644 --- a/backend/infrastructure/vector_stores/seekdb_vector_store.py +++ b/backend/infrastructure/vector_stores/seekdb_vector_store.py @@ -132,7 +132,19 @@ async def get_document_chunks(self, doc_id: str) -> list[dict[str, Any]]: for chunk in chunks ] + def storage_status(self) -> dict[str, Any]: + repository_status = getattr(self.repository, "storage_status", None) + if callable(repository_status): + return repository_status() + return {"vector_backend": "unknown", "native_available": False} + async def get_stats(self) -> dict[str, Any]: sources = await self.repository.list_sources() chunk_count = sum(source.chunk_count for source in sources) - return {"total_documents": len(sources), "total_chunks": chunk_count, "backend": "seekdb"} + status = self.storage_status() + return { + "total_documents": len(sources), + "total_chunks": chunk_count, + "backend": status.get("vector_backend", "unknown"), + "storage": status, + } diff --git a/backend/main.py b/backend/main.py index c0df7e8..353da55 100644 --- a/backend/main.py +++ b/backend/main.py @@ -67,7 +67,19 @@ async def lifespan(app: FastAPI): # 健康检查 @app.get("/health") async def health_check(): - return {"status": "healthy", "version": settings.app_version} + from .dependencies import get_vector_store + + vector_store = get_vector_store() + stats = await vector_store.get_stats() + storage = stats.get("storage", {}) + return { + "status": "healthy", + "version": settings.app_version, + "storage": { + "actual_vector_backend": storage.get("vector_backend", stats.get("backend", "unknown")), + "native_available": storage.get("native_available", False), + }, + } # 向量存储统计 diff --git a/tests/conftest.py b/tests/conftest.py index a85c033..2457278 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ def sample_config_file(tmp_path: Path) -> Path: config_path = tmp_path / "config.yaml" config_path.write_text( - """ + f""" api: models: text_model: @@ -49,7 +49,7 @@ def sample_config_file(tmp_path: Path) -> Path: adapter: raw_chat_multimodal storage: vector_store_type: seekdb - seekdb_path: ./data/test_seekdb.db + seekdb_path: {tmp_path / "data" / "test_seekdb.db"} documents: chunk_size: 512 chunk_overlap: 64 diff --git a/tests/test_config_settings.py b/tests/test_config_settings.py index 7564911..ab44dad 100644 --- a/tests/test_config_settings.py +++ b/tests/test_config_settings.py @@ -15,7 +15,7 @@ def test_loads_yaml_model_profiles_without_using_local_config(sample_config_file assert settings.api.models.audio_model.response_format == "mp3" assert settings.api.models.audio_model.stream is True assert settings.vector_store_type == "seekdb" - assert settings.seekdb_path == "./data/test_seekdb.db" + assert settings.seekdb_path.endswith("/data/test_seekdb.db") assert settings.chunk_size == 512 assert settings.chunk_overlap == 64 assert settings.chunking.provider == "chonkie" diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index d9d64f6..3120e9f 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -8,6 +8,19 @@ from backend.dependencies import DependencyContainer +class FakeVectorStore: + def __init__(self, status: dict) -> None: + self.status = status + + async def get_stats(self) -> dict: + return { + "total_documents": 0, + "total_chunks": 0, + "backend": self.status["vector_backend"], + "storage": self.status, + } + + def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample_config_file): monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) _RUNTIME_MODEL_OVERRIDES.clear() @@ -72,3 +85,76 @@ def test_runtime_cache_reset_recreates_knowledge_repository_with_new_storage_set assert second_repo.allow_sqlite_vector_fallback is False finally: DependencyContainer.reset_runtime_caches() + + +def test_runtime_config_exposes_actual_vector_backend(monkeypatch, sample_config_file): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True} + ) + ), + ) + get_settings.cache_clear() + DependencyContainer.reset_runtime_caches() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert storage["configured_vector_store_type"] == "seekdb" + assert storage["vector_store_type"] == "seekdb" + assert storage["actual_vector_backend"] == "seekdb" + assert storage["native_available"] is True + + get_settings.cache_clear() + DependencyContainer.reset_runtime_caches() + + +def test_runtime_config_exposes_sqlite_fallback_status(monkeypatch, sample_config_file): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "sqlite_fallback", "native_available": False} + ) + ), + ) + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert storage["configured_vector_store_type"] == "seekdb" + assert storage["actual_vector_backend"] == "sqlite_fallback" + assert storage["native_available"] is False + + +def test_health_exposes_actual_vector_backend(monkeypatch): + from backend import main as backend_main + + def fake_get_vector_store(): + return FakeVectorStore({"vector_backend": "unavailable", "native_available": False}) + + monkeypatch.setattr("backend.dependencies.get_vector_store", fake_get_vector_store) + client = TestClient(backend_main.app) + + response = client.get("/health") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert storage["actual_vector_backend"] == "unavailable" + assert storage["native_available"] is False From c8846f099c9a82dca1cab3248880482950b3ee8e Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 08:21:02 +0800 Subject: [PATCH 15/22] feat(seekdb): backfill existing chunks into native index --- .../repositories/seekdb_repository.py | 119 +++- tests/test_chunking_and_rerank.py | 30 + tests/test_source_knowledge_base.py | 643 ++++++++++++++++++ 3 files changed, 777 insertions(+), 15 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index bab9c2b..0c173d9 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -259,11 +259,13 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non "native SeekDB chunk writes require embeddings for all chunks; " f"missing embeddings for: {', '.join(missing_embeddings)}" ) + native_eligible_chunks = self._native_eligible_chunks(chunks) + if len(native_eligible_chunks) != len(chunks): + raise RuntimeError("native SeekDB chunk writes require same-dimension embeddings for all chunks") native_chunks: list[KnowledgeChunk] | None = None if self.native_chunk_index is not None: - has_all_embeddings = bool(chunks) and all(chunk.embedding is not None for chunk in chunks) - native_chunks = chunks if has_all_embeddings else [] + native_chunks = self._native_eligible_chunks(chunks) with self._conn: self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) @@ -294,6 +296,20 @@ async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: ).fetchall() return [KnowledgeChunk.model_validate_json(row["payload"]) for row in rows] + async def backfill_native_chunks(self) -> int: + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + return 0 + raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + + written = 0 + for source in await self.list_sources(): + chunks = await self.get_chunks(source.id) + chunks_with_embeddings = self._native_eligible_chunks(chunks) + self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + written += len(chunks_with_embeddings) + return written + async def search_chunks( self, query: str, @@ -302,18 +318,50 @@ async def search_chunks( query_embedding: list[float] | None = None, rerank_provider: object | None = None, ) -> list[dict]: + if source_ids == []: + return [] if self.native_chunk_index is not None and query_embedding is not None: - selected_source_ids = source_ids or self._current_source_ids() + if not query_embedding: + if self.allow_sqlite_vector_fallback: + rows = self._chunk_rows(source_ids) + return await self._search_sqlite_chunk_rows(query, rows, top_k, None, rerank_provider) + raise RuntimeError("native SeekDB vector search requires a non-empty query embedding") + requested_source_ids = source_ids if source_ids is not None else self._current_source_ids() + selected_source_ids = self._native_search_source_ids(requested_source_ids, query_embedding) + results: list[dict] = [] hybrid_search = getattr(self.native_chunk_index, "hybrid_search", None) - if callable(hybrid_search): - results = hybrid_search( - query_text=query, - query_embedding=query_embedding, - source_ids=selected_source_ids, - top_k=top_k, - ) - else: - results = self.native_chunk_index.search(query_embedding, selected_source_ids, top_k) + if selected_source_ids: + try: + if callable(hybrid_search): + results.extend( + hybrid_search( + query_text=query, + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + ) + else: + results.extend(self.native_chunk_index.search(query_embedding, selected_source_ids, top_k)) + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + selected_source_ids = [] + if self.allow_sqlite_vector_fallback: + native_source_id_set = set(selected_source_ids) + fallback_source_ids = [ + source_id for source_id in requested_source_ids if source_id not in native_source_id_set + ] + if fallback_source_ids: + fallback_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(fallback_source_ids), + top_k, + query_embedding, + None, + ) + results.extend(fallback_results) + results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] return await self._maybe_rerank(query, results, top_k, rerank_provider) if self.native_chunk_index is not None and not self.allow_sqlite_vector_fallback: @@ -331,10 +379,47 @@ def _current_source_ids(self) -> list[str]: ).fetchall() return [row["id"] for row in rows] + @staticmethod + def _native_eligible_chunks(chunks: list[KnowledgeChunk]) -> list[KnowledgeChunk]: + if not chunks: + return [] + dimension: int | None = None + for chunk in chunks: + if not chunk.embedding: + return [] + if dimension is None: + dimension = len(chunk.embedding) + elif len(chunk.embedding) != dimension: + return [] + return chunks + + def _native_search_source_ids(self, source_ids: list[str], query_embedding: list[float]) -> list[str]: + if not source_ids: + return [] + query_dimension = len(query_embedding) + selected: list[str] = [] + for source_id in source_ids: + chunks = self._native_eligible_chunks(self._get_chunks_sync(source_id)) + if not chunks: + continue + if len(chunks[0].embedding or []) != query_dimension: + continue + selected.append(source_id) + return selected + + def _get_chunks_sync(self, source_id: str) -> list[KnowledgeChunk]: + rows = self._conn.execute( + "SELECT payload FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", + (source_id,), + ).fetchall() + return [KnowledgeChunk.model_validate_json(row["payload"]) for row in rows] + def _chunk_rows(self, source_ids: list[str] | None = None) -> list[sqlite3.Row]: + if source_ids == []: + return [] params: list[Any] = [] sql = "SELECT payload, content, embedding FROM chunks" - if source_ids: + if source_ids is not None: placeholders = ",".join("?" for _ in source_ids) sql += f" WHERE source_id IN ({placeholders})" params.extend(source_ids) @@ -353,10 +438,14 @@ async def _search_sqlite_chunk_rows( for row, bm25_score in zip(rows, bm25_scores): chunk = KnowledgeChunk.model_validate_json(row["payload"]) vector_score = 0.0 + has_comparable_embedding = False if query_embedding and row["embedding"]: - vector_score = _cosine(query_embedding, json.loads(row["embedding"])) + row_embedding = json.loads(row["embedding"]) + has_comparable_embedding = len(row_embedding) == len(query_embedding) + if has_comparable_embedding: + vector_score = _cosine(query_embedding, row_embedding) score = bm25_score + max(vector_score, 0.0) - if bm25_score > 0 or query_embedding: + if bm25_score > 0 or has_comparable_embedding: results.append({"chunk": chunk, "score": score}) results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index c69f2fe..6f73992 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -37,6 +37,10 @@ def upsert_source_chunks(self, source_id, chunks): class HybridRecordingIndex: def __init__(self) -> None: self.calls = [] + self.upserts = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) def hybrid_search(self, query_text, query_embedding, source_ids, top_k): self.calls.append((query_text, query_embedding, source_ids, top_k)) @@ -174,6 +178,19 @@ async def test_seekdb_repository_prefers_native_hybrid_search(tmp_path: Path): ) source = KnowledgeSource(id="src-hybrid", kind=SourceKind.TEXT, title="Hybrid") await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="seed-hybrid", + source_id=source.id, + content="seed hybrid content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) results = await repo.search_chunks( "TLS handshake", @@ -230,6 +247,19 @@ async def rerank(self, query, results, top_k=5): ) source = KnowledgeSource(id="src-hybrid-rerank", kind=SourceKind.TEXT, title="Hybrid rerank") await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="seed-hybrid-rerank", + source_id=source.id, + content="seed hybrid rerank content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) results = await repo.search_chunks( "TLS handshake", diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 3e27e9d..7a3f975 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -92,6 +92,26 @@ def upsert_source_chunks(self, source_id, chunks): raise RuntimeError(self.message) +class DimensionCheckingNativeIndex(RecordingNativeIndex): + def search(self, query_embedding, source_ids, top_k): + bad_source_ids = [ + source_id + for source_id, chunks in self.upserts + if source_id in source_ids + and chunks + and len(chunks[0].embedding or []) != len(query_embedding) + ] + if bad_source_ids: + raise ValueError(f"dimension mismatch for {bad_source_ids}") + return super().search(query_embedding, source_ids, top_k) + + +class QueryFailingNativeIndex(RecordingNativeIndex): + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + raise KeyError("collection not found") + + class DeterministicEmbeddingProvider: def __init__(self) -> None: self.calls = [] @@ -433,6 +453,58 @@ async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_pat assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] +@pytest.mark.asyncio +async def test_strict_native_rejects_mixed_dimension_embeddings_before_sqlite_mutation( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-mixed-dim", kind=SourceKind.TEXT, title="Mixed Dimension") + old_chunk = KnowledgeChunk( + id="chunk-old", + source_id=source.id, + content="old sqlite content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks(source.id, [old_chunk]) + await seed_repo.close() + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="same-dimension embeddings"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-new-a", + source_id=source.id, + content="new sqlite content a", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-new-b", + source_id=source.id, + content="new sqlite content b", + chunk_index=1, + embedding=[0.1, 0.2], + metadata={"source_id": source.id}, + ), + ], + ) + + assert native_index.upserts == [] + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + + @pytest.mark.asyncio async def test_native_clear_failure_rolls_back_fallback_sqlite_chunk_replacement(tmp_path: Path): db_path = tmp_path / "knowledge.db" @@ -471,6 +543,577 @@ async def test_native_clear_failure_rolls_back_fallback_sqlite_chunk_replacement assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] +@pytest.mark.asyncio +async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-backfill", kind=SourceKind.TEXT, title="Backfill") + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-backfill", + source_id=source.id, + content="backfill content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + written = await repo.backfill_native_chunks() + + assert written == 1 + assert native_index.upserts[0][0] == source.id + assert native_index.upserts[0][1][0].id == "chunk-backfill" + + +@pytest.mark.asyncio +async def test_repository_backfill_requires_native_index_without_fallback(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="cannot backfill chunks"): + await repo.backfill_native_chunks() + + +@pytest.mark.asyncio +async def test_repository_backfill_clears_native_chunks_for_sources_without_embeddings(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-backfill-clear", kind=SourceKind.TEXT, title="Backfill Clear") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain backfill content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "plain backfill", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert native_index.upserts == [(source.id, [])] + assert results == [] + + +@pytest.mark.asyncio +async def test_repository_native_search_skips_legacy_sources_without_embeddings_after_backfill( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + embedded_source = KnowledgeSource(id="src-embedded", kind=SourceKind.TEXT, title="Embedded") + plain_source = KnowledgeSource(id="src-plain", kind=SourceKind.TEXT, title="Plain") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(embedded_source) + await fallback_repo.save_chunks( + embedded_source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=embedded_source.id, + content="embedded legacy content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": embedded_source.id}, + ) + ], + ) + await fallback_repo.save_source(plain_source) + await fallback_repo.save_chunks( + plain_source.id, + [ + KnowledgeChunk( + id="chunk-plain", + source_id=plain_source.id, + content="plain legacy content", + chunk_index=0, + metadata={"source_id": plain_source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + await repo.backfill_native_chunks() + await repo.search_chunks("legacy", source_ids=None, top_k=1, query_embedding=[0.1, 0.2, 0.3]) + + assert {source_id for source_id, _chunks in native_index.upserts} == { + embedded_source.id, + plain_source.id, + } + assert native_index.searches == [([0.1, 0.2, 0.3], [embedded_source.id], 1)] + + +@pytest.mark.asyncio +async def test_repository_backfill_clears_partially_embedded_sources_and_skips_native_search( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-partial", kind=SourceKind.TEXT, title="Partial") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=source.id, + content="embedded partial content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain partial content", + chunk_index=1, + metadata={"source_id": source.id}, + ), + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "partial", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results == [] + + +@pytest.mark.asyncio +async def test_repository_native_search_uses_sqlite_fallback_for_partial_writes(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-partial-fallback", kind=SourceKind.TEXT, title="Partial Fallback") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-embedded", + source_id=source.id, + content="embedded fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ), + KnowledgeChunk( + id="chunk-plain", + source_id=source.id, + content="plain fallback content", + chunk_index=1, + metadata={"source_id": source.id}, + ), + ], + ) + + results = await repo.search_chunks( + "fallback", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results + assert results[0]["chunk"].source_id == source.id + + +@pytest.mark.asyncio +async def test_repository_backfill_treats_empty_embedding_as_native_ineligible(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-empty-embedding", kind=SourceKind.TEXT, title="Empty Embedding") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-embedding", + source_id=source.id, + content="empty embedding content", + chunk_index=0, + embedding=[], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = StaleAwareNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + results = await repo.search_chunks( + "empty embedding", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert written == 0 + assert native_index.upserts == [(source.id, [])] + assert native_index.searches == [] + assert results == [] + + +@pytest.mark.asyncio +async def test_repository_native_search_filters_sources_by_query_embedding_dimension( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source_3d = KnowledgeSource(id="src-3d", kind=SourceKind.TEXT, title="3D") + source_2d = KnowledgeSource(id="src-2d", kind=SourceKind.TEXT, title="2D") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source_3d) + await fallback_repo.save_chunks( + source_3d.id, + [ + KnowledgeChunk( + id="chunk-3d", + source_id=source_3d.id, + content="three dimension content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_3d.id}, + ) + ], + ) + await fallback_repo.save_source(source_2d) + await fallback_repo.save_chunks( + source_2d.id, + [ + KnowledgeChunk( + id="chunk-2d", + source_id=source_2d.id, + content="two dimension content", + chunk_index=0, + embedding=[0.1, 0.2], + metadata={"source_id": source_2d.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = DimensionCheckingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.backfill_native_chunks() + + await repo.search_chunks("dimension", source_ids=None, top_k=1, query_embedding=[0.1, 0.2, 0.3]) + + assert native_index.searches == [([0.1, 0.2, 0.3], [source_3d.id], 1)] + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_handles_dimension_mismatched_sources( + tmp_path: Path, +): + native_index = DimensionCheckingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source_3d = KnowledgeSource(id="src-3d-fallback", kind=SourceKind.TEXT, title="3D Fallback") + source_2d = KnowledgeSource(id="src-2d-fallback", kind=SourceKind.TEXT, title="2D Fallback") + await repo.save_source(source_3d) + await repo.save_chunks( + source_3d.id, + [ + KnowledgeChunk( + id="chunk-3d-fallback", + source_id=source_3d.id, + content="three dimension fallback", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_3d.id}, + ) + ], + ) + await repo.save_source(source_2d) + await repo.save_chunks( + source_2d.id, + [ + KnowledgeChunk( + id="chunk-2d-fallback", + source_id=source_2d.id, + content="two dimension fallback", + chunk_index=0, + embedding=[0.1, 0.2], + metadata={"source_id": source_2d.id}, + ) + ], + ) + + results = await repo.search_chunks( + "two dimension", + source_ids=[source_2d.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [] + assert results + assert results[0]["chunk"].id == "chunk-2d-fallback" + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_handles_missing_native_collection_for_eligible_chunks( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-legacy-eligible", kind=SourceKind.TEXT, title="Legacy Eligible") + seed_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await seed_repo.save_source(source) + await seed_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-legacy-eligible", + source_id=source.id, + content="legacy eligible fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await seed_repo.close() + + native_index = QueryFailingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=True) + + results = await repo.search_chunks( + "legacy eligible", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results + assert results[0]["chunk"].id == "chunk-legacy-eligible" + + +@pytest.mark.asyncio +async def test_repository_sqlite_fallback_keeps_vector_only_zero_cosine_results( + tmp_path: Path, +): + source = KnowledgeSource(id="src-zero-cosine", kind=SourceKind.TEXT, title="Zero Cosine") + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=QueryFailingNativeIndex(), + allow_sqlite_vector_fallback=True, + ) + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-zero-cosine", + source_id=source.id, + content="orthogonal vector content without lexical query overlap", + chunk_index=0, + embedding=[0.0, 1.0], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "unrelated", + source_ids=[source.id], + top_k=1, + query_embedding=[1.0, 0.0], + ) + + assert results + assert results[0]["chunk"].id == "chunk-zero-cosine" + assert results[0]["score"] == 0.0 + + +@pytest.mark.asyncio +async def test_strict_native_search_rejects_empty_query_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-empty-query", kind=SourceKind.TEXT, title="Empty Query") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-query", + source_id=source.id, + content="empty query content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with pytest.raises(RuntimeError, match="non-empty query embedding"): + await repo.search_chunks( + "empty query", + source_ids=[source.id], + top_k=1, + query_embedding=[], + ) + + assert native_index.searches == [] + + +@pytest.mark.asyncio +async def test_sqlite_fallback_handles_empty_query_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-empty-query-fallback", kind=SourceKind.TEXT, title="Empty Query Fallback") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-query-fallback", + source_id=source.id, + content="empty query fallback content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + results = await repo.search_chunks( + "empty query fallback", + source_ids=[source.id], + top_k=1, + query_embedding=[], + ) + + assert native_index.searches == [] + assert results + assert results[0]["chunk"].id == "chunk-empty-query-fallback" + + +@pytest.mark.asyncio +async def test_sqlite_fallback_respects_empty_source_selection(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-empty-selection", kind=SourceKind.TEXT, title="Empty Selection") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-empty-selection", + source_id=source.id, + content="empty selection fallback content", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + + no_embedding_results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=None, + ) + empty_embedding_results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=[], + ) + + assert no_embedding_results == [] + assert empty_embedding_results == [] + + +@pytest.mark.asyncio +async def test_strict_native_search_respects_empty_source_selection_without_embedding(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + results = await repo.search_chunks( + "empty selection", + source_ids=[], + top_k=1, + query_embedding=None, + ) + + assert results == [] + assert native_index.searches == [] + + @pytest.mark.asyncio async def test_source_service_ingestion_and_vector_store_search_use_native_seekdb(tmp_path: Path): native_index = RecordingNativeIndex() From 50e9f55dc83d79035257c4709335d6cf764a9398 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 12:28:37 +0800 Subject: [PATCH 16/22] fix(seekdb): separate native vectors from sqlite metadata --- .../repositories/seekdb_repository.py | 253 +++++++++++--- .../vector_stores/seekdb_chunk_index.py | 45 +++ tests/test_chunking_and_rerank.py | 10 + tests/test_seekdb_chunk_index.py | 38 ++- tests/test_source_knowledge_base.py | 314 +++++++++++++++++- 5 files changed, 607 insertions(+), 53 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 0c173d9..696e7f4 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -1,8 +1,8 @@ -"""SeekDB repository wrapper. +"""SQLite entity repository with a native SeekDB knowledge-chunk index. -SQLite stores source and chunk payload metadata. Native SeekDB owns chunk vector -indexing whenever embeddings are present; SQLite vector scoring is retained only -as an explicit fallback path. +In strict mode, SeekDB owns chunk vectors and retrieval. SQLite keeps source and +chunk metadata without embeddings. SQLite vector scoring is retained only for +the explicitly enabled compatibility fallback. """ from __future__ import annotations @@ -128,6 +128,7 @@ def _init_schema(self) -> None: content TEXT NOT NULL, chunk_index INTEGER NOT NULL, embedding TEXT, + vector_state TEXT NOT NULL DEFAULT 'legacy', payload TEXT NOT NULL, FOREIGN KEY(source_id) REFERENCES sources(id) ON DELETE CASCADE ); @@ -182,6 +183,13 @@ def _init_schema(self) -> None: CREATE INDEX IF NOT EXISTS idx_slide_deck_jobs_deck_id ON slide_deck_jobs(deck_id); """ ) + chunk_columns = { + row["name"] for row in self._conn.execute("PRAGMA table_info(chunks)").fetchall() + } + if "vector_state" not in chunk_columns: + self._conn.execute( + "ALTER TABLE chunks ADD COLUMN vector_state TEXT NOT NULL DEFAULT 'legacy'" + ) self._conn.commit() async def close(self) -> None: @@ -235,11 +243,24 @@ async def delete_source(self, source_id: str) -> bool: if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) - with self._conn: - if self.native_chunk_index is not None: - self.native_chunk_index.upsert_source_chunks(source_id, []) - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + previous_native_chunks: list[KnowledgeChunk] | None = None + if self.native_chunk_index is not None: + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) + try: + with self._conn: + if self.native_chunk_index is not None: + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + except Exception: + if self.native_chunk_index is not None and previous_native_chunks is not None: + try: + self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks after a source delete failure") + raise return True async def delete_source_metadata_only(self, source_id: str) -> None: @@ -264,37 +285,47 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non raise RuntimeError("native SeekDB chunk writes require same-dimension embeddings for all chunks") native_chunks: list[KnowledgeChunk] | None = None + previous_native_chunks: list[KnowledgeChunk] | None = None if self.native_chunk_index is not None: native_chunks = self._native_eligible_chunks(chunks) + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) - with self._conn: - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.executemany( - """ - INSERT INTO chunks(id, source_id, content, chunk_index, embedding, payload) - VALUES (?, ?, ?, ?, ?, ?) - """, - [ - ( - chunk.id, - chunk.source_id, - chunk.content, - chunk.chunk_index, - json.dumps(chunk.embedding) if chunk.embedding is not None else None, - _dump_model(chunk), - ) - for chunk in chunks - ], - ) - if native_chunks is not None: - self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) + try: + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.executemany( + """ + INSERT INTO chunks(id, source_id, content, chunk_index, embedding, vector_state, payload) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [self._sqlite_chunk_row(chunk) for chunk in chunks], + ) + if native_chunks is not None: + self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) + except Exception: + if native_chunks is not None and previous_native_chunks is not None: + try: + self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks after a metadata transaction failure") + raise async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: - rows = self._conn.execute( - "SELECT payload FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", - (source_id,), - ).fetchall() - return [KnowledgeChunk.model_validate_json(row["payload"]) for row in rows] + if self.native_chunk_index is not None: + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + try: + chunks = get_native_chunks(source_id) + if chunks or not self.allow_sqlite_vector_fallback: + return chunks + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + if not self.allow_sqlite_vector_fallback: + raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) + return self._get_sqlite_chunks_sync(source_id) async def backfill_native_chunks(self) -> int: if self.native_chunk_index is None: @@ -304,9 +335,73 @@ async def backfill_native_chunks(self) -> int: written = 0 for source in await self.list_sources(): - chunks = await self.get_chunks(source.id) + sqlite_rows = self._conn.execute( + "SELECT payload, vector_state FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", + (source.id,), + ).fetchall() + chunks = [KnowledgeChunk.model_validate_json(row["payload"]) for row in sqlite_rows] + if not chunks: + continue + reconciled_states = {"seekdb", "sqlite_fallback_reconciled"} + if all(row["vector_state"] in reconciled_states for row in sqlite_rows): + continue chunks_with_embeddings = self._native_eligible_chunks(chunks) - self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + previous_native_chunks = get_native_chunks(source.id) if callable(get_native_chunks) else None + if len(chunks_with_embeddings) != len(chunks): + if all(row["vector_state"] == "legacy" for row in sqlite_rows) and self._chunks_match_without_vectors( + chunks, + previous_native_chunks or [], + ): + with self._conn: + self._conn.execute( + "UPDATE chunks SET vector_state = 'seekdb' WHERE source_id = ?", + (source.id,), + ) + continue + logger.warning( + "Clearing stale native chunks for fallback source %s because embeddings are missing or inconsistent", + source.id, + ) + try: + self.native_chunk_index.upsert_source_chunks(source.id, []) + with self._conn: + self._conn.execute( + """ + UPDATE chunks + SET vector_state = 'sqlite_fallback_reconciled' + WHERE source_id = ? + """, + (source.id,), + ) + except Exception: + if previous_native_chunks is not None: + try: + self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks after a backfill clear failure") + raise + continue + try: + self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + if not self.allow_sqlite_vector_fallback: + with self._conn: + for chunk in chunks: + self._conn.execute( + """ + UPDATE chunks + SET embedding = NULL, payload = ?, vector_state = 'seekdb' + WHERE id = ? + """, + (self._chunk_payload_without_embedding(chunk), chunk.id), + ) + except Exception: + if previous_native_chunks is not None: + try: + self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks after a backfill failure") + raise written += len(chunks_with_embeddings) return written @@ -320,6 +415,42 @@ async def search_chunks( ) -> list[dict]: if source_ids == []: return [] + if self.native_chunk_index is not None and query_embedding is None: + query_text = query.strip() + if not query_text: + raise RuntimeError("native SeekDB full-text search requires query text") + requested_source_ids = source_ids if source_ids is not None else self._current_source_ids() + selected_source_ids = self._native_source_ids_with_chunks(requested_source_ids) + text_search = getattr(self.native_chunk_index, "text_search", None) + results: list[dict] = [] + if not callable(text_search): + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB full-text search is unavailable") + selected_source_ids = [] + elif selected_source_ids: + try: + results.extend(text_search(query_text, selected_source_ids, top_k)) + except Exception: + if not self.allow_sqlite_vector_fallback: + raise + selected_source_ids = [] + if self.allow_sqlite_vector_fallback: + native_source_id_set = set(selected_source_ids) + fallback_source_ids = [ + source_id for source_id in requested_source_ids if source_id not in native_source_id_set + ] + if fallback_source_ids: + results.extend( + await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(fallback_source_ids), + top_k, + None, + None, + ) + ) + results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + return await self._maybe_rerank(query_text, results, top_k, rerank_provider) if self.native_chunk_index is not None and query_embedding is not None: if not query_embedding: if self.allow_sqlite_vector_fallback: @@ -364,8 +495,6 @@ async def search_chunks( results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] return await self._maybe_rerank(query, results, top_k, rerank_provider) - if self.native_chunk_index is not None and not self.allow_sqlite_vector_fallback: - raise RuntimeError("native SeekDB vector search requires query embeddings") if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) @@ -398,8 +527,11 @@ def _native_search_source_ids(self, source_ids: list[str], query_embedding: list return [] query_dimension = len(query_embedding) selected: list[str] = [] + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if not callable(get_native_chunks): + raise RuntimeError("native SeekDB chunk index cannot inspect stored chunks") for source_id in source_ids: - chunks = self._native_eligible_chunks(self._get_chunks_sync(source_id)) + chunks = self._native_eligible_chunks(get_native_chunks(source_id)) if not chunks: continue if len(chunks[0].embedding or []) != query_dimension: @@ -407,13 +539,52 @@ def _native_search_source_ids(self, source_ids: list[str], query_embedding: list selected.append(source_id) return selected - def _get_chunks_sync(self, source_id: str) -> list[KnowledgeChunk]: + def _native_source_ids_with_chunks(self, source_ids: list[str]) -> list[str]: + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if not callable(get_native_chunks): + if self.allow_sqlite_vector_fallback: + return [] + raise RuntimeError("native SeekDB chunk index cannot inspect stored chunks") + return [source_id for source_id in source_ids if get_native_chunks(source_id)] + + def _get_sqlite_chunks_sync(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( "SELECT payload FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", (source_id,), ).fetchall() return [KnowledgeChunk.model_validate_json(row["payload"]) for row in rows] + @staticmethod + def _chunk_payload_without_embedding(chunk: KnowledgeChunk) -> str: + return chunk.model_copy(update={"embedding": None}).model_dump_json() + + @staticmethod + def _chunks_match_without_vectors( + sqlite_chunks: list[KnowledgeChunk], + native_chunks: list[KnowledgeChunk], + ) -> bool: + if len(sqlite_chunks) != len(native_chunks): + return False + + def signature(chunk: KnowledgeChunk) -> dict[str, Any]: + return chunk.model_dump(exclude={"embedding", "created_at"}) + + sqlite_signatures = {chunk.id: signature(chunk) for chunk in sqlite_chunks} + native_signatures = {chunk.id: signature(chunk) for chunk in native_chunks} + return sqlite_signatures == native_signatures + + def _sqlite_chunk_row(self, chunk: KnowledgeChunk) -> tuple[Any, ...]: + persist_embedding = self.allow_sqlite_vector_fallback + return ( + chunk.id, + chunk.source_id, + chunk.content, + chunk.chunk_index, + json.dumps(chunk.embedding) if persist_embedding and chunk.embedding is not None else None, + "sqlite_fallback" if persist_embedding else "seekdb", + _dump_model(chunk) if persist_embedding else self._chunk_payload_without_embedding(chunk), + ) + def _chunk_rows(self, source_ids: list[str] | None = None) -> list[sqlite3.Row]: if source_ids == []: return [] diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index 968ee71..fdab067 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -146,6 +146,20 @@ def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: except Exception as exc: logger.warning("Failed to delete SeekDB chunks for source %s: %s", source_id, exc) + def get_source_chunks(self, source_id: str) -> list[KnowledgeChunk]: + collection = self._existing_collection(source_id) + if collection is None: + return [] + result = collection.get(include=["metadatas"]) + metadatas = result.get("metadatas", []) or [] + chunks: list[KnowledgeChunk] = [] + for metadata in metadatas: + try: + chunks.append(KnowledgeChunk.model_validate_json(metadata["payload"])) + except Exception as exc: + raise ValueError("Malformed SeekDB chunk payload") from exc + return sorted(chunks, key=lambda chunk: chunk.chunk_index) + def search( self, query_embedding: list[float], @@ -170,6 +184,37 @@ def search( return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + def text_search( + self, + query_text: str, + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + query_text = query_text.strip() + if not query_text: + raise ValueError("Query text is required") + if top_k <= 0 or not source_ids: + return [] + + results: list[dict[str, Any]] = [] + include = ["documents", "metadatas", "distances"] + n_results = max(top_k, 1) + for source_id in source_ids: + collection = self._collection(source_id) + collection_hybrid_search = getattr(collection, "hybrid_search", None) + if not callable(collection_hybrid_search): + raise RuntimeError("SeekDB collection does not support native full-text search") + query_result = collection_hybrid_search( + query={"where_document": {"$contains": query_text}, "n_results": n_results}, + knn=None, + rank=None, + n_results=n_results, + include=include, + ) + results.extend(self._rows_to_results(query_result, score_mode="similarity")) + + return sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] + def hybrid_search( self, query_text: str, diff --git a/tests/test_chunking_and_rerank.py b/tests/test_chunking_and_rerank.py index 6f73992..5484d89 100644 --- a/tests/test_chunking_and_rerank.py +++ b/tests/test_chunking_and_rerank.py @@ -17,9 +17,14 @@ def sqlite_fallback_repo(path: Path) -> SeekDBRepository: class RecordingNativeIndex: def __init__(self) -> None: self.upserts = [] + self.chunks_by_source = {} def upsert_source_chunks(self, source_id, chunks): self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + return list(self.chunks_by_source.get(source_id, [])) def search(self, query_embedding, source_ids, top_k): return [] @@ -38,9 +43,14 @@ class HybridRecordingIndex: def __init__(self) -> None: self.calls = [] self.upserts = [] + self.chunks_by_source = {} def upsert_source_chunks(self, source_id, chunks): self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + return list(self.chunks_by_source.get(source_id, [])) def hybrid_search(self, query_text, query_embedding, source_ids, top_k): self.calls.append((query_text, query_embedding, source_ids, top_k)) diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 44dc749..6419cf1 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -105,7 +105,13 @@ def count(self): return len(self.rows) def get(self, include=None, **kwargs): - return {"ids": list(self.rows)} + ids = list(self.rows) + result = {"ids": ids} + if include and "documents" in include: + result["documents"] = [self.rows[chunk_id]["document"] for chunk_id in ids] + if include and "metadatas" in include: + result["metadatas"] = [self.rows[chunk_id]["metadata"] for chunk_id in ids] + return result def refresh_index(self): self.refresh_calls += 1 @@ -202,6 +208,18 @@ def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Pat assert source_b_query["include"] == ["documents", "metadatas", "distances"] +def test_get_source_chunks_reads_chunk_payloads_from_seekdb(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + stored = chunk("chunk-a", "source-a", [0.1, 0.2, 0.3]) + index.upsert_source_chunks("source-a", [stored]) + + loaded = index.get_source_chunks("source-a") + + assert [item.id for item in loaded] == ["chunk-a"] + assert loaded[0].embedding == [0.1, 0.2, 0.3] + assert index.get_source_chunks("missing-source") == [] + + def test_hybrid_search_calls_seekdb_hybrid_and_reconstructs_chunks(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) @@ -227,6 +245,24 @@ def test_hybrid_search_calls_seekdb_hybrid_and_reconstructs_chunks(tmp_path: Pat ] +def test_text_search_uses_seekdb_fulltext_without_knn(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + results = index.text_search(query_text="TLS handshake", source_ids=["source-a"], top_k=1) + + assert [item["chunk"].id for item in results] == ["chunk-a"] + assert results[0]["backend"] == "seekdb" + collection = FakeClient.instances[0].collections[index.collection_name("source-a")] + assert collection.hybrid_calls[-1] == { + "query": {"where_document": {"$contains": "TLS handshake"}, "n_results": 1}, + "knn": None, + "rank": None, + "n_results": 1, + "include": ["documents", "metadatas", "distances"], + } + + def test_hybrid_search_preserves_native_relevance_score_order(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks( diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 7a3f975..c19fe4d 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -2,6 +2,7 @@ import logging from pathlib import Path +import sqlite3 from types import ModuleType, SimpleNamespace import sys import threading @@ -19,14 +20,112 @@ from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository +def test_repository_migrates_legacy_chunk_schema_with_vector_state(tmp_path: Path): + db_path = tmp_path / "legacy.db" + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE sources ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, payload TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, source_id TEXT NOT NULL, content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, embedding TEXT, payload TEXT NOT NULL + ); + """ + ) + conn.close() + + repo = SeekDBRepository( + db_path, + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + columns = {row["name"] for row in repo._conn.execute("PRAGMA table_info(chunks)").fetchall()} + assert "vector_state" in columns + + +@pytest.mark.asyncio +async def test_backfill_recognizes_pre_vector_state_strict_native_rows(tmp_path: Path): + db_path = tmp_path / "legacy-strict.db" + source = KnowledgeSource(id="src-legacy-strict", kind=SourceKind.TEXT, title="Legacy strict") + native_chunk = KnowledgeChunk( + id="chunk-legacy-strict", + source_id=source.id, + content="already stored natively", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + sqlite_chunk = native_chunk.model_copy(update={"embedding": None}) + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE sources ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, payload TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, source_id TEXT NOT NULL, content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, embedding TEXT, payload TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO sources VALUES (?, ?, ?, ?, ?)", + ( + source.id, + source.status.value, + source.model_dump_json(), + source.created_at.isoformat(), + source.updated_at.isoformat(), + ), + ) + conn.execute( + "INSERT INTO chunks VALUES (?, ?, ?, ?, ?, ?)", + ( + sqlite_chunk.id, + sqlite_chunk.source_id, + sqlite_chunk.content, + sqlite_chunk.chunk_index, + None, + sqlite_chunk.model_dump_json(), + ), + ) + conn.commit() + conn.close() + native_index = RecordingNativeIndex() + native_index.chunks_by_source[source.id] = [native_chunk] + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + written = await repo.backfill_native_chunks() + + assert written == 0 + assert native_index.upserts == [] + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [native_chunk.id] + state = repo._conn.execute( + "SELECT vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone()["vector_state"] + assert state == "seekdb" + + class RecordingNativeIndex: def __init__(self) -> None: self.upserts = [] self.deletes = [] self.searches = [] + self.text_searches = [] + self.chunks_by_source = {} def upsert_source_chunks(self, source_id, chunks): self.upserts.append((source_id, chunks)) + self.chunks_by_source[source_id] = list(chunks) + + def get_source_chunks(self, source_id): + return list(self.chunks_by_source.get(source_id, [])) def delete_source_chunks(self, source_id, chunk_ids): self.deletes.append((source_id, chunk_ids)) @@ -48,6 +147,23 @@ def search(self, query_embedding, source_ids, top_k): } ] + def text_search(self, query_text, source_ids, top_k): + self.text_searches.append((query_text, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_text_chunk", + source_id=source_ids[0], + content="native fulltext result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.97, + "backend": "seekdb", + } + ] + def status(self): return {"vector_backend": "seekdb", "native_available": True} @@ -92,6 +208,20 @@ def upsert_source_chunks(self, source_id, chunks): raise RuntimeError(self.message) +class FailOnceNativeIndex(RecordingNativeIndex): + def __init__(self, source_id: str, existing_chunks: list[KnowledgeChunk]) -> None: + super().__init__() + self.chunks_by_source[source_id] = list(existing_chunks) + self.fail_next_upsert = True + + def upsert_source_chunks(self, source_id, chunks): + if self.fail_next_upsert: + self.fail_next_upsert = False + self.chunks_by_source[source_id] = list(chunks) + raise RuntimeError("native write failed") + super().upsert_source_chunks(source_id, chunks) + + class DimensionCheckingNativeIndex(RecordingNativeIndex): def search(self, query_embedding, source_ids, top_k): bad_source_ids = [ @@ -161,6 +291,40 @@ async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp assert "Skipping optional pyseekdb chunk mirror" not in caplog.text +@pytest.mark.asyncio +async def test_strict_native_mode_keeps_vectors_out_of_sqlite(tmp_path: Path): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native-only", kind=SourceKind.TEXT, title="Native only") + chunk = KnowledgeChunk( + id="chunk-native-only", + source_id=source.id, + content="stored in SeekDB", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + await repo.save_source(source) + + await repo.save_chunks(source.id, [chunk]) + + sqlite_row = repo._conn.execute( + "SELECT embedding, payload, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert sqlite_row is not None + assert sqlite_row["embedding"] is None + assert '"embedding":null' in sqlite_row["payload"] + assert sqlite_row["vector_state"] == "seekdb" + loaded = await repo.get_chunks(source.id) + assert [item.id for item in loaded] == [chunk.id] + assert loaded[0].embedding == [0.1, 0.2, 0.3] + + @pytest.mark.asyncio async def test_repository_requires_embeddings_when_native_seekdb_is_primary(tmp_path: Path): repo = SeekDBRepository( @@ -210,7 +374,9 @@ async def test_repository_requires_native_seekdb_for_vector_save_unless_fallback ) ], ) - assert await repo.get_chunks(source.id) == [] + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert repo._get_sqlite_chunks_sync(source.id) == [] @pytest.mark.asyncio @@ -234,7 +400,9 @@ async def test_strict_empty_save_with_native_unavailable_keeps_existing_sqlite_c with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): await repo.save_chunks(source.id, []) - assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] @pytest.mark.asyncio @@ -263,7 +431,7 @@ async def test_repository_requires_native_seekdb_for_non_vector_chunk_save_unles @pytest.mark.asyncio -async def test_repository_requires_query_embedding_when_native_seekdb_is_primary(tmp_path: Path): +async def test_repository_uses_native_fulltext_when_query_embedding_is_unavailable(tmp_path: Path): native_index = RecordingNativeIndex() repo = SeekDBRepository( tmp_path / "knowledge.db", @@ -286,8 +454,23 @@ async def test_repository_requires_query_embedding_when_native_seekdb_is_primary ], ) - with pytest.raises(RuntimeError, match="native SeekDB vector search requires query embeddings"): - await repo.search_chunks("native query", source_ids=[source.id], top_k=1) + results = await repo.search_chunks("native query", source_ids=[source.id], top_k=1) + + assert native_index.text_searches == [("native query", [source.id], 1)] + assert results[0]["chunk"].id == "native_text_chunk" + assert results[0]["backend"] == "seekdb" + + +@pytest.mark.asyncio +async def test_strict_native_get_chunks_rejects_unavailable_seekdb(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks("src-missing-native") @pytest.mark.asyncio @@ -410,7 +593,9 @@ async def test_delete_source_requires_native_seekdb_before_sqlite_mutation(tmp_p await repo.delete_source(source.id) assert await repo.get_source(source.id) is not None - assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-strict-delete"] + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.get_chunks(source.id) + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-strict-delete"] @pytest.mark.asyncio @@ -429,9 +614,10 @@ async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_pat await seed_repo.save_source(source) await seed_repo.save_chunks(source.id, [old_chunk]) await seed_repo.close() + native_index = FailOnceNativeIndex(source.id, [old_chunk]) repo = SeekDBRepository( db_path, - native_chunk_index=FailingNativeIndex(), + native_chunk_index=native_index, allow_sqlite_vector_fallback=False, ) @@ -451,6 +637,40 @@ async def test_native_upsert_failure_rolls_back_sqlite_chunk_replacement(tmp_pat ) assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] + + +@pytest.mark.asyncio +async def test_delete_source_restores_native_chunks_when_sqlite_delete_fails(tmp_path: Path): + source = KnowledgeSource(id="src-delete-compensation", kind=SourceKind.TEXT, title="Delete compensation") + old_chunk = KnowledgeChunk( + id="chunk-delete-compensation", + source_id=source.id, + content="must survive a failed delete", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [old_chunk]) + repo._conn.execute( + """ + CREATE TRIGGER fail_source_delete BEFORE DELETE ON sources + BEGIN SELECT RAISE(ABORT, 'delete failed'); END + """ + ) + + with pytest.raises(Exception, match="delete failed"): + await repo.delete_source(source.id) + + assert await repo.get_source(source.id) is not None + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == [old_chunk.id] @pytest.mark.asyncio @@ -502,7 +722,8 @@ async def test_strict_native_rejects_mixed_dimension_embeddings_before_sqlite_mu ) assert native_index.upserts == [] - assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] + assert await repo.get_chunks(source.id) == [] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] @pytest.mark.asyncio @@ -576,10 +797,63 @@ async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_p ) written = await repo.backfill_native_chunks() + second_written = await repo.backfill_native_chunks() assert written == 1 + assert second_written == 0 + assert len(native_index.upserts) == 1 assert native_index.upserts[0][0] == source.id assert native_index.upserts[0][1][0].id == "chunk-backfill" + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == ["chunk-backfill"] + sqlite_row = repo._conn.execute( + "SELECT embedding, payload FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert sqlite_row["embedding"] is None + assert '"embedding":null' in sqlite_row["payload"] + + +@pytest.mark.asyncio +async def test_backfill_restores_native_chunks_when_sqlite_scrub_fails(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-backfill-compensation", kind=SourceKind.TEXT, title="Backfill compensation") + legacy_chunk = KnowledgeChunk( + id="chunk-legacy", + source_id=source.id, + content="legacy vector", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + previous_native_chunk = KnowledgeChunk( + id="chunk-native-old", + source_id=source.id, + content="previous native vector", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks(source.id, [legacy_chunk]) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + native_index.chunks_by_source[source.id] = [previous_native_chunk] + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + repo._conn.execute( + """ + CREATE TRIGGER fail_chunk_scrub BEFORE UPDATE ON chunks + BEGIN SELECT RAISE(ABORT, 'scrub failed'); END + """ + ) + + with pytest.raises(Exception, match="scrub failed"): + await repo.backfill_native_chunks() + + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [previous_native_chunk.id] + sqlite_chunk = repo._get_sqlite_chunks_sync(source.id)[0] + assert sqlite_chunk.embedding == [0.1, 0.2, 0.3] @pytest.mark.asyncio @@ -595,7 +869,7 @@ async def test_repository_backfill_requires_native_index_without_fallback(tmp_pa @pytest.mark.asyncio -async def test_repository_backfill_clears_native_chunks_for_sources_without_embeddings(tmp_path: Path): +async def test_repository_backfill_clears_stale_native_for_fallback_rows_without_embeddings(tmp_path: Path): db_path = tmp_path / "knowledge.db" source = KnowledgeSource(id="src-backfill-clear", kind=SourceKind.TEXT, title="Backfill Clear") fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) @@ -615,9 +889,20 @@ async def test_repository_backfill_clears_native_chunks_for_sources_without_embe await fallback_repo.close() native_index = StaleAwareNativeIndex() + native_index.chunks_by_source[source.id] = [ + KnowledgeChunk( + id="stale-native", + source_id=source.id, + content="stale native content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ] repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) written = await repo.backfill_native_chunks() + second_written = await repo.backfill_native_chunks() results = await repo.search_chunks( "plain backfill", source_ids=[source.id], @@ -626,7 +911,14 @@ async def test_repository_backfill_clears_native_chunks_for_sources_without_embe ) assert written == 0 + assert second_written == 0 assert native_index.upserts == [(source.id, [])] + assert native_index.get_source_chunks(source.id) == [] + state = repo._conn.execute( + "SELECT vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone()["vector_state"] + assert state == "sqlite_fallback_reconciled" assert results == [] @@ -681,7 +973,7 @@ async def test_repository_native_search_skips_legacy_sources_without_embeddings_ @pytest.mark.asyncio -async def test_repository_backfill_clears_partially_embedded_sources_and_skips_native_search( +async def test_repository_backfill_skips_partially_embedded_sources_and_native_search( tmp_path: Path, ): db_path = tmp_path / "knowledge.db" @@ -942,7 +1234,7 @@ async def test_repository_sqlite_fallback_handles_missing_native_collection_for_ query_embedding=[0.1, 0.2, 0.3], ) - assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert native_index.searches == [] assert results assert results[0]["chunk"].id == "chunk-legacy-eligible" From c2b87c64d3e0755d739344e4772f4f183d298862 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 12:28:56 +0800 Subject: [PATCH 17/22] test(seekdb): add native vector verification script --- scripts/verify_seekdb_native.py | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 scripts/verify_seekdb_native.py diff --git a/scripts/verify_seekdb_native.py b/scripts/verify_seekdb_native.py new file mode 100644 index 0000000..0fb6432 --- /dev/null +++ b/scripts/verify_seekdb_native.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind +from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + warning_messages: list[str] = [] + + class WarningCapture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + if record.levelno >= logging.WARNING: + warning_messages.append(record.getMessage()) + + warning_capture = WarningCapture() + root_logger = logging.getLogger() + root_logger.addHandler(warning_capture) + repo = SeekDBRepository(Path(tmp) / "knowledge.db", allow_sqlite_vector_fallback=False) + try: + source = KnowledgeSource(id="verify-source", kind=SourceKind.TEXT, title="Verify") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="verify-chunk", + source_id=source.id, + content="SeekDB native vector retrieval verification", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + results = await repo.search_chunks( + "SeekDB native retrieval", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + if not results: + raise RuntimeError("expected at least one native SeekDB result") + if results[0]["chunk"].id != "verify-chunk": + raise RuntimeError("native SeekDB returned the wrong chunk") + if results[0].get("backend") != "seekdb": + raise RuntimeError("retrieval did not use the native SeekDB backend") + text_results = await repo.search_chunks( + "native vector retrieval verification", + source_ids=[source.id], + top_k=1, + ) + if not text_results or text_results[0].get("backend") != "seekdb": + raise RuntimeError("full-text retrieval did not use the native SeekDB backend") + if text_results[0]["chunk"].id != "verify-chunk": + raise RuntimeError("native SeekDB full-text search returned the wrong chunk") + if repo.storage_status()["vector_backend"] != "seekdb": + raise RuntimeError("repository did not report the native SeekDB backend") + loaded_chunks = await repo.get_chunks(source.id) + if len(loaded_chunks) != 1 or loaded_chunks[0].embedding != [0.1, 0.2, 0.3]: + raise RuntimeError("repository did not read the stored chunk back from SeekDB") + sqlite_row = repo._conn.execute( + "SELECT embedding, payload, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + if sqlite_row is None: + raise RuntimeError("SQLite chunk metadata row is missing") + if sqlite_row["embedding"] is not None or '"embedding":null' not in sqlite_row["payload"]: + raise RuntimeError("SQLite retained a chunk vector in strict SeekDB mode") + if sqlite_row["vector_state"] != "seekdb": + raise RuntimeError("SQLite chunk metadata does not mark SeekDB as vector authority") + if any("Skipping optional pyseekdb chunk mirror" in message for message in warning_messages): + raise RuntimeError("legacy optional pyseekdb mirror warning was emitted") + finally: + await repo.close() + root_logger.removeHandler(warning_capture) + print("SeekDB native vector verification passed") + + +if __name__ == "__main__": + asyncio.run(main()) From 936c0e9d3457056e5ad8f36971578e40d0379325 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 13:50:56 +0800 Subject: [PATCH 18/22] docs: add SeekDB primary retrieval implementation plan --- ...6-07-09-seekdb-primary-vector-retrieval.md | 1405 +++++++++++++++++ 1 file changed, 1405 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md diff --git a/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md b/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md new file mode 100644 index 0000000..300e30c --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-seekdb-primary-vector-retrieval.md @@ -0,0 +1,1405 @@ +# SeekDB Primary Vector Retrieval Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make SeekDB the real primary vector and hybrid retrieval backend for NotebookLM-Lite RAG chunks, with SQLite kept only for entity metadata and explicit fallback. + +**Architecture:** Keep the current repository interface and SQLite-backed CRUD tables for sources, artifacts, notes, jobs, and slide-deck state. Move chunk embedding writes and retrieval into a dedicated SeekDB native chunk index using `pyseekdb` collections with explicit HNSW dimensions and `embedding_function=None`; SQLite chunk rows remain a metadata/fallback copy, not the default vector search engine. + +**Tech Stack:** FastAPI, Python 3.12, `pyseekdb>=1.3.0`, SQLite metadata tables, LiteLLM embeddings, pytest, Playwright smoke validation. + +--- + +## Current Problem + +Current code sets `vector_store_type: seekdb`, but `backend/infrastructure/repositories/seekdb_repository.py` writes embeddings into SQLite as JSON and searches by reading SQLite rows and calculating BM25/cosine in Python. The optional `pyseekdb` mirror is attempted after SQLite commit and fails in real uploads with: + +```text +Skipping optional pyseekdb chunk mirror after save failure: execute sql failed OB_INVALID_ARGUMENT(1210): Incorrect arguments to %s +``` + +Local SDK inspection shows the failure is caused by creating a `pyseekdb` collection without matching explicit embedding settings and then searching before refreshing the native index. A working explicit-embedding path, verified against the installed `pyseekdb`, is: + +```python +import pyseekdb + +client = pyseekdb.Client(path="/tmp/seekdb") +collection = client.create_collection( + name="chunks_source_abc", + configuration=pyseekdb.HNSWConfiguration(dimension=3), + embedding_function=None, +) +collection.upsert( + ids=["chunk_1"], + documents=["content"], + metadatas=[{"source_id": "source_1", "payload": "{}"}], + embeddings=[[0.1, 0.2, 0.3]], +) +collection.refresh_index() +collection.query( + query_embeddings=[0.1, 0.2, 0.3], + n_results=1, + include=["documents", "metadatas", "distances"], +) +``` + +This plan makes that working path the default RAG chunk path. + +## File Structure + +- Create `backend/infrastructure/vector_stores/seekdb_chunk_index.py` + - Owns all native `pyseekdb` collection creation, chunk upsert/delete/query, index refresh, collection naming, score normalization, and backend status. +- Modify `backend/infrastructure/repositories/seekdb_repository.py` + - Delegates chunk vector writes/searches to `SeekDBChunkIndex`. + - Keeps SQLite rows for source/chunk payload metadata and explicit fallback. + - Removes the optional mirror warning path. +- Modify `backend/config.py` + - Adds explicit fallback configuration and default primary semantics. +- Modify `config_example.yaml` + - Documents that SeekDB native vector retrieval is default and fallback is opt-in. +- Modify `backend/infrastructure/vector_stores/seekdb_vector_store.py` + - Reports the actual backend status from the repository. +- Modify `backend/api/routes/config.py` and `backend/api/schemas/config.py` + - Exposes actual storage status to the frontend config modal. +- Modify `backend/main.py` + - Extends `/health` with actual vector backend status. +- Modify tests: + - `tests/test_seekdb_chunk_index.py` + - `tests/test_chunking_and_rerank.py` + - `tests/test_source_knowledge_base.py` + - `tests/test_runtime_config_api.py` + - `tests/test_config_settings.py` + +## Decisions + +- The scope is RAG chunk vector storage and retrieval. SQLite may remain the metadata store for non-vector entities in this iteration. +- SeekDB native mode is the default for `vector_store_type: seekdb`. +- SQLite vector search is available only when `seekdb_allow_sqlite_fallback: true` or in tests that explicitly construct the repository with fallback enabled. +- The repository must never silently label SQLite vector search as SeekDB. +- Multi-source filtering will use one SeekDB collection per source. This avoids relying on metadata `where` filtering, which did not work reliably in local SDK checks. + +## Collection Model + +For source `source_id`, native chunks are stored in: + +```text +chunks_ +``` + +Each collection uses the embedding dimension of the first chunk with an embedding: + +```python +pyseekdb.HNSWConfiguration(dimension=len(first_embedding)) +embedding_function=None +``` + +Each metadata row contains enough data to reconstruct `KnowledgeChunk`: + +```python +{ + "source_id": chunk.source_id, + "chunk_index": chunk.chunk_index, + "payload": chunk.model_dump_json(), +} +``` + +Searching selected sources queries each selected source collection, merges results by normalized score, applies optional rerank, and returns the existing `list[dict]` shape: + +```python +{"chunk": KnowledgeChunk(...), "score": 0.82} +``` + +--- + +### Task 1: Add a Native SeekDB Chunk Index + +**Files:** +- Create: `backend/infrastructure/vector_stores/seekdb_chunk_index.py` +- Test: `tests/test_seekdb_chunk_index.py` + +- [ ] **Step 1: Write failing tests for native collection creation and query** + +Create `tests/test_seekdb_chunk_index.py`: + +```python +import sys +import types +from pathlib import Path + +import pytest + +from backend.domain.source import KnowledgeChunk +from backend.infrastructure.vector_stores.seekdb_chunk_index import ( + SeekDBChunkIndex, + SeekDBUnavailableError, +) + + +class FakeHNSWConfiguration: + def __init__(self, dimension: int, distance: str = "cosine") -> None: + self.dimension = dimension + self.distance = distance + + +class FakeCollection: + def __init__(self, name: str, dimension: int) -> None: + self.name = name + self.dimension = dimension + self.upsert_calls = [] + self.delete_calls = [] + self.query_calls = [] + self.refresh_calls = 0 + self.rows = {} + + def upsert(self, ids, documents, metadatas, embeddings): + self.upsert_calls.append( + { + "ids": ids, + "documents": documents, + "metadatas": metadatas, + "embeddings": embeddings, + } + ) + for index, chunk_id in enumerate(ids): + self.rows[chunk_id] = { + "document": documents[index], + "metadata": metadatas[index], + "embedding": embeddings[index], + } + + def delete(self, ids=None, **kwargs): + self.delete_calls.append({"ids": ids, **kwargs}) + if ids: + for chunk_id in ids: + self.rows.pop(chunk_id, None) + + def query(self, query_embeddings, n_results, include): + self.query_calls.append( + { + "query_embeddings": query_embeddings, + "n_results": n_results, + "include": include, + } + ) + ids = list(self.rows)[:n_results] + return { + "ids": [ids], + "distances": [[0.0 + index for index, _ in enumerate(ids)]], + "documents": [[self.rows[chunk_id]["document"] for chunk_id in ids]], + "metadatas": [[self.rows[chunk_id]["metadata"] for chunk_id in ids]], + } + + def count(self): + return len(self.rows) + + def refresh_index(self): + self.refresh_calls += 1 + + +class FakeClient: + instances = [] + + def __init__(self, path: str) -> None: + self.path = path + self.collections = {} + self.created = [] + FakeClient.instances.append(self) + + def get_or_create_collection(self, name, configuration=None, embedding_function=None): + if embedding_function is not None: + raise AssertionError("explicit LiteLLM embeddings require embedding_function=None") + if configuration is None: + raise AssertionError("native SeekDB collection must declare vector dimension") + if name not in self.collections: + self.created.append((name, configuration.dimension)) + self.collections[name] = FakeCollection(name, configuration.dimension) + return self.collections[name] + + def get_collection(self, name, embedding_function=None): + if name not in self.collections: + raise KeyError(name) + return self.collections[name] + + +@pytest.fixture(autouse=True) +def fake_pyseekdb(monkeypatch, request): + if request.node.name == "test_real_pyseekdb_upsert_refresh_and_query_round_trip": + yield + return + FakeClient.instances.clear() + module = types.SimpleNamespace(Client=FakeClient, HNSWConfiguration=FakeHNSWConfiguration) + monkeypatch.setitem(sys.modules, "pyseekdb", module) + yield + + +def chunk(chunk_id: str, source_id: str, embedding: list[float]) -> KnowledgeChunk: + return KnowledgeChunk( + id=chunk_id, + source_id=source_id, + content=f"{chunk_id} content", + chunk_index=0, + embedding=embedding, + metadata={"source_id": source_id}, + ) + + +def test_upsert_creates_dimensioned_source_collection(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + + client = FakeClient.instances[0] + assert client.created == [(index.collection_name("source-a"), 3)] + collection = client.collections[index.collection_name("source-a")] + assert collection.upsert_calls[0]["ids"] == ["chunk-a"] + assert collection.upsert_calls[0]["embeddings"] == [[0.1, 0.2, 0.3]] + assert collection.upsert_calls[0]["metadatas"][0]["source_id"] == "source-a" + assert "payload" in collection.upsert_calls[0]["metadatas"][0] + assert collection.refresh_calls == 1 + + +def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) + index.upsert_source_chunks("source-b", [chunk("chunk-b", "source-b", [0.9, 0.1, 0.1])]) + + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["source-a", "source-b"], + top_k=2, + ) + + assert [item["chunk"].id for item in results] == ["chunk-a", "chunk-b"] + assert all(item["backend"] == "seekdb" for item in results) + client = FakeClient.instances[0] + assert client.collections[index.collection_name("source-a")].query_calls + assert client.collections[index.collection_name("source-b")].query_calls + + +def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: Path, monkeypatch): + monkeypatch.setitem(sys.modules, "pyseekdb", None) + + with pytest.raises(SeekDBUnavailableError): + SeekDBChunkIndex(tmp_path / "native.seekdb") + + +def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path, monkeypatch): + pyseekdb = pytest.importorskip("pyseekdb") + monkeypatch.setitem(sys.modules, "pyseekdb", pyseekdb) + index = SeekDBChunkIndex(tmp_path / "real.seekdb") + + index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) + results = index.search( + query_embedding=[0.1, 0.2, 0.3], + source_ids=["real-source"], + top_k=1, + ) + + assert results[0]["chunk"].id == "real-chunk" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_seekdb_chunk_index.py -q +``` + +Expected: FAIL with `ModuleNotFoundError` or import error for `seekdb_chunk_index`. + +- [ ] **Step 3: Implement the native chunk index** + +Create `backend/infrastructure/vector_stores/seekdb_chunk_index.py`: + +```python +"""Native SeekDB chunk vector index. + +This module owns pyseekdb usage. The repository may keep SQLite metadata rows, +but RAG vector search should go through this index when vector_store_type=seekdb. +""" + +from __future__ import annotations + +import hashlib +import logging +from pathlib import Path +from typing import Any + +from ...domain.source import KnowledgeChunk + +logger = logging.getLogger(__name__) + + +class SeekDBUnavailableError(RuntimeError): + """Raised when native SeekDB is required but the SDK cannot be initialized.""" + + +class SeekDBChunkIndex: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.path.mkdir(parents=True, exist_ok=True) + try: + import pyseekdb # type: ignore + except Exception as exc: + raise SeekDBUnavailableError("pyseekdb is required for native SeekDB vector retrieval") from exc + + self._pyseekdb = pyseekdb + try: + self.client = pyseekdb.Client(path=str(self.path)) + except Exception as exc: + raise SeekDBUnavailableError(f"failed to initialize pyseekdb at {self.path}") from exc + + @staticmethod + def collection_name(source_id: str) -> str: + digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] + return f"chunks_{digest}" + + @staticmethod + def _first_embedding_dimension(chunks: list[KnowledgeChunk]) -> int: + for chunk in chunks: + if chunk.embedding: + return len(chunk.embedding) + raise ValueError("SeekDB native indexing requires chunk embeddings") + + def _collection(self, source_id: str, dimension: int | None = None) -> Any: + name = self.collection_name(source_id) + if dimension is None: + return self.client.get_collection(name=name, embedding_function=None) + return self.client.get_or_create_collection( + name=name, + configuration=self._pyseekdb.HNSWConfiguration(dimension=dimension), + embedding_function=None, + ) + + def upsert_source_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: + if not chunks: + return + dimension = self._first_embedding_dimension(chunks) + missing = [chunk.id for chunk in chunks if not chunk.embedding or len(chunk.embedding) != dimension] + if missing: + raise ValueError(f"all chunks require {dimension}-dimensional embeddings: {missing}") + + collection = self._collection(source_id, dimension=dimension) + collection.upsert( + ids=[chunk.id for chunk in chunks], + documents=[chunk.content for chunk in chunks], + metadatas=[ + { + "source_id": chunk.source_id, + "chunk_index": chunk.chunk_index, + "payload": chunk.model_dump_json(), + } + for chunk in chunks + ], + embeddings=[chunk.embedding for chunk in chunks], + ) + collection.refresh_index() + + def delete_source_chunks(self, source_id: str, chunk_ids: list[str]) -> None: + if not chunk_ids: + return + try: + collection = self._collection(source_id) + collection.delete(ids=chunk_ids) + except Exception as exc: + logger.warning("SeekDB native chunk delete skipped for %s: %s", source_id, exc) + + def search( + self, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_embedding: + raise ValueError("SeekDB native search requires a query embedding") + + merged: list[dict[str, Any]] = [] + per_source_k = max(top_k, 1) + for source_id in source_ids: + try: + collection = self._collection(source_id) + response = collection.query( + query_embeddings=query_embedding, + n_results=per_source_k, + include=["documents", "metadatas", "distances"], + ) + except Exception as exc: + logger.warning("SeekDB native search skipped source %s: %s", source_id, exc) + continue + + ids = (response.get("ids") or [[]])[0] + distances = (response.get("distances") or [[]])[0] + metadatas = (response.get("metadatas") or [[]])[0] + for index, chunk_id in enumerate(ids): + metadata = metadatas[index] if index < len(metadatas) else {} + payload = metadata.get("payload") + if not payload: + continue + distance = distances[index] if index < len(distances) else 1.0 + merged.append( + { + "chunk": KnowledgeChunk.model_validate_json(payload), + "score": 1.0 / (1.0 + max(float(distance), 0.0)), + "backend": "seekdb", + "id": chunk_id, + } + ) + + return sorted(merged, key=lambda item: item["score"], reverse=True)[:top_k] + + def status(self) -> dict[str, Any]: + return { + "vector_backend": "seekdb", + "seekdb_path": str(self.path), + "native_available": True, + } +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_seekdb_chunk_index.py -q +``` + +Expected: `4 passed`. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_chunk_index.py tests/test_seekdb_chunk_index.py +git commit -m "feat(seekdb): add native chunk vector index" +``` + +--- + +### Task 2: Wire Repository Chunk Writes to Native SeekDB Primary + +**Files:** +- Modify: `backend/config.py` +- Modify: `config_example.yaml` +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_source_knowledge_base.py` +- Test: `tests/test_config_settings.py` + +- [ ] **Step 1: Write failing repository tests** + +Append to `tests/test_source_knowledge_base.py`: + +```python +import logging + + +class RecordingNativeIndex: + def __init__(self) -> None: + self.upserts = [] + self.deletes = [] + self.searches = [] + + def upsert_source_chunks(self, source_id, chunks): + self.upserts.append((source_id, chunks)) + + def delete_source_chunks(self, source_id, chunk_ids): + self.deletes.append((source_id, chunk_ids)) + + def search(self, query_embedding, source_ids, top_k): + self.searches.append((query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="native_chunk", + source_id=source_ids[0], + content="native search result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.99, + "backend": "seekdb", + } + ] + + def status(self): + return {"vector_backend": "seekdb", "native_available": True} + + +@pytest.mark.asyncio +async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp_path: Path, caplog): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-native", kind=SourceKind.TEXT, title="Native") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-native", + source_id=source.id, + content="native indexed content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + + with caplog.at_level(logging.WARNING): + results = await repo.search_chunks( + "native indexed", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts[0][0] == source.id + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "native_chunk" + assert "Skipping optional pyseekdb chunk mirror" not in caplog.text + + +@pytest.mark.asyncio +async def test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=None, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-no-native", kind=SourceKind.TEXT, title="No Native") + await repo.save_source(source) + + with pytest.raises(RuntimeError, match="native SeekDB vector index is unavailable"): + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-no-native", + source_id=source.id, + content="content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) +``` + +Modify `tests/test_config_settings.py` to assert fallback is disabled by default: + +```python +def test_seekdb_native_vector_search_is_default(sample_config_file): + settings = get_settings() + + assert settings.vector_store_type == "seekdb" + assert settings.seekdb_allow_sqlite_fallback is False +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_uses_seekdb_native_index_for_chunk_save_and_search tests/test_source_knowledge_base.py::test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled tests/test_config_settings.py::test_seekdb_native_vector_search_is_default -q +``` + +Expected: FAIL because `SeekDBRepository` does not accept `native_chunk_index` or `allow_sqlite_vector_fallback`, and `Settings` lacks `seekdb_allow_sqlite_fallback`. + +- [ ] **Step 3: Add explicit config** + +Modify `backend/config.py` in `Settings`: + +```python + # SeekDB settings + seekdb_path: str = "./data/seekdb.db" + seekdb_allow_sqlite_fallback: bool = False +``` + +Modify the public dict construction in `Settings.model_config` related code if the project currently enumerates storage keys. Include: + +```python + for key in ( + "vector_store_type", + "chroma_persist_dir", + "seekdb_path", + "seekdb_allow_sqlite_fallback", + ): +``` + +Modify `config_example.yaml`: + +```yaml +vector_store: + vector_store_type: "seekdb" + seekdb_path: "./data/seekdb.db" + # SeekDB is the primary vector/hybrid retrieval backend. + # Set this true only for local debugging when pyseekdb is unavailable. + seekdb_allow_sqlite_fallback: false +``` + +If the file uses a flat `vector_store_type` block instead of `vector_store:`, preserve the existing shape and add: + +```yaml + seekdb_allow_sqlite_fallback: false +``` + +- [ ] **Step 4: Wire native chunk index into repository** + +Modify `backend/infrastructure/repositories/seekdb_repository.py`: + +```python +from ...infrastructure.vector_stores.seekdb_chunk_index import SeekDBChunkIndex, SeekDBUnavailableError +``` + +Change `__init__`: + +```python + def __init__( + self, + db_path: str | Path, + native_chunk_index: object | None = None, + allow_sqlite_vector_fallback: bool = False, + ) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback + self.native_chunk_index = native_chunk_index + if self.native_chunk_index is None: + try: + self.native_chunk_index = SeekDBChunkIndex(self.db_path.parent / f"{self.db_path.stem}.seekdb") + except SeekDBUnavailableError as exc: + if not allow_sqlite_vector_fallback: + logger.warning("Native SeekDB vector index unavailable: %s", exc) + self.native_chunk_index = None + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() +``` + +Remove `_try_pyseekdb` and `self.seekdb_client`. + +Replace the end of `save_chunks` after SQLite commit: + +```python + if self.native_chunk_index is not None: + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + return + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB vector index is unavailable; enable seekdb_allow_sqlite_fallback for SQLite fallback") +``` + +Replace the top of `search_chunks`: + +```python + if self.native_chunk_index is not None and query_embedding: + selected_source_ids = source_ids or [source.id for source in await self.list_sources()] + results = self.native_chunk_index.search( + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + if rerank_provider and results: + try: + return await rerank_provider.rerank(query, results, top_k=top_k) + except Exception: + return results + return results + if self.native_chunk_index is not None and not query_embedding: + logger.warning("SeekDB native vector search skipped because query embedding is missing") + if not self.allow_sqlite_vector_fallback: + raise RuntimeError("native SeekDB vector search requires query embeddings") +``` + +Keep the existing SQLite BM25/cosine code below as fallback only. + +Modify `delete_source` before SQLite deletion: + +```python + chunks = await self.get_chunks(source_id) + if self.native_chunk_index is not None: + self.native_chunk_index.delete_source_chunks(source_id, [chunk.id for chunk in chunks]) +``` + +Add: + +```python + def storage_status(self) -> dict[str, Any]: + if self.native_chunk_index is not None: + return self.native_chunk_index.status() + return { + "vector_backend": "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable", + "seekdb_path": str(self.db_path.parent / f"{self.db_path.stem}.seekdb"), + "native_available": False, + } +``` + +- [ ] **Step 5: Update dependency injection** + +Modify `backend/dependencies.py`: + +```python + cls._knowledge_repository = SeekDBRepository( + settings.seekdb_path, + allow_sqlite_vector_fallback=settings.seekdb_allow_sqlite_fallback, + ) +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_uses_seekdb_native_index_for_chunk_save_and_search tests/test_source_knowledge_base.py::test_repository_requires_native_seekdb_for_vector_search_unless_fallback_enabled tests/test_config_settings.py::test_seekdb_native_vector_search_is_default -q +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add backend/config.py config_example.yaml backend/dependencies.py backend/infrastructure/repositories/seekdb_repository.py tests/test_source_knowledge_base.py tests/test_config_settings.py +git commit -m "feat(seekdb): make native vector index primary" +``` + +--- + +### Task 3: Preserve Lexical Recall Without Making SQLite Primary + +**Files:** +- Modify: `backend/infrastructure/vector_stores/seekdb_chunk_index.py` +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_chunking_and_rerank.py` + +- [ ] **Step 1: Write failing test for native hybrid call** + +Append to `tests/test_chunking_and_rerank.py`: + +```python +class HybridRecordingIndex: + def __init__(self) -> None: + self.calls = [] + + def hybrid_search(self, query_text, query_embedding, source_ids, top_k): + self.calls.append((query_text, query_embedding, source_ids, top_k)) + return [ + { + "chunk": KnowledgeChunk( + id="hybrid_chunk", + source_id=source_ids[0], + content="hybrid TLS result", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source_ids[0]}, + ), + "score": 0.88, + "backend": "seekdb", + } + ] + + def search(self, query_embedding, source_ids, top_k): + raise AssertionError("hybrid_search should be preferred when query text is available") + + +@pytest.mark.asyncio +async def test_seekdb_repository_prefers_native_hybrid_search(tmp_path: Path): + native_index = HybridRecordingIndex() + repo = SeekDBRepository( + tmp_path / "hybrid.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-hybrid", kind=SourceKind.TEXT, title="Hybrid") + await repo.save_source(source) + + results = await repo.search_chunks( + "TLS handshake", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.calls == [("TLS handshake", [0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["chunk"].id == "hybrid_chunk" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_chunking_and_rerank.py::test_seekdb_repository_prefers_native_hybrid_search -q +``` + +Expected: FAIL because `hybrid_search` is not called. + +- [ ] **Step 3: Implement `hybrid_search` in `SeekDBChunkIndex`** + +Add to `backend/infrastructure/vector_stores/seekdb_chunk_index.py`: + +```python + def hybrid_search( + self, + query_text: str, + query_embedding: list[float], + source_ids: list[str], + top_k: int, + ) -> list[dict[str, Any]]: + if not query_text.strip(): + return self.search(query_embedding=query_embedding, source_ids=source_ids, top_k=top_k) + + merged: list[dict[str, Any]] = [] + for source_id in source_ids: + try: + collection = self._collection(source_id) + response = collection.hybrid_search( + query={"match": {"document": query_text}}, + knn={"query_embeddings": query_embedding, "k": max(top_k, 1)}, + n_results=max(top_k, 1), + include=["documents", "metadatas", "distances"], + ) + except AttributeError: + return self.search(query_embedding=query_embedding, source_ids=source_ids, top_k=top_k) + except Exception as exc: + logger.warning("SeekDB hybrid search skipped source %s: %s", source_id, exc) + continue + + ids = (response.get("ids") or [[]])[0] + distances = (response.get("distances") or [[]])[0] + metadatas = (response.get("metadatas") or [[]])[0] + for index, chunk_id in enumerate(ids): + metadata = metadatas[index] if index < len(metadatas) else {} + payload = metadata.get("payload") + if not payload: + continue + distance = distances[index] if index < len(distances) else 1.0 + merged.append( + { + "chunk": KnowledgeChunk.model_validate_json(payload), + "score": 1.0 / (1.0 + max(float(distance), 0.0)), + "backend": "seekdb", + "id": chunk_id, + } + ) + + return sorted(merged, key=lambda item: item["score"], reverse=True)[:top_k] +``` + +Modify `search_chunks` in `backend/infrastructure/repositories/seekdb_repository.py`: + +```python + if self.native_chunk_index is not None and query_embedding: + selected_source_ids = source_ids or [source.id for source in await self.list_sources()] + if hasattr(self.native_chunk_index, "hybrid_search"): + results = self.native_chunk_index.hybrid_search( + query_text=query, + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) + else: + results = self.native_chunk_index.search( + query_embedding=query_embedding, + source_ids=selected_source_ids, + top_k=top_k, + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_chunking_and_rerank.py::test_seekdb_repository_prefers_native_hybrid_search -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_chunk_index.py backend/infrastructure/repositories/seekdb_repository.py tests/test_chunking_and_rerank.py +git commit -m "feat(seekdb): use native hybrid retrieval" +``` + +--- + +### Task 4: Expose Actual Backend Status + +**Files:** +- Modify: `backend/infrastructure/vector_stores/seekdb_vector_store.py` +- Modify: `backend/api/schemas/config.py` +- Modify: `backend/api/routes/config.py` +- Modify: `backend/main.py` +- Test: `tests/test_runtime_config_api.py` + +- [ ] **Step 1: Write failing API status tests** + +Append to `tests/test_runtime_config_api.py`: + +```python +def test_runtime_config_exposes_actual_vector_backend(client): + response = client.get("/api/config") + + assert response.status_code == 200 + storage = response.json()["storage"] + assert "actual_vector_backend" in storage + assert storage["configured_vector_store_type"] == "seekdb" + + +def test_health_exposes_actual_vector_backend(client): + response = client.get("/health") + + assert response.status_code == 200 + assert "storage" in response.json() + assert "actual_vector_backend" in response.json()["storage"] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_runtime_config_api.py::test_runtime_config_exposes_actual_vector_backend tests/test_runtime_config_api.py::test_health_exposes_actual_vector_backend -q +``` + +Expected: FAIL because the response does not include actual backend status. + +- [ ] **Step 3: Add status helpers** + +Modify `backend/infrastructure/vector_stores/seekdb_vector_store.py`: + +```python + def storage_status(self) -> dict[str, Any]: + if hasattr(self.repository, "storage_status"): + return self.repository.storage_status() + return {"vector_backend": "unknown", "native_available": False} + + async def get_stats(self) -> dict[str, Any]: + sources = await self.repository.list_sources() + chunk_count = sum(source.chunk_count for source in sources) + status = self.storage_status() + return { + "total_documents": len(sources), + "total_chunks": chunk_count, + "backend": status["vector_backend"], + "storage": status, + } +``` + +- [ ] **Step 4: Add config schema/status fields** + +Modify `backend/api/schemas/config.py` so `RuntimeConfigResponse.storage` allows untyped status values: + +```python +class RuntimeConfigResponse(BaseModel): + models: dict[str, PublicModelProfile] + chunking: dict[str, Any] + storage: dict[str, Any] + message: str = "" +``` + +Modify `_response` in `backend/api/routes/config.py`: + +```python + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() if hasattr(vector_store, "get_stats") else {} +``` + +Because `_response` is currently synchronous, change it to: + +```python +async def _response(message: str = "") -> RuntimeConfigResponse: + settings = get_settings() + models = settings.api.models + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + storage_status = stats.get("storage", {}) + return RuntimeConfigResponse( + models={name: _public_profile(getattr(models, name)) for name in ModelProfiles.model_fields}, + chunking={ + "provider": settings.chunking.provider, + "tokenizer": settings.chunking.tokenizer, + "chunk_size": settings.chunk_size, + "chunk_overlap": settings.chunk_overlap, + }, + storage={ + "configured_vector_store_type": settings.vector_store_type, + "seekdb_path": settings.seekdb_path, + "seekdb_allow_sqlite_fallback": settings.seekdb_allow_sqlite_fallback, + "actual_vector_backend": storage_status.get("vector_backend", "unknown"), + "native_available": storage_status.get("native_available", False), + }, + message=message, + ) +``` + +Then update route returns: + +```python +@router.get("", response_model=RuntimeConfigResponse) +async def get_runtime_config() -> RuntimeConfigResponse: + return await _response() + + +@router.post("", response_model=RuntimeConfigResponse) +async def update_runtime_config(request: RuntimeConfigUpdate) -> RuntimeConfigResponse: + profile_data = { + name: profile.model_dump(exclude_none=True) + for name, profile in request.models.items() + } + update_runtime_model_profiles(profile_data) + DependencyContainer.reset_runtime_caches() + return await _response("Runtime model configuration updated.") +``` + +Modify `backend/main.py`: + +```python +@app.get("/health") +async def health_check(): + from .dependencies import get_vector_store + + vector_store = get_vector_store() + stats = await vector_store.get_stats() + return { + "status": "healthy", + "version": settings.app_version, + "storage": { + "actual_vector_backend": stats.get("storage", {}).get("vector_backend", stats.get("backend")), + "native_available": stats.get("storage", {}).get("native_available", False), + }, + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_runtime_config_api.py::test_runtime_config_exposes_actual_vector_backend tests/test_runtime_config_api.py::test_health_exposes_actual_vector_backend -q +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add backend/infrastructure/vector_stores/seekdb_vector_store.py backend/api/schemas/config.py backend/api/routes/config.py backend/main.py tests/test_runtime_config_api.py +git commit -m "feat(config): expose actual vector backend status" +``` + +--- + +### Task 5: Backfill Existing SQLite Chunk Embeddings Into SeekDB + +**Files:** +- Modify: `backend/infrastructure/repositories/seekdb_repository.py` +- Test: `tests/test_source_knowledge_base.py` + +- [ ] **Step 1: Write failing test for lazy backfill** + +Append to `tests/test_source_knowledge_base.py`: + +```python +@pytest.mark.asyncio +async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=None, + allow_sqlite_vector_fallback=True, + ) + source = KnowledgeSource(id="src-backfill", kind=SourceKind.TEXT, title="Backfill") + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-backfill", + source_id=source.id, + content="backfill content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + + await repo.backfill_native_chunks() + + assert native_index.upserts[0][0] == source.id + assert native_index.upserts[0][1][0].id == "chunk-backfill" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_backfills_existing_sqlite_chunks_to_native_index -q +``` + +Expected: FAIL because `backfill_native_chunks` does not exist. + +- [ ] **Step 3: Implement backfill** + +Add to `backend/infrastructure/repositories/seekdb_repository.py`: + +```python + async def backfill_native_chunks(self) -> int: + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + return 0 + raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + + sources = await self.list_sources() + written = 0 + for source in sources: + chunks = await self.get_chunks(source.id) + chunks_with_embeddings = [chunk for chunk in chunks if chunk.embedding] + if chunks_with_embeddings: + self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) + written += len(chunks_with_embeddings) + return written +``` + +Call it in `DependencyContainer.get_knowledge_repository` is not safe because it is async. Instead call it opportunistically from `SourceService` only after new writes and expose a manual helper for migration. Add a script in Task 6 for explicit migration. + +- [ ] **Step 4: Run test to verify it passes** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest tests/test_source_knowledge_base.py::test_repository_backfills_existing_sqlite_chunks_to_native_index -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/infrastructure/repositories/seekdb_repository.py tests/test_source_knowledge_base.py +git commit -m "feat(seekdb): backfill existing chunks into native index" +``` + +--- + +### Task 6: Add a Local Verification Script for the Warning Regression + +**Files:** +- Create: `scripts/verify_seekdb_native.py` +- Test: manual command in this task + +- [ ] **Step 1: Create verification script** + +Create `scripts/verify_seekdb_native.py`: + +```python +from __future__ import annotations + +import asyncio +import tempfile +from pathlib import Path + +from backend.domain.source import KnowledgeChunk, KnowledgeSource, SourceKind +from backend.infrastructure.repositories.seekdb_repository import SeekDBRepository + + +async def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = SeekDBRepository(Path(tmp) / "knowledge.db", allow_sqlite_vector_fallback=False) + source = KnowledgeSource(id="verify-source", kind=SourceKind.TEXT, title="Verify") + await repo.save_source(source) + await repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="verify-chunk", + source_id=source.id, + content="SeekDB native vector retrieval verification", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + results = await repo.search_chunks( + "SeekDB native retrieval", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + assert results, "expected at least one native SeekDB result" + assert results[0]["chunk"].id == "verify-chunk" + assert repo.storage_status()["vector_backend"] == "seekdb" + await repo.close() + print("SeekDB native vector verification passed") + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +- [ ] **Step 2: Run script** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 scripts/verify_seekdb_native.py +``` + +Expected: + +```text +SeekDB native vector verification passed +``` + +There must be no log line containing: + +```text +Skipping optional pyseekdb chunk mirror +``` + +- [ ] **Step 3: Commit** + +```bash +git add scripts/verify_seekdb_native.py +git commit -m "test(seekdb): add native vector verification script" +``` + +--- + +### Task 7: Full Regression and Real App Smoke + +**Files:** +- No required source edits if all prior tasks pass. +- Use existing local `config.yaml`; do not commit it. + +- [ ] **Step 1: Run backend unit tests** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m pytest -q +``` + +Expected: all tests pass. + +- [ ] **Step 2: Run backend compile check** + +Run: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m compileall -q backend +``` + +Expected: exit code `0`. + +- [ ] **Step 3: Run frontend build and tests** + +Run: + +```bash +cd frontend +npm test -- --reporter=dot +npm run build +npm run lint +npm run test:e2e -- --project=chromium +``` + +Expected: +- Vitest passes. +- Build passes. +- Lint has no errors. +- Playwright passes. + +- [ ] **Step 4: Start services** + +Run backend: + +```bash +/Users/lzj/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3 -m uvicorn backend.main:app --host 127.0.0.1 --port 8000 +``` + +Run frontend in another shell: + +```bash +cd frontend +npm run dev -- --host 127.0.0.1 --port 5173 +``` + +Expected: +- Backend logs show startup without SeekDB mirror warning. +- Frontend is available at `http://127.0.0.1:5173`. + +- [ ] **Step 5: Verify health and config report native SeekDB** + +Run: + +```bash +curl -s http://127.0.0.1:8000/health +curl -s http://127.0.0.1:8000/api/config +``` + +Expected JSON contains: + +```json +{ + "actual_vector_backend": "seekdb", + "native_available": true +} +``` + +- [ ] **Step 6: Browser smoke upload and RAG** + +Use Playwright or the browser manually: + +1. Open `http://127.0.0.1:5173`. +2. Upload `/Users/lzj/proj/notebook/NotebookLM-Lite/doc/L9.md`. +3. Ask: `理想 L9 的智能座舱有哪些核心能力？` +4. Confirm the answer references uploaded content and sources. +5. Check backend logs for absence of: + +```text +Skipping optional pyseekdb chunk mirror +``` + +- [ ] **Step 7: Commit if a smoke-only doc update is added** + +If this task adds a small note to README or docs, commit: + +```bash +git add README.md README_zn.md +git commit -m "docs: clarify SeekDB native vector retrieval" +``` + +If no doc edit is needed, skip this commit. + +--- + +## Self-Review + +**Spec coverage:** The plan makes SeekDB native chunk vector storage and retrieval primary, keeps SQLite as explicit fallback only, exposes actual backend status, and directly targets the observed `Skipping optional pyseekdb chunk mirror` warning by deleting the optional mirror path. + +**Placeholder scan:** No implementation step relies on TBD, TODO, or undefined behavior. Each code-changing task includes concrete test code, implementation code, commands, and expected results. + +**Type consistency:** The new `SeekDBChunkIndex` methods are used consistently: +- `upsert_source_chunks(source_id, chunks)` +- `delete_source_chunks(source_id, chunk_ids)` +- `search(query_embedding, source_ids, top_k)` +- `hybrid_search(query_text, query_embedding, source_ids, top_k)` +- `status()` + +**Risk boundary:** This plan does not migrate artifacts, notes, jobs, or slide decks from SQLite into SeekDB. That is a separate broader database migration. This plan fixes the vector/RAG path that currently claims SeekDB while using SQLite. From 2c1c8ded7cd8df2eaf935e5edad6a8567f61cc18 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 13:51:00 +0800 Subject: [PATCH 19/22] fix(storage): make SeekDB synchronization recoverable --- .../repositories/seekdb_repository.py | 506 +++++++++++++++--- backend/main.py | 8 + tests/test_source_knowledge_base.py | 202 ++++++- 3 files changed, 644 insertions(+), 72 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 696e7f4..6bc7bb2 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -12,8 +12,10 @@ import math import re import sqlite3 +from datetime import datetime, timezone from pathlib import Path from typing import Any +from uuid import uuid4 from ...core.interfaces.knowledge_repository import KnowledgeRepositoryInterface from ...domain.slide_deck import SlideAsset, SlideDeckExport, SlideDeckJob, SlideDeckProject @@ -103,6 +105,7 @@ def __init__( self._conn = sqlite3.connect(self.db_path, check_same_thread=False) self._conn.row_factory = sqlite3.Row self._init_schema() + self._storage_initialized = False def _try_native_chunk_index(self, path: Path) -> Any | None: try: @@ -133,6 +136,25 @@ def _init_schema(self) -> None: FOREIGN KEY(source_id) REFERENCES sources(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_chunks_source_id ON chunks(source_id); + CREATE TABLE IF NOT EXISTS chunk_index_state ( + source_id TEXT PRIMARY KEY, + revision TEXT NOT NULL, + chunk_count INTEGER NOT NULL, + embedding_dimension INTEGER, + storage_mode TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS chunk_sync_operations ( + source_id TEXT PRIMARY KEY, + revision TEXT NOT NULL, + operation TEXT NOT NULL, + payload TEXT, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS repository_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); CREATE TABLE IF NOT EXISTS artifacts ( id TEXT PRIMARY KEY, payload TEXT NOT NULL, @@ -243,23 +265,31 @@ async def delete_source(self, source_id: str) -> bool: if self.native_chunk_index is None and not self.allow_sqlite_vector_fallback: raise RuntimeError(_NATIVE_UNAVAILABLE_MESSAGE) - previous_native_chunks: list[KnowledgeChunk] | None = None - if self.native_chunk_index is not None: - get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) - if callable(get_native_chunks): - previous_native_chunks = get_native_chunks(source_id) - try: + await self._ensure_storage_initialized() + + if self.native_chunk_index is None: with self._conn: - if self.native_chunk_index is not None: - self.native_chunk_index.upsert_source_chunks(source_id, []) self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + return True + + previous_native_chunks: list[KnowledgeChunk] | None = None + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) + revision = uuid4().hex + self._record_sync_operation(source_id, revision, "delete", None) + try: + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_source_delete(source_id, revision) except Exception: - if self.native_chunk_index is not None and previous_native_chunks is not None: + if previous_native_chunks is not None: try: self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) except Exception: logger.exception("Failed to restore SeekDB chunks after a source delete failure") + self._discard_sync_operation(source_id, revision) raise return True @@ -267,6 +297,8 @@ async def delete_source_metadata_only(self, source_id: str) -> None: """Rollback local metadata without touching native SeekDB.""" with self._conn: self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_sync_operations WHERE source_id = ?", (source_id,)) self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> None: @@ -284,35 +316,52 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non if len(native_eligible_chunks) != len(chunks): raise RuntimeError("native SeekDB chunk writes require same-dimension embeddings for all chunks") - native_chunks: list[KnowledgeChunk] | None = None - previous_native_chunks: list[KnowledgeChunk] | None = None - if self.native_chunk_index is not None: - native_chunks = self._native_eligible_chunks(chunks) - get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) - if callable(get_native_chunks): - previous_native_chunks = get_native_chunks(source_id) + await self._ensure_storage_initialized() - try: + if self.native_chunk_index is None: with self._conn: - self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) - self._conn.executemany( - """ - INSERT INTO chunks(id, source_id, content, chunk_index, embedding, vector_state, payload) - VALUES (?, ?, ?, ?, ?, ?, ?) - """, - [self._sqlite_chunk_row(chunk) for chunk in chunks], + self._replace_sqlite_chunks(source_id, chunks, persist_embedding=True) + self._upsert_chunk_index_state( + source_id, + uuid4().hex, + chunks, + storage_mode="sqlite_fallback", ) - if native_chunks is not None: - self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) + return + + native_chunks = self._native_eligible_chunks(chunks) + previous_native_chunks: list[KnowledgeChunk] | None = None + get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) + if callable(get_native_chunks): + previous_native_chunks = get_native_chunks(source_id) + revision = uuid4().hex + persist_embedding = self.allow_sqlite_vector_fallback + self._record_sync_operation( + source_id, + revision, + "replace", + self._replacement_operation_payload(chunks, persist_embedding), + ) + + try: + self.native_chunk_index.upsert_source_chunks(source_id, native_chunks) + self._finalize_chunk_replace( + source_id, + revision, + chunks, + persist_embedding=persist_embedding, + ) except Exception: - if native_chunks is not None and previous_native_chunks is not None: + if previous_native_chunks is not None: try: self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) except Exception: logger.exception("Failed to restore SeekDB chunks after a metadata transaction failure") + self._discard_sync_operation(source_id, revision) raise async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: + await self._ensure_storage_initialized() if self.native_chunk_index is not None: get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) if callable(get_native_chunks): @@ -333,23 +382,66 @@ async def backfill_native_chunks(self) -> int: return 0 raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + self._recover_pending_sync_operations() + written = self._backfill_native_chunks_sync() + if not self.allow_sqlite_vector_fallback: + self._set_repository_meta("native_chunk_migration_v1", "complete") + self._storage_initialized = True + return written + + async def initialize_storage(self) -> int: + """Recover interrupted writes and migrate legacy SQLite vectors into SeekDB.""" + if self._storage_initialized or self.native_chunk_index is None: + return 0 + self._recover_pending_sync_operations() written = 0 - for source in await self.list_sources(): + if ( + not self.allow_sqlite_vector_fallback + and self._get_repository_meta("native_chunk_migration_v1") != "complete" + ): + written = self._backfill_native_chunks_sync() + self._set_repository_meta("native_chunk_migration_v1", "complete") + self._storage_initialized = True + return written + + async def _ensure_storage_initialized(self) -> None: + if not self._storage_initialized and self.native_chunk_index is not None: + await self.initialize_storage() + + def _backfill_native_chunks_sync(self) -> int: + written = 0 + source_rows = self._conn.execute( + "SELECT payload FROM sources WHERE status != ? ORDER BY created_at DESC", + (SourceStatus.DELETED.value,), + ).fetchall() + sources = [KnowledgeSource.model_validate_json(row["payload"]) for row in source_rows] + for source in sources: sqlite_rows = self._conn.execute( "SELECT payload, vector_state FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", (source.id,), ).fetchall() chunks = [KnowledgeChunk.model_validate_json(row["payload"]) for row in sqlite_rows] if not chunks: + with self._conn: + self._upsert_chunk_index_state( + source.id, + uuid4().hex, + [], + storage_mode="seekdb", + ) continue reconciled_states = {"seekdb", "sqlite_fallback_reconciled"} - if all(row["vector_state"] in reconciled_states for row in sqlite_rows): + state_row = self._conn.execute( + "SELECT source_id FROM chunk_index_state WHERE source_id = ?", + (source.id,), + ).fetchone() + if state_row and all(row["vector_state"] in reconciled_states for row in sqlite_rows): continue chunks_with_embeddings = self._native_eligible_chunks(chunks) get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) previous_native_chunks = get_native_chunks(source.id) if callable(get_native_chunks) else None if len(chunks_with_embeddings) != len(chunks): - if all(row["vector_state"] == "legacy" for row in sqlite_rows) and self._chunks_match_without_vectors( + if all(row["vector_state"] in {"legacy", "seekdb"} for row in sqlite_rows) and self._chunks_match_without_vectors( chunks, previous_native_chunks or [], ): @@ -358,49 +450,74 @@ async def backfill_native_chunks(self) -> int: "UPDATE chunks SET vector_state = 'seekdb' WHERE source_id = ?", (source.id,), ) + self._upsert_chunk_index_state( + source.id, + uuid4().hex, + previous_native_chunks or [], + storage_mode="seekdb", + ) continue logger.warning( "Clearing stale native chunks for fallback source %s because embeddings are missing or inconsistent", source.id, ) + revision = uuid4().hex + self._record_sync_operation( + source.id, + revision, + "replace", + self._replacement_operation_payload(chunks, persist_embedding=True), + ) try: self.native_chunk_index.upsert_source_chunks(source.id, []) - with self._conn: - self._conn.execute( - """ - UPDATE chunks - SET vector_state = 'sqlite_fallback_reconciled' - WHERE source_id = ? - """, - (source.id,), - ) + self._finalize_chunk_replace( + source.id, + revision, + chunks, + persist_embedding=True, + storage_mode="sqlite_fallback", + vector_state="sqlite_fallback_reconciled", + ) except Exception: if previous_native_chunks is not None: try: self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) except Exception: logger.exception("Failed to restore SeekDB chunks after a backfill clear failure") + self._discard_sync_operation(source.id, revision) raise continue + revision = uuid4().hex + persist_embedding = self.allow_sqlite_vector_fallback + finalize_mode = "replace" if persist_embedding else "scrub" + self._record_sync_operation( + source.id, + revision, + "replace", + self._replacement_operation_payload( + chunks, + persist_embedding, + finalize_mode=finalize_mode, + ), + ) try: self.native_chunk_index.upsert_source_chunks(source.id, chunks_with_embeddings) - if not self.allow_sqlite_vector_fallback: - with self._conn: - for chunk in chunks: - self._conn.execute( - """ - UPDATE chunks - SET embedding = NULL, payload = ?, vector_state = 'seekdb' - WHERE id = ? - """, - (self._chunk_payload_without_embedding(chunk), chunk.id), - ) + if finalize_mode == "scrub": + self._finalize_chunk_migration(source.id, revision, chunks) + else: + self._finalize_chunk_replace( + source.id, + revision, + chunks, + persist_embedding=persist_embedding, + ) except Exception: if previous_native_chunks is not None: try: self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) except Exception: logger.exception("Failed to restore SeekDB chunks after a backfill failure") + self._discard_sync_operation(source.id, revision) raise written += len(chunks_with_embeddings) return written @@ -415,6 +532,7 @@ async def search_chunks( ) -> list[dict]: if source_ids == []: return [] + await self._ensure_storage_initialized() if self.native_chunk_index is not None and query_embedding is None: query_text = query.strip() if not query_text: @@ -526,26 +644,265 @@ def _native_search_source_ids(self, source_ids: list[str], query_embedding: list if not source_ids: return [] query_dimension = len(query_embedding) - selected: list[str] = [] - get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) - if not callable(get_native_chunks): - raise RuntimeError("native SeekDB chunk index cannot inspect stored chunks") - for source_id in source_ids: - chunks = self._native_eligible_chunks(get_native_chunks(source_id)) - if not chunks: - continue - if len(chunks[0].embedding or []) != query_dimension: - continue - selected.append(source_id) - return selected + states = self._chunk_index_states(source_ids) + return [ + source_id + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "seekdb" + and states[source_id]["chunk_count"] > 0 + and states[source_id]["embedding_dimension"] == query_dimension + ] def _native_source_ids_with_chunks(self, source_ids: list[str]) -> list[str]: - get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) - if not callable(get_native_chunks): - if self.allow_sqlite_vector_fallback: - return [] - raise RuntimeError("native SeekDB chunk index cannot inspect stored chunks") - return [source_id for source_id in source_ids if get_native_chunks(source_id)] + states = self._chunk_index_states(source_ids) + return [ + source_id + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "seekdb" + and states[source_id]["chunk_count"] > 0 + ] + + def _record_sync_operation( + self, + source_id: str, + revision: str, + operation: str, + payload: str | None, + ) -> None: + with self._conn: + self._conn.execute( + """ + INSERT INTO chunk_sync_operations(source_id, revision, operation, payload, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + revision=excluded.revision, + operation=excluded.operation, + payload=excluded.payload, + created_at=excluded.created_at + """, + (source_id, revision, operation, payload, datetime.now(timezone.utc).isoformat()), + ) + + def _discard_sync_operation(self, source_id: str, revision: str) -> None: + try: + with self._conn: + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + except Exception: + logger.exception("Failed to discard completed chunk sync operation for %s", source_id) + + @staticmethod + def _replacement_operation_payload( + chunks: list[KnowledgeChunk], + persist_embedding: bool, + *, + finalize_mode: str = "replace", + ) -> str: + return json.dumps( + { + "chunks": [chunk.model_dump_json() for chunk in chunks], + "persist_embedding": persist_embedding, + "finalize_mode": finalize_mode, + } + ) + + @staticmethod + def _load_replacement_operation(payload: str | None) -> tuple[list[KnowledgeChunk], bool, str]: + if payload is None: + raise ValueError("replace operation is missing its payload") + value = json.loads(payload) + chunks = [KnowledgeChunk.model_validate_json(item) for item in value.get("chunks", [])] + return chunks, bool(value.get("persist_embedding", False)), value.get("finalize_mode", "replace") + + def _recover_pending_sync_operations(self) -> None: + if self.native_chunk_index is None: + return + rows = self._conn.execute( + """ + SELECT source_id, revision, operation, payload + FROM chunk_sync_operations + ORDER BY created_at ASC + """ + ).fetchall() + for row in rows: + source_id = row["source_id"] + revision = row["revision"] + if row["operation"] == "replace": + chunks, persist_embedding, finalize_mode = self._load_replacement_operation(row["payload"]) + self.native_chunk_index.upsert_source_chunks( + source_id, + self._native_eligible_chunks(chunks), + ) + if finalize_mode == "scrub": + self._finalize_chunk_migration(source_id, revision, chunks) + else: + self._finalize_chunk_replace( + source_id, + revision, + chunks, + persist_embedding=persist_embedding, + ) + elif row["operation"] == "delete": + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_source_delete(source_id, revision) + else: + raise ValueError(f"Unknown chunk sync operation: {row['operation']}") + + def _finalize_chunk_replace( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + *, + persist_embedding: bool, + storage_mode: str | None = None, + vector_state: str | None = None, + ) -> None: + eligible_chunks = self._native_eligible_chunks(chunks) + resolved_storage_mode = storage_mode or ( + "seekdb" if not chunks or len(eligible_chunks) == len(chunks) else "sqlite_fallback" + ) + with self._conn: + self._replace_sqlite_chunks( + source_id, + chunks, + persist_embedding=persist_embedding, + vector_state=vector_state, + ) + self._upsert_chunk_index_state( + source_id, + revision, + eligible_chunks if resolved_storage_mode == "seekdb" else chunks, + storage_mode=resolved_storage_mode, + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _finalize_source_delete(self, source_id: str, revision: str) -> None: + with self._conn: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM chunk_index_state WHERE source_id = ?", (source_id,)) + self._conn.execute("DELETE FROM sources WHERE id = ?", (source_id,)) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _finalize_chunk_migration( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + ) -> None: + with self._conn: + for chunk in chunks: + self._conn.execute( + """ + UPDATE chunks + SET embedding = NULL, payload = ?, vector_state = 'seekdb' + WHERE id = ? AND source_id = ? + """, + (self._chunk_payload_without_embedding(chunk), chunk.id, source_id), + ) + self._upsert_chunk_index_state( + source_id, + revision, + chunks, + storage_mode="seekdb", + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + + def _replace_sqlite_chunks( + self, + source_id: str, + chunks: list[KnowledgeChunk], + *, + persist_embedding: bool, + vector_state: str | None = None, + ) -> None: + self._conn.execute("DELETE FROM chunks WHERE source_id = ?", (source_id,)) + self._conn.executemany( + """ + INSERT INTO chunks(id, source_id, content, chunk_index, embedding, vector_state, payload) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + self._sqlite_chunk_row( + chunk, + persist_embedding=persist_embedding, + vector_state=vector_state, + ) + for chunk in chunks + ], + ) + + def _upsert_chunk_index_state( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + *, + storage_mode: str, + ) -> None: + dimension = len(chunks[0].embedding or []) if chunks and chunks[0].embedding else None + self._conn.execute( + """ + INSERT INTO chunk_index_state( + source_id, revision, chunk_count, embedding_dimension, storage_mode, updated_at + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + revision=excluded.revision, + chunk_count=excluded.chunk_count, + embedding_dimension=excluded.embedding_dimension, + storage_mode=excluded.storage_mode, + updated_at=excluded.updated_at + """, + ( + source_id, + revision, + len(chunks), + dimension, + storage_mode, + datetime.now(timezone.utc).isoformat(), + ), + ) + + def _chunk_index_states(self, source_ids: list[str]) -> dict[str, sqlite3.Row]: + if not source_ids: + return {} + placeholders = ",".join("?" for _ in source_ids) + rows = self._conn.execute( + f""" + SELECT source_id, chunk_count, embedding_dimension, storage_mode + FROM chunk_index_state + WHERE source_id IN ({placeholders}) + """, + source_ids, + ).fetchall() + return {row["source_id"]: row for row in rows} + + def _get_repository_meta(self, key: str) -> str | None: + row = self._conn.execute("SELECT value FROM repository_meta WHERE key = ?", (key,)).fetchone() + return row["value"] if row else None + + def _set_repository_meta(self, key: str, value: str) -> None: + with self._conn: + self._conn.execute( + """ + INSERT INTO repository_meta(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value + """, + (key, value), + ) def _get_sqlite_chunks_sync(self, source_id: str) -> list[KnowledgeChunk]: rows = self._conn.execute( @@ -573,15 +930,22 @@ def signature(chunk: KnowledgeChunk) -> dict[str, Any]: native_signatures = {chunk.id: signature(chunk) for chunk in native_chunks} return sqlite_signatures == native_signatures - def _sqlite_chunk_row(self, chunk: KnowledgeChunk) -> tuple[Any, ...]: - persist_embedding = self.allow_sqlite_vector_fallback + def _sqlite_chunk_row( + self, + chunk: KnowledgeChunk, + *, + persist_embedding: bool | None = None, + vector_state: str | None = None, + ) -> tuple[Any, ...]: + if persist_embedding is None: + persist_embedding = self.allow_sqlite_vector_fallback return ( chunk.id, chunk.source_id, chunk.content, chunk.chunk_index, json.dumps(chunk.embedding) if persist_embedding and chunk.embedding is not None else None, - "sqlite_fallback" if persist_embedding else "seekdb", + vector_state or ("sqlite_fallback" if persist_embedding else "seekdb"), _dump_model(chunk) if persist_embedding else self._chunk_payload_without_embedding(chunk), ) diff --git a/backend/main.py b/backend/main.py index 353da55..a944017 100644 --- a/backend/main.py +++ b/backend/main.py @@ -31,6 +31,14 @@ async def lifespan(app: FastAPI): os.makedirs(settings.upload_dir, exist_ok=True) os.makedirs(settings.output_dir, exist_ok=True) os.makedirs(settings.chroma_persist_dir, exist_ok=True) + + if settings.vector_store_type == "seekdb": + from .dependencies import DependencyContainer + + repository = DependencyContainer.get_knowledge_repository(settings=settings) + initialize_storage = getattr(repository, "initialize_storage", None) + if callable(initialize_storage): + await initialize_storage() yield diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index c19fe4d..b24411d 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -119,12 +119,14 @@ def __init__(self) -> None: self.searches = [] self.text_searches = [] self.chunks_by_source = {} + self.get_source_calls = [] def upsert_source_chunks(self, source_id, chunks): self.upserts.append((source_id, chunks)) self.chunks_by_source[source_id] = list(chunks) def get_source_chunks(self, source_id): + self.get_source_calls.append(source_id) return list(self.chunks_by_source.get(source_id, [])) def delete_source_chunks(self, source_id, chunk_ids): @@ -291,6 +293,204 @@ async def test_repository_uses_seekdb_native_index_for_chunk_save_and_search(tmp assert "Skipping optional pyseekdb chunk mirror" not in caplog.text +@pytest.mark.asyncio +async def test_repository_search_automatically_migrates_legacy_sqlite_vectors(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-auto-migrate", kind=SourceKind.TEXT, title="Auto migrate") + chunk = KnowledgeChunk( + id="chunk-auto-migrate", + source_id=source.id, + content="legacy vector should migrate before search", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + legacy_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await legacy_repo.save_source(source) + await legacy_repo.save_chunks(source.id, [chunk]) + await legacy_repo.close() + + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + + results = await repo.search_chunks( + "legacy vector", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.upserts == [(source.id, [chunk])] + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert results[0]["backend"] == "seekdb" + row = repo._conn.execute( + "SELECT embedding, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert row["embedding"] is None + assert row["vector_state"] == "seekdb" + + +@pytest.mark.asyncio +async def test_pending_seekdb_replace_is_recovered_after_process_interruption( + tmp_path: Path, + monkeypatch, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-crash-recovery", kind=SourceKind.TEXT, title="Crash recovery") + chunk = KnowledgeChunk( + id="chunk-crash-recovery", + source_id=source.id, + content="durable pending replacement", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + + def interrupt_after_native_write(*_args, **_kwargs): + raise SystemExit("simulated process interruption") + + monkeypatch.setattr(repo, "_finalize_chunk_replace", interrupt_after_native_write) + with pytest.raises(SystemExit, match="process interruption"): + await repo.save_chunks(source.id, [chunk]) + + pending = repo._conn.execute( + "SELECT operation FROM chunk_sync_operations WHERE source_id = ?", + (source.id,), + ).fetchone() + assert pending["operation"] == "replace" + assert repo._get_sqlite_chunks_sync(source.id) == [] + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert [item.id for item in await recovered.get_chunks(source.id)] == [chunk.id] + assert [item.id for item in recovered._get_sqlite_chunks_sync(source.id)] == [chunk.id] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_pending_seekdb_delete_is_recovered_after_process_interruption( + tmp_path: Path, + monkeypatch, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-delete-crash", kind=SourceKind.TEXT, title="Delete crash") + chunk = KnowledgeChunk( + id="chunk-delete-crash", + source_id=source.id, + content="delete must finish after restart", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + + def interrupt_after_native_delete(*_args, **_kwargs): + raise SystemExit("simulated delete interruption") + + monkeypatch.setattr(repo, "_finalize_source_delete", interrupt_after_native_delete) + with pytest.raises(SystemExit, match="delete interruption"): + await repo.delete_source(source.id) + assert await repo.get_source(source.id) is not None + assert repo._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 1 + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert await recovered.get_source(source.id) is None + assert native_index.get_source_chunks(source.id) == [] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_native_search_uses_source_state_without_loading_all_chunks(tmp_path: Path): + source = KnowledgeSource(id="src-lightweight-search", kind=SourceKind.TEXT, title="Lightweight") + chunk = KnowledgeChunk( + id="chunk-lightweight-search", + source_id=source.id, + content="search should not deserialize the whole collection", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "knowledge.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + native_index.get_source_calls.clear() + + await repo.search_chunks( + "deserialize", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + await repo.search_chunks("deserialize", source_ids=[source.id], top_k=1) + + assert native_index.get_source_calls == [] + assert native_index.searches == [([0.1, 0.2, 0.3], [source.id], 1)] + assert native_index.text_searches == [("deserialize", [source.id], 1)] + + +@pytest.mark.asyncio +async def test_strict_mode_scrubs_vectors_after_fallback_backfill(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-mode-switch", kind=SourceKind.TEXT, title="Mode switch") + chunk = KnowledgeChunk( + id="chunk-mode-switch", + source_id=source.id, + content="fallback vectors must be scrubbed in strict mode", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = RecordingNativeIndex() + fallback_repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + ) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks(source.id, [chunk]) + await fallback_repo.backfill_native_chunks() + await fallback_repo.close() + + strict_repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await strict_repo.initialize_storage() + + row = strict_repo._conn.execute( + "SELECT embedding, vector_state FROM chunks WHERE source_id = ?", + (source.id,), + ).fetchone() + assert row["embedding"] is None + assert row["vector_state"] == "seekdb" + + @pytest.mark.asyncio async def test_strict_native_mode_keeps_vectors_out_of_sqlite(tmp_path: Path): native_index = RecordingNativeIndex() @@ -722,7 +922,7 @@ async def test_strict_native_rejects_mixed_dimension_embeddings_before_sqlite_mu ) assert native_index.upserts == [] - assert await repo.get_chunks(source.id) == [] + assert [chunk.id for chunk in await repo.get_chunks(source.id)] == ["chunk-old"] assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == ["chunk-old"] From a6e1b8072fc95cb0111110ac1789cfc3dd973e8e Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 14:11:23 +0800 Subject: [PATCH 20/22] fix(storage): harden SeekDB recovery and lifecycle --- backend/dependencies.py | 10 + .../repositories/seekdb_repository.py | 172 ++++++++++++++---- .../vector_stores/seekdb_chunk_index.py | 9 + backend/main.py | 11 +- tests/test_runtime_config_api.py | 46 +++++ tests/test_seekdb_chunk_index.py | 33 +++- tests/test_source_knowledge_base.py | 122 ++++++++++++- 7 files changed, 356 insertions(+), 47 deletions(-) diff --git a/backend/dependencies.py b/backend/dependencies.py index 7c039a8..4430a46 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -25,9 +25,13 @@ class DependencyContainer: def reset_runtime_caches(cls) -> None: """Drop cached services that depend on model settings.""" + repository = cls._knowledge_repository cls._vector_store = None cls._knowledge_repository = None cls._slide_deck_service = None + close_repository = getattr(repository, "close_sync", None) + if callable(close_repository): + close_repository() @classmethod def get_vector_store( @@ -87,6 +91,12 @@ def get_knowledge_repository( force_new: bool = False, ) -> KnowledgeRepositoryInterface: if cls._knowledge_repository is None or force_new: + if force_new and cls._knowledge_repository is not None: + cls._vector_store = None + cls._slide_deck_service = None + close_repository = getattr(cls._knowledge_repository, "close_sync", None) + if callable(close_repository): + close_repository() settings = settings or get_settings() from .infrastructure.repositories.seekdb_repository import SeekDBRepository cls._knowledge_repository = SeekDBRepository( diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index 6bc7bb2..da2a427 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -106,6 +106,8 @@ def __init__( self._conn.row_factory = sqlite3.Row self._init_schema() self._storage_initialized = False + self._migration_has_unresolved_sources = False + self._closed = False def _try_native_chunk_index(self, path: Path) -> Any | None: try: @@ -215,7 +217,18 @@ def _init_schema(self) -> None: self._conn.commit() async def close(self) -> None: - self._conn.close() + self.close_sync() + + def close_sync(self) -> None: + if self._closed: + return + try: + close_native = getattr(self.native_chunk_index, "close", None) + if callable(close_native): + close_native() + finally: + self._conn.close() + self._closed = True def storage_status(self) -> dict[str, Any]: if self.native_chunk_index is not None: @@ -284,12 +297,7 @@ async def delete_source(self, source_id: str) -> bool: self.native_chunk_index.upsert_source_chunks(source_id, []) self._finalize_source_delete(source_id, revision) except Exception: - if previous_native_chunks is not None: - try: - self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) - except Exception: - logger.exception("Failed to restore SeekDB chunks after a source delete failure") - self._discard_sync_operation(source_id, revision) + self._compensate_native_failure(source_id, revision, previous_native_chunks) raise return True @@ -352,12 +360,7 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non persist_embedding=persist_embedding, ) except Exception: - if previous_native_chunks is not None: - try: - self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) - except Exception: - logger.exception("Failed to restore SeekDB chunks after a metadata transaction failure") - self._discard_sync_operation(source_id, revision) + self._compensate_native_failure(source_id, revision, previous_native_chunks) raise async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: @@ -384,8 +387,8 @@ async def backfill_native_chunks(self) -> int: self._recover_pending_sync_operations() written = self._backfill_native_chunks_sync() - if not self.allow_sqlite_vector_fallback: - self._set_repository_meta("native_chunk_migration_v1", "complete") + if not self.allow_sqlite_vector_fallback and not self._migration_has_unresolved_sources: + self._set_repository_meta("native_chunk_migration_v2", "complete") self._storage_initialized = True return written @@ -397,10 +400,11 @@ async def initialize_storage(self) -> int: written = 0 if ( not self.allow_sqlite_vector_fallback - and self._get_repository_meta("native_chunk_migration_v1") != "complete" + and self._get_repository_meta("native_chunk_migration_v2") != "complete" ): written = self._backfill_native_chunks_sync() - self._set_repository_meta("native_chunk_migration_v1", "complete") + if not self._migration_has_unresolved_sources: + self._set_repository_meta("native_chunk_migration_v2", "complete") self._storage_initialized = True return written @@ -410,6 +414,7 @@ async def _ensure_storage_initialized(self) -> None: def _backfill_native_chunks_sync(self) -> int: written = 0 + self._migration_has_unresolved_sources = False source_rows = self._conn.execute( "SELECT payload FROM sources WHERE status != ? ORDER BY created_at DESC", (SourceStatus.DELETED.value,), @@ -430,13 +435,18 @@ def _backfill_native_chunks_sync(self) -> int: storage_mode="seekdb", ) continue - reconciled_states = {"seekdb", "sqlite_fallback_reconciled"} + reconciled_states = {"seekdb"} + if self.allow_sqlite_vector_fallback: + reconciled_states.add("sqlite_fallback_reconciled") state_row = self._conn.execute( "SELECT source_id FROM chunk_index_state WHERE source_id = ?", (source.id,), ).fetchone() if state_row and all(row["vector_state"] in reconciled_states for row in sqlite_rows): continue + if state_row and all(row["vector_state"] == "legacy_needs_embedding" for row in sqlite_rows): + self._migration_has_unresolved_sources = True + continue chunks_with_embeddings = self._native_eligible_chunks(chunks) get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) previous_native_chunks = get_native_chunks(source.id) if callable(get_native_chunks) else None @@ -458,10 +468,20 @@ def _backfill_native_chunks_sync(self) -> int: ) continue logger.warning( - "Clearing stale native chunks for fallback source %s because embeddings are missing or inconsistent", + "Legacy source %s has missing or inconsistent embeddings", source.id, ) revision = uuid4().hex + if not self.allow_sqlite_vector_fallback: + self._migration_has_unresolved_sources = True + self._record_sync_operation(source.id, revision, "legacy_clear", None) + try: + self.native_chunk_index.upsert_source_chunks(source.id, []) + self._finalize_legacy_source(source.id, revision, chunks) + except Exception: + self._compensate_native_failure(source.id, revision, previous_native_chunks) + raise + continue self._record_sync_operation( source.id, revision, @@ -479,12 +499,7 @@ def _backfill_native_chunks_sync(self) -> int: vector_state="sqlite_fallback_reconciled", ) except Exception: - if previous_native_chunks is not None: - try: - self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) - except Exception: - logger.exception("Failed to restore SeekDB chunks after a backfill clear failure") - self._discard_sync_operation(source.id, revision) + self._compensate_native_failure(source.id, revision, previous_native_chunks) raise continue revision = uuid4().hex @@ -512,12 +527,7 @@ def _backfill_native_chunks_sync(self) -> int: persist_embedding=persist_embedding, ) except Exception: - if previous_native_chunks is not None: - try: - self.native_chunk_index.upsert_source_chunks(source.id, previous_native_chunks) - except Exception: - logger.exception("Failed to restore SeekDB chunks after a backfill failure") - self._discard_sync_operation(source.id, revision) + self._compensate_native_failure(source.id, revision, previous_native_chunks) raise written += len(chunks_with_embeddings) return written @@ -567,6 +577,19 @@ async def search_chunks( None, ) ) + else: + legacy_source_ids = self._legacy_source_ids(requested_source_ids) + if legacy_source_ids: + legacy_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(legacy_source_ids), + top_k, + None, + None, + ) + for result in legacy_results: + result["backend"] = "sqlite_legacy_text" + results.extend(legacy_results) results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] return await self._maybe_rerank(query_text, results, top_k, rerank_provider) if self.native_chunk_index is not None and query_embedding is not None: @@ -610,6 +633,19 @@ async def search_chunks( None, ) results.extend(fallback_results) + else: + legacy_source_ids = self._legacy_source_ids(requested_source_ids) + if legacy_source_ids: + legacy_results = await self._search_sqlite_chunk_rows( + query, + self._chunk_rows(legacy_source_ids), + top_k, + None, + None, + ) + for result in legacy_results: + result["backend"] = "sqlite_legacy_text" + results.extend(legacy_results) results = sorted(results, key=lambda item: item["score"], reverse=True)[:top_k] return await self._maybe_rerank(query, results, top_k, rerank_provider) @@ -664,6 +700,16 @@ def _native_source_ids_with_chunks(self, source_ids: list[str]) -> list[str]: and states[source_id]["chunk_count"] > 0 ] + def _legacy_source_ids(self, source_ids: list[str]) -> list[str]: + states = self._chunk_index_states(source_ids) + return [ + source_id + for source_id in source_ids + if source_id in states + and states[source_id]["storage_mode"] == "sqlite_legacy" + and states[source_id]["chunk_count"] > 0 + ] + def _record_sync_operation( self, source_id: str, @@ -695,6 +741,39 @@ def _discard_sync_operation(self, source_id: str, revision: str) -> None: except Exception: logger.exception("Failed to discard completed chunk sync operation for %s", source_id) + def _compensate_native_failure( + self, + source_id: str, + revision: str, + previous_native_chunks: list[KnowledgeChunk] | None, + ) -> None: + if previous_native_chunks is None: + logger.error( + "Cannot restore SeekDB source %s because its previous state is unavailable; " + "the original sync operation remains pending", + source_id, + ) + return + try: + self._record_sync_operation( + source_id, + revision, + "native_restore", + json.dumps([chunk.model_dump_json() for chunk in previous_native_chunks]), + ) + except Exception: + logger.exception( + "Failed to persist native restore operation for %s; original operation remains pending", + source_id, + ) + return + try: + self.native_chunk_index.upsert_source_chunks(source_id, previous_native_chunks) + except Exception: + logger.exception("Failed to restore SeekDB chunks for %s; restore remains pending", source_id) + return + self._discard_sync_operation(source_id, revision) + @staticmethod def _replacement_operation_payload( chunks: list[KnowledgeChunk], @@ -749,6 +828,15 @@ def _recover_pending_sync_operations(self) -> None: elif row["operation"] == "delete": self.native_chunk_index.upsert_source_chunks(source_id, []) self._finalize_source_delete(source_id, revision) + elif row["operation"] == "native_restore": + payload = json.loads(row["payload"] or "[]") + chunks = [KnowledgeChunk.model_validate_json(item) for item in payload] + self.native_chunk_index.upsert_source_chunks(source_id, chunks) + self._discard_sync_operation(source_id, revision) + elif row["operation"] == "legacy_clear": + chunks = self._get_sqlite_chunks_sync(source_id) + self.native_chunk_index.upsert_source_chunks(source_id, []) + self._finalize_legacy_source(source_id, revision, chunks) else: raise ValueError(f"Unknown chunk sync operation: {row['operation']}") @@ -821,6 +909,28 @@ def _finalize_chunk_migration( (source_id, revision), ) + def _finalize_legacy_source( + self, + source_id: str, + revision: str, + chunks: list[KnowledgeChunk], + ) -> None: + with self._conn: + self._conn.execute( + "UPDATE chunks SET vector_state = 'legacy_needs_embedding' WHERE source_id = ?", + (source_id,), + ) + self._upsert_chunk_index_state( + source_id, + revision, + chunks, + storage_mode="sqlite_legacy", + ) + self._conn.execute( + "DELETE FROM chunk_sync_operations WHERE source_id = ? AND revision = ?", + (source_id, revision), + ) + def _replace_sqlite_chunks( self, source_id: str, diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index fdab067..cad6c36 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -23,12 +23,21 @@ class SeekDBChunkIndex: def __init__(self, path: str | Path) -> None: self.path = Path(path) self.path.mkdir(parents=True, exist_ok=True) + self._closed = False try: self.pyseekdb = importlib.import_module("pyseekdb") self.client = self.pyseekdb.Client(path=str(self.path)) except Exception as exc: raise SeekDBUnavailableError("Native SeekDB chunk index is unavailable") from exc + def close(self) -> None: + if self._closed: + return + client_exit = getattr(self.client, "__exit__", None) + if callable(client_exit): + client_exit(None, None, None) + self._closed = True + def collection_name(self, source_id: str) -> str: digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] return f"chunks_{digest}" diff --git a/backend/main.py b/backend/main.py index a944017..9848caf 100644 --- a/backend/main.py +++ b/backend/main.py @@ -32,15 +32,18 @@ async def lifespan(app: FastAPI): os.makedirs(settings.output_dir, exist_ok=True) os.makedirs(settings.chroma_persist_dir, exist_ok=True) - if settings.vector_store_type == "seekdb": - from .dependencies import DependencyContainer + from .dependencies import DependencyContainer + if settings.vector_store_type == "seekdb": repository = DependencyContainer.get_knowledge_repository(settings=settings) initialize_storage = getattr(repository, "initialize_storage", None) if callable(initialize_storage): await initialize_storage() - - yield + + try: + yield + finally: + DependencyContainer.reset_runtime_caches() # 创建应用 diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index 3120e9f..a36a16e 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -87,6 +87,52 @@ def test_runtime_cache_reset_recreates_knowledge_repository_with_new_storage_set DependencyContainer.reset_runtime_caches() +def test_runtime_cache_reset_closes_cached_repository(): + class ClosableRepository: + def __init__(self) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + repository = ClosableRepository() + DependencyContainer._knowledge_repository = repository + + DependencyContainer.reset_runtime_caches() + + assert repository.close_calls == 1 + assert DependencyContainer._knowledge_repository is None + + +def test_force_new_repository_closes_old_repository_and_invalidates_dependents(monkeypatch): + class ClosableRepository: + def __init__(self, *_args, **_kwargs) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + old_repository = ClosableRepository() + DependencyContainer._knowledge_repository = old_repository + DependencyContainer._vector_store = object() + DependencyContainer._slide_deck_service = object() + monkeypatch.setattr( + "backend.infrastructure.repositories.seekdb_repository.SeekDBRepository", + ClosableRepository, + ) + + new_repository = DependencyContainer.get_knowledge_repository( + settings=Settings(seekdb_allow_sqlite_fallback=True), + force_new=True, + ) + + assert new_repository is not old_repository + assert old_repository.close_calls == 1 + assert DependencyContainer._vector_store is None + assert DependencyContainer._slide_deck_service is None + DependencyContainer.reset_runtime_caches() + + def test_runtime_config_exposes_actual_vector_backend(monkeypatch, sample_config_file): monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) monkeypatch.setattr( diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 6419cf1..4254f93 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -124,8 +124,12 @@ def __init__(self, path: str) -> None: self.path = path self.collections = {} self.created = [] + self.exit_calls = 0 FakeClient.instances.append(self) + def __exit__(self, exc_type, exc_value, traceback): + self.exit_calls += 1 + def get_or_create_collection(self, name, configuration=None, embedding_function=None): if embedding_function is not None: raise AssertionError("explicit LiteLLM embeddings require embedding_function=None") @@ -184,6 +188,16 @@ def test_upsert_creates_dimensioned_source_collection(tmp_path: Path): assert collection.refresh_calls == 1 +def test_close_releases_native_client_once(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + client = FakeClient.instances[0] + + index.close() + index.close() + + assert client.exit_calls == 1 + + def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) @@ -561,13 +575,12 @@ def run_real_pyseekdb_subprocess(script: str) -> None: env["PYTHONPATH"] = os.pathsep.join( value for value in [str(repo_root), env.get("PYTHONPATH", "")] if value ) - subprocess_script = textwrap.dedent( + subprocess_prelude = textwrap.dedent( """ try: import pyseekdb # noqa: F401 except Exception: - print("PYSEEKDB_UNAVAILABLE") - raise SystemExit(0) + raise SystemExit(77) from backend.domain.source import KnowledgeChunk from backend.infrastructure.vector_stores.seekdb_chunk_index import SeekDBChunkIndex @@ -583,22 +596,26 @@ def chunk(chunk_id: str, source_id: str, embedding: list[float], metadata: dict metadata=metadata if metadata is not None else {"source_id": source_id}, ) """ - ) + "\n" + textwrap.dedent(script) + ) + subprocess_script = ( + subprocess_prelude + + "\nindex = None\ntry:\n" + + textwrap.indent(textwrap.dedent(script), " ") + + "\nfinally:\n if index is not None:\n index.close()\n" + ) try: completed = subprocess.run( [sys.executable, "-c", subprocess_script], cwd=repo_root, env=env, - capture_output=True, - text=True, timeout=60, check=False, ) except subprocess.TimeoutExpired: pytest.skip("real pyseekdb subprocess timed out after 60 seconds") - if "PYSEEKDB_UNAVAILABLE" in completed.stdout: + if completed.returncode == 77: pytest.skip("pyseekdb is unavailable in subprocess") - assert completed.returncode == 0, completed.stdout + completed.stderr + assert completed.returncode == 0 def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path): diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index b24411d..b7e4bcc 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -224,6 +224,22 @@ def upsert_source_chunks(self, source_id, chunks): super().upsert_source_chunks(source_id, chunks) +class PartialWriteAndRestoreFailingNativeIndex(RecordingNativeIndex): + def __init__(self) -> None: + super().__init__() + self.failures_remaining = 0 + + def upsert_source_chunks(self, source_id, chunks): + if self.failures_remaining == 2: + self.failures_remaining -= 1 + self.chunks_by_source[source_id] = list(chunks) + raise RuntimeError("partial native write") + if self.failures_remaining == 1: + self.failures_remaining -= 1 + raise RuntimeError("native restore failed") + super().upsert_source_chunks(source_id, chunks) + + class DimensionCheckingNativeIndex(RecordingNativeIndex): def search(self, query_embedding, source_ids, top_k): bad_source_ids = [ @@ -377,6 +393,56 @@ def interrupt_after_native_write(*_args, **_kwargs): assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 +@pytest.mark.asyncio +async def test_failed_native_compensation_keeps_a_durable_restore_operation(tmp_path: Path): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-restore-retry", kind=SourceKind.TEXT, title="Restore retry") + old_chunk = KnowledgeChunk( + id="chunk-old-restore-retry", + source_id=source.id, + content="old canonical content", + chunk_index=0, + embedding=[0.4, 0.5, 0.6], + metadata={"source_id": source.id}, + ) + new_chunk = KnowledgeChunk( + id="chunk-new-restore-retry", + source_id=source.id, + content="partially written content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + native_index = PartialWriteAndRestoreFailingNativeIndex() + repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) + await repo.save_source(source) + await repo.save_chunks(source.id, [old_chunk]) + native_index.failures_remaining = 2 + + with pytest.raises(RuntimeError, match="partial native write"): + await repo.save_chunks(source.id, [new_chunk]) + + pending = repo._conn.execute( + "SELECT operation FROM chunk_sync_operations WHERE source_id = ?", + (source.id,), + ).fetchone() + assert pending["operation"] == "native_restore" + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [new_chunk.id] + assert [chunk.id for chunk in repo._get_sqlite_chunks_sync(source.id)] == [old_chunk.id] + await repo.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + assert [chunk.id for chunk in native_index.get_source_chunks(source.id)] == [old_chunk.id] + assert [chunk.id for chunk in recovered._get_sqlite_chunks_sync(source.id)] == [old_chunk.id] + assert recovered._conn.execute("SELECT COUNT(*) FROM chunk_sync_operations").fetchone()[0] == 0 + + @pytest.mark.asyncio async def test_pending_seekdb_delete_is_recovered_after_process_interruption( tmp_path: Path, @@ -1118,8 +1184,54 @@ async def test_repository_backfill_clears_stale_native_for_fallback_rows_without "SELECT vector_state FROM chunks WHERE source_id = ?", (source.id,), ).fetchone()["vector_state"] - assert state == "sqlite_fallback_reconciled" - assert results == [] + assert state == "legacy_needs_embedding" + assert results[0]["chunk"].id == "chunk-plain" + assert results[0]["backend"] == "sqlite_legacy_text" + + +@pytest.mark.asyncio +async def test_strict_backfill_keeps_unembedded_legacy_sources_lexically_searchable_and_retryable( + tmp_path: Path, +): + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-legacy-lexical", kind=SourceKind.TEXT, title="Legacy lexical") + fallback_repo = SeekDBRepository(db_path, native_chunk_index=None, allow_sqlite_vector_fallback=True) + await fallback_repo.save_source(source) + await fallback_repo.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-legacy-lexical", + source_id=source.id, + content="Asterion protocol preserves lexical retrieval", + chunk_index=0, + metadata={"source_id": source.id}, + ) + ], + ) + await fallback_repo.close() + + repo = SeekDBRepository( + db_path, + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + await repo.initialize_storage() + results = await repo.search_chunks( + "Asterion protocol", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert results[0]["chunk"].id == "chunk-legacy-lexical" + assert results[0]["backend"] == "sqlite_legacy_text" + state = repo._conn.execute( + "SELECT storage_mode FROM chunk_index_state WHERE source_id = ?", + (source.id,), + ).fetchone() + assert state["storage_mode"] == "sqlite_legacy" + assert repo._get_repository_meta("native_chunk_migration_v2") is None @pytest.mark.asyncio @@ -1216,7 +1328,8 @@ async def test_repository_backfill_skips_partially_embedded_sources_and_native_s assert written == 0 assert native_index.upserts == [(source.id, [])] assert native_index.searches == [] - assert results == [] + assert results[0]["chunk"].source_id == source.id + assert results[0]["backend"] == "sqlite_legacy_text" @pytest.mark.asyncio @@ -1298,7 +1411,8 @@ async def test_repository_backfill_treats_empty_embedding_as_native_ineligible(t assert written == 0 assert native_index.upserts == [(source.id, [])] assert native_index.searches == [] - assert results == [] + assert results[0]["chunk"].id == "chunk-empty-embedding" + assert results[0]["backend"] == "sqlite_legacy_text" @pytest.mark.asyncio From 8a47d8f4ecf075e164c2e14ce940e5854fdd9d89 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 15:02:50 +0800 Subject: [PATCH 21/22] fix(storage): harden SeekDB retrieval lifecycle --- .github/workflows/ci.yml | 3 + README.md | 10 +- README_zh.md | 10 +- backend/api/routes/config.py | 42 +++- backend/api/routes/documents.py | 4 + backend/api/routes/sources.py | 11 + backend/config.py | 14 ++ backend/core/services/source_service.py | 4 +- backend/dependencies.py | 13 +- .../repositories/seekdb_repository.py | 170 ++++++++++++- .../vector_stores/seekdb_chunk_index.py | 11 + .../vector_stores/seekdb_vector_store.py | 8 + backend/main.py | 20 +- tests/test_runtime_config_api.py | 116 ++++++++- tests/test_seekdb_chunk_index.py | 236 ++---------------- tests/test_slide_deck_domain_repository.py | 3 + tests/test_source_knowledge_base.py | 221 +++++++++++++++- 17 files changed, 639 insertions(+), 257 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c06b14..69f30cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,9 @@ jobs: PYTHONPATH: . run: python -m pytest -q + - name: Verify native SeekDB retrieval + run: timeout 120s python scripts/verify_seekdb_native.py + - name: Compile backend modules run: python -m compileall -q backend diff --git a/README.md b/README.md index e84e925..a9aa56e 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,8 @@ Key backend choices: - Python 3.10+ - Node.js 18+ - A text model endpoint configured in `config.yaml` -- Optional: embedding, rerank, and speech model endpoints +- An embedding endpoint when using the default strict SeekDB mode +- Optional: rerank and speech model endpoints ### Backend @@ -111,16 +112,19 @@ cp config_example.yaml config.yaml Edit `config.yaml` and fill in the profiles you want to use: - `api.models.text_model`: chat, RAG answers, Studio artifacts, podcast scripts -- `api.models.embedding_model`: optional vector retrieval +- `api.models.embedding_model`: required for ingestion and vector retrieval when `seekdb_allow_sqlite_fallback` is `false` - `api.models.rerank_model`: optional rerank - `api.models.audio_model`: optional speech/audio generation - `api.models.image_model`: slide image generation - `api.models.edit_model`: single-slide image editing -- `storage.seekdb_path`: local knowledge database +- `storage.seekdb_path`: local repository base path; SQLite keeps business metadata and native SeekDB keeps chunk vectors and retrieval indexes beside it +- `storage.seekdb_allow_sqlite_fallback`: compatibility-only SQLite retrieval fallback; keep `false` to make SeekDB authoritative - `documents.chunking`: Chonkie/simple chunking settings Keep `config.yaml` local. It is ignored by git and should not contain committed keys. +Once sources are indexed, the runtime settings API rejects changes to the embedding model, adapter, or base URL because existing vectors would no longer be compatible. Delete and re-import the sources before changing that embedding identity. API-key rotation does not require re-indexing. + Start the backend: ```bash diff --git a/README_zh.md b/README_zh.md index c23c448..afc1926 100644 --- a/README_zh.md +++ b/README_zh.md @@ -99,7 +99,8 @@ backend/ - Python 3.10+ - Node.js 18+ - 一个可用的文本模型接口 -- 可选：embedding、rerank、语音模型接口 +- 默认严格 SeekDB 模式下需要可用的 embedding 接口 +- 可选：rerank、语音模型接口 ### 后端 @@ -111,16 +112,19 @@ cp config_example.yaml config.yaml 编辑 `config.yaml`，填写模型与存储配置： - `api.models.text_model`：聊天、RAG 回答、Studio artifact、播客脚本 -- `api.models.embedding_model`：可选向量召回 +- `api.models.embedding_model`：当 `seekdb_allow_sqlite_fallback` 为 `false` 时，文档摄取与向量召回必填 - `api.models.rerank_model`：可选 rerank - `api.models.audio_model`：可选语音/音频生成 - `api.models.image_model`：Slide Deck 图片生成 - `api.models.edit_model`：单页图片编辑 -- `storage.seekdb_path`：本地知识库路径 +- `storage.seekdb_path`：本地 repository 基础路径；SQLite 保存业务元数据，原生 SeekDB 在相邻目录保存 chunk 向量与检索索引 +- `storage.seekdb_allow_sqlite_fallback`：仅用于兼容的 SQLite 检索回退；保持 `false` 时 SeekDB 是检索权威存储 - `documents.chunking`：Chonkie/simple chunk 参数 `config.yaml` 只用于本地运行，已被 git 忽略，不要提交真实 key。 +已有 source 完成索引后，运行时配置接口会拒绝修改 embedding 模型、adapter 或 base URL，避免用不兼容的新查询向量检索旧索引。需要先删除并重新导入 source；仅轮换 API key 不需要重建索引。 + 启动后端： ```bash diff --git a/backend/api/routes/config.py b/backend/api/routes/config.py index eb5eaf6..a9761f2 100644 --- a/backend/api/routes/config.py +++ b/backend/api/routes/config.py @@ -2,9 +2,15 @@ from __future__ import annotations -from fastapi import APIRouter +from fastapi import APIRouter, HTTPException -from ...config import ModelProfile, ModelProfiles, get_settings, update_runtime_model_profiles +from ...config import ( + ModelProfile, + ModelProfiles, + get_settings, + model_profile_identity, + update_runtime_model_profiles, +) from ...dependencies import DependencyContainer from ..schemas.config import PublicModelProfile, RuntimeConfigResponse, RuntimeConfigUpdate @@ -48,6 +54,13 @@ async def _response(message: str = "") -> RuntimeConfigResponse: "seekdb_allow_sqlite_fallback": settings.seekdb_allow_sqlite_fallback, "actual_vector_backend": storage_status.get("vector_backend", stats.get("backend", "unknown")), "native_available": storage_status.get("native_available", False), + "embedding_required": ( + settings.vector_store_type == "seekdb" + and not settings.seekdb_allow_sqlite_fallback + ), + "embedding_configured": bool( + models.embedding_model.model and models.embedding_model.api_key + ), }, message=message, ) @@ -60,10 +73,33 @@ async def get_runtime_config() -> RuntimeConfigResponse: @router.post("", response_model=RuntimeConfigResponse) async def update_runtime_config(request: RuntimeConfigUpdate) -> RuntimeConfigResponse: + requested_embedding = request.models.get("embedding_model") + if requested_embedding is not None: + settings = get_settings() + current_profile = settings.api.models.embedding_model + requested_values = requested_embedding.model_dump(exclude_none=True) + if requested_values.get("api_key") == "": + requested_values.pop("api_key") + next_profile = current_profile.model_copy(update=requested_values) + if model_profile_identity(next_profile) != model_profile_identity(current_profile): + vector_store = DependencyContainer.get_vector_store(settings=settings) + stats = await vector_store.get_stats() + indexed_vector_chunks = stats.get( + "indexed_vector_chunks", + stats.get("total_chunks", 0), + ) + if indexed_vector_chunks > 0: + raise HTTPException( + status_code=409, + detail=( + "The embedding model, adapter, or base URL cannot change while indexed " + "sources exist. Delete and re-index the sources first." + ), + ) profile_data = { name: profile.model_dump(exclude_none=True) for name, profile in request.models.items() } update_runtime_model_profiles(profile_data) - DependencyContainer.reset_runtime_caches() + DependencyContainer.reset_runtime_caches(preserve_repository=True) return await _response("Runtime model configuration updated.") diff --git a/backend/api/routes/documents.py b/backend/api/routes/documents.py index 1277ded..084bbbb 100644 --- a/backend/api/routes/documents.py +++ b/backend/api/routes/documents.py @@ -61,6 +61,10 @@ async def upload_document( char_count=source.char_count ) + except HTTPException: + if os.path.exists(file_path): + os.remove(file_path) + raise except Exception as e: # 清理文件 if os.path.exists(file_path): diff --git a/backend/api/routes/sources.py b/backend/api/routes/sources.py index 3c5e2df..c313c07 100644 --- a/backend/api/routes/sources.py +++ b/backend/api/routes/sources.py @@ -45,6 +45,11 @@ def _detail(source: KnowledgeSource) -> SourceDetail: return SourceDetail(**_info(source).model_dump(), text=source.text) +def _raise_if_failed(source: KnowledgeSource) -> None: + if source.status.value == "error": + raise HTTPException(status_code=422, detail=source.error or "Source ingestion failed") + + @router.post("/text", response_model=SourceDetail) async def create_text_source( request: SourceCreateTextRequest, @@ -55,6 +60,7 @@ async def create_text_source( text=request.text, metadata=request.metadata, ) + _raise_if_failed(source) return _detail(source) @@ -76,7 +82,12 @@ async def upload_source( filename=filename, mime_type=file.content_type, ) + _raise_if_failed(source) return _detail(source) + except HTTPException: + if file_path.exists(): + file_path.unlink() + raise except Exception as exc: if file_path.exists(): file_path.unlink() diff --git a/backend/config.py b/backend/config.py index d06deeb..7f3aeea 100644 --- a/backend/config.py +++ b/backend/config.py @@ -7,6 +7,8 @@ from __future__ import annotations +import hashlib +import json import os from functools import lru_cache from pathlib import Path @@ -36,6 +38,18 @@ class ModelProfile(BaseModel): stream: bool = True +def model_profile_identity(profile: ModelProfile) -> str: + """Return a stable, secret-free identity for embedding compatibility checks.""" + + payload = { + "adapter": profile.adapter.strip(), + "base_url": profile.base_url.strip().rstrip("/"), + "model": profile.model.strip(), + } + serialized = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + class ModelProfiles(BaseModel): """Configured model roles.""" diff --git a/backend/core/services/source_service.py b/backend/core/services/source_service.py index e12cc75..0b54bec 100644 --- a/backend/core/services/source_service.py +++ b/backend/core/services/source_service.py @@ -97,10 +97,12 @@ async def _finish_source( for chunk, embedding in zip(chunks, embeddings): chunk.embedding = embedding source.chunk_count = len(chunks) - source.status = SourceStatus.READY source.updated_at = utc_now() await self.repository.save_source(source) await self.repository.save_chunks(source.id, chunks) + source.status = SourceStatus.READY + source.updated_at = utc_now() + await self.repository.save_source(source) return source async def _fail_source(self, source: KnowledgeSource, exc: Exception) -> KnowledgeSource: diff --git a/backend/dependencies.py b/backend/dependencies.py index 4430a46..df5be28 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -6,7 +6,7 @@ """ from typing import Any, Optional -from .config import ModelProfile, Settings, get_settings +from .config import ModelProfile, Settings, get_settings, model_profile_identity from .core.interfaces.vector_store import VectorStoreInterface from .core.interfaces.knowledge_repository import KnowledgeRepositoryInterface from .core.interfaces.llm_provider import LLMProviderInterface @@ -22,13 +22,15 @@ class DependencyContainer: _slide_deck_service: Optional[Any] = None @classmethod - def reset_runtime_caches(cls) -> None: + def reset_runtime_caches(cls, *, preserve_repository: bool = False) -> None: """Drop cached services that depend on model settings.""" repository = cls._knowledge_repository cls._vector_store = None - cls._knowledge_repository = None cls._slide_deck_service = None + if preserve_repository: + return + cls._knowledge_repository = None close_repository = getattr(repository, "close_sync", None) if callable(close_repository): close_repository() @@ -102,7 +104,12 @@ def get_knowledge_repository( cls._knowledge_repository = SeekDBRepository( settings.seekdb_path, allow_sqlite_vector_fallback=settings.seekdb_allow_sqlite_fallback, + embedding_profile_id=model_profile_identity(settings.api.models.embedding_model), ) + elif settings is not None: + embedding_profile_id = model_profile_identity(settings.api.models.embedding_model) + if hasattr(cls._knowledge_repository, "embedding_profile_id"): + cls._knowledge_repository.embedding_profile_id = embedding_profile_id return cls._knowledge_repository @classmethod diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index da2a427..f56f733 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -92,11 +92,13 @@ def __init__( db_path: str | Path, native_chunk_index: object = _AUTO_NATIVE_INDEX, allow_sqlite_vector_fallback: bool = False, + embedding_profile_id: str | None = None, ) -> None: self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) self.seekdb_path = self.db_path.parent / f"{self.db_path.stem}.seekdb" self.allow_sqlite_vector_fallback = allow_sqlite_vector_fallback + self.embedding_profile_id = embedding_profile_id self.native_chunk_index = ( self._try_native_chunk_index(self.seekdb_path) if native_chunk_index is _AUTO_NATIVE_INDEX @@ -143,6 +145,7 @@ def _init_schema(self) -> None: revision TEXT NOT NULL, chunk_count INTEGER NOT NULL, embedding_dimension INTEGER, + embedding_profile TEXT, storage_mode TEXT NOT NULL, updated_at TEXT NOT NULL ); @@ -214,6 +217,20 @@ def _init_schema(self) -> None: self._conn.execute( "ALTER TABLE chunks ADD COLUMN vector_state TEXT NOT NULL DEFAULT 'legacy'" ) + state_columns = { + row["name"] for row in self._conn.execute("PRAGMA table_info(chunk_index_state)").fetchall() + } + if "embedding_profile" not in state_columns: + self._conn.execute("ALTER TABLE chunk_index_state ADD COLUMN embedding_profile TEXT") + if self.embedding_profile_id: + self._conn.execute( + """ + UPDATE chunk_index_state + SET embedding_profile = ? + WHERE embedding_profile IS NULL AND storage_mode = 'seekdb' + """, + (self.embedding_profile_id,), + ) self._conn.commit() async def close(self) -> None: @@ -232,10 +249,23 @@ def close_sync(self) -> None: def storage_status(self) -> dict[str, Any]: if self.native_chunk_index is not None: - status = dict(self.native_chunk_index.status()) - status.setdefault("seekdb_path", str(self.seekdb_path)) - status.setdefault("native_available", True) - return status + try: + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() + status = dict(self.native_chunk_index.status()) + status.setdefault("seekdb_path", str(self.seekdb_path)) + status.setdefault("native_available", True) + return status + except Exception as exc: + return { + "vector_backend": ( + "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable" + ), + "seekdb_path": str(self.seekdb_path), + "native_available": False, + "error": str(exc), + } return { "vector_backend": "sqlite_fallback" if self.allow_sqlite_vector_fallback else "unavailable", "seekdb_path": str(self.seekdb_path), @@ -365,6 +395,9 @@ async def save_chunks(self, source_id: str, chunks: list[KnowledgeChunk]) -> Non async def get_chunks(self, source_id: str) -> list[KnowledgeChunk]: await self._ensure_storage_initialized() + state = self._chunk_index_states([source_id]).get(source_id) + if state is not None and state["storage_mode"] == "sqlite_legacy": + return self._get_sqlite_chunks_sync(source_id) if self.native_chunk_index is not None: get_native_chunks = getattr(self.native_chunk_index, "get_source_chunks", None) if callable(get_native_chunks): @@ -385,17 +418,29 @@ async def backfill_native_chunks(self) -> int: return 0 raise RuntimeError("native SeekDB vector index is unavailable; cannot backfill chunks") + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() self._recover_pending_sync_operations() written = self._backfill_native_chunks_sync() if not self.allow_sqlite_vector_fallback and not self._migration_has_unresolved_sources: self._set_repository_meta("native_chunk_migration_v2", "complete") + self._reconcile_processing_sources() self._storage_initialized = True return written async def initialize_storage(self) -> int: """Recover interrupted writes and migrate legacy SQLite vectors into SeekDB.""" - if self._storage_initialized or self.native_chunk_index is None: + if self._storage_initialized: return 0 + if self.native_chunk_index is None: + if self.allow_sqlite_vector_fallback: + self._reconcile_processing_sources() + self._storage_initialized = True + return 0 + probe_native = getattr(self.native_chunk_index, "probe", None) + if callable(probe_native): + probe_native() self._recover_pending_sync_operations() written = 0 if ( @@ -405,11 +450,12 @@ async def initialize_storage(self) -> int: written = self._backfill_native_chunks_sync() if not self._migration_has_unresolved_sources: self._set_repository_meta("native_chunk_migration_v2", "complete") + self._reconcile_processing_sources() self._storage_initialized = True return written async def _ensure_storage_initialized(self) -> None: - if not self._storage_initialized and self.native_chunk_index is not None: + if not self._storage_initialized: await self.initialize_storage() def _backfill_native_chunks_sync(self) -> int: @@ -625,11 +671,16 @@ async def search_chunks( source_id for source_id in requested_source_ids if source_id not in native_source_id_set ] if fallback_source_ids: + fallback_query_embedding = ( + None + if self._has_embedding_profile_mismatch(fallback_source_ids) + else query_embedding + ) fallback_results = await self._search_sqlite_chunk_rows( query, self._chunk_rows(fallback_source_ids), top_k, - query_embedding, + fallback_query_embedding, None, ) results.extend(fallback_results) @@ -681,13 +732,47 @@ def _native_search_source_ids(self, source_ids: list[str], query_embedding: list return [] query_dimension = len(query_embedding) states = self._chunk_index_states(source_ids) - return [ - source_id + seekdb_states = { + source_id: states[source_id] for source_id in source_ids if source_id in states and states[source_id]["storage_mode"] == "seekdb" and states[source_id]["chunk_count"] > 0 - and states[source_id]["embedding_dimension"] == query_dimension + } + if not self.allow_sqlite_vector_fallback: + profile_mismatches = [ + source_id + for source_id, state in seekdb_states.items() + if self.embedding_profile_id + and state["embedding_profile"] + and state["embedding_profile"] != self.embedding_profile_id + ] + if profile_mismatches: + raise RuntimeError( + "Selected sources were indexed with a different embedding profile; " + "re-index the sources before vector retrieval: " + + ", ".join(profile_mismatches) + ) + dimension_mismatches = [ + source_id + for source_id, state in seekdb_states.items() + if state["embedding_dimension"] != query_dimension + ] + if dimension_mismatches: + raise RuntimeError( + "Selected sources use an incompatible embedding dimension; " + "re-index the sources before vector retrieval: " + + ", ".join(dimension_mismatches) + ) + return [ + source_id + for source_id, state in seekdb_states.items() + if state["embedding_dimension"] == query_dimension + and ( + not self.embedding_profile_id + or not state["embedding_profile"] + or state["embedding_profile"] == self.embedding_profile_id + ) ] def _native_source_ids_with_chunks(self, source_ids: list[str]) -> list[str]: @@ -710,6 +795,62 @@ def _legacy_source_ids(self, source_ids: list[str]) -> list[str]: and states[source_id]["chunk_count"] > 0 ] + def _has_embedding_profile_mismatch(self, source_ids: list[str]) -> bool: + if not self.embedding_profile_id: + return False + states = self._chunk_index_states(source_ids) + return any( + state["embedding_profile"] + and state["embedding_profile"] != self.embedding_profile_id + for state in states.values() + ) + + def indexed_vector_chunk_count(self) -> int: + row = self._conn.execute( + """ + SELECT COALESCE(SUM(chunk_count), 0) AS total + FROM chunk_index_state + WHERE storage_mode = 'seekdb' + """ + ).fetchone() + return int(row["total"]) + + def _reconcile_processing_sources(self) -> int: + rows = self._conn.execute( + """ + SELECT sources.payload, chunk_index_state.chunk_count AS indexed_chunk_count + FROM sources + JOIN chunk_index_state ON chunk_index_state.source_id = sources.id + WHERE sources.status = ? + """, + (SourceStatus.PROCESSING.value,), + ).fetchall() + recovered = 0 + with self._conn: + for row in rows: + source = KnowledgeSource.model_validate_json(row["payload"]) + if not source.text or source.chunk_count != row["indexed_chunk_count"]: + continue + source.status = SourceStatus.READY + source.error = None + source.updated_at = utc_now() + self._conn.execute( + """ + UPDATE sources + SET status = ?, payload = ?, updated_at = ? + WHERE id = ? AND status = ? + """, + ( + source.status.value, + _dump_model(source), + source.updated_at.isoformat(), + source.id, + SourceStatus.PROCESSING.value, + ), + ) + recovered += 1 + return recovered + def _record_sync_operation( self, source_id: str, @@ -967,12 +1108,14 @@ def _upsert_chunk_index_state( self._conn.execute( """ INSERT INTO chunk_index_state( - source_id, revision, chunk_count, embedding_dimension, storage_mode, updated_at - ) VALUES (?, ?, ?, ?, ?, ?) + source_id, revision, chunk_count, embedding_dimension, + embedding_profile, storage_mode, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source_id) DO UPDATE SET revision=excluded.revision, chunk_count=excluded.chunk_count, embedding_dimension=excluded.embedding_dimension, + embedding_profile=excluded.embedding_profile, storage_mode=excluded.storage_mode, updated_at=excluded.updated_at """, @@ -981,6 +1124,7 @@ def _upsert_chunk_index_state( revision, len(chunks), dimension, + self.embedding_profile_id, storage_mode, datetime.now(timezone.utc).isoformat(), ), @@ -992,7 +1136,7 @@ def _chunk_index_states(self, source_ids: list[str]) -> dict[str, sqlite3.Row]: placeholders = ",".join("?" for _ in source_ids) rows = self._conn.execute( f""" - SELECT source_id, chunk_count, embedding_dimension, storage_mode + SELECT source_id, chunk_count, embedding_dimension, embedding_profile, storage_mode FROM chunk_index_state WHERE source_id IN ({placeholders}) """, diff --git a/backend/infrastructure/vector_stores/seekdb_chunk_index.py b/backend/infrastructure/vector_stores/seekdb_chunk_index.py index cad6c36..840bc68 100644 --- a/backend/infrastructure/vector_stores/seekdb_chunk_index.py +++ b/backend/infrastructure/vector_stores/seekdb_chunk_index.py @@ -38,6 +38,17 @@ def close(self) -> None: client_exit(None, None, None) self._closed = True + def probe(self) -> None: + """Force the lazy embedded client to connect before request-time work starts.""" + + list_collections = getattr(self.client, "list_collections", None) + if not callable(list_collections): + raise SeekDBUnavailableError("Native SeekDB client does not support a readiness probe") + try: + list_collections() + except Exception as exc: + raise SeekDBUnavailableError("Native SeekDB readiness probe failed") from exc + def collection_name(self, source_id: str) -> str: digest = hashlib.sha1(source_id.encode("utf-8")).hexdigest()[:16] return f"chunks_{digest}" diff --git a/backend/infrastructure/vector_stores/seekdb_vector_store.py b/backend/infrastructure/vector_stores/seekdb_vector_store.py index 7d93374..999eafd 100644 --- a/backend/infrastructure/vector_stores/seekdb_vector_store.py +++ b/backend/infrastructure/vector_stores/seekdb_vector_store.py @@ -139,12 +139,20 @@ def storage_status(self) -> dict[str, Any]: return {"vector_backend": "unknown", "native_available": False} async def get_stats(self) -> dict[str, Any]: + initialize_storage = getattr(self.repository, "initialize_storage", None) + if callable(initialize_storage): + await initialize_storage() sources = await self.repository.list_sources() chunk_count = sum(source.chunk_count for source in sources) + indexed_chunk_count = getattr(self.repository, "indexed_vector_chunk_count", None) + indexed_vector_chunks = ( + indexed_chunk_count() if callable(indexed_chunk_count) else chunk_count + ) status = self.storage_status() return { "total_documents": len(sources), "total_chunks": chunk_count, + "indexed_vector_chunks": indexed_vector_chunks, "backend": status.get("vector_backend", "unknown"), "storage": status, } diff --git a/backend/main.py b/backend/main.py index 9848caf..ffde0e3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,6 +10,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from .config import get_settings from .api.routes.chat import router as chat_router @@ -80,17 +81,28 @@ async def lifespan(app: FastAPI): async def health_check(): from .dependencies import get_vector_store + current_settings = get_settings() vector_store = get_vector_store() stats = await vector_store.get_stats() storage = stats.get("storage", {}) - return { - "status": "healthy", - "version": settings.app_version, + native_available = storage.get("native_available", False) + embedding_profile = current_settings.api.models.embedding_model + embedding_configured = bool(embedding_profile.model and embedding_profile.api_key) + strict_seekdb = ( + current_settings.vector_store_type == "seekdb" + and not current_settings.seekdb_allow_sqlite_fallback + ) + healthy = not strict_seekdb or (native_available and embedding_configured) + payload = { + "status": "healthy" if healthy else "unhealthy", + "version": current_settings.app_version, "storage": { "actual_vector_backend": storage.get("vector_backend", stats.get("backend", "unknown")), - "native_available": storage.get("native_available", False), + "native_available": native_available, + "embedding_configured": embedding_configured, }, } + return JSONResponse(status_code=200 if healthy else 503, content=payload) # 向量存储统计 diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index a36a16e..681b6be 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -9,13 +9,23 @@ class FakeVectorStore: - def __init__(self, status: dict) -> None: + def __init__( + self, + status: dict, + total_chunks: int = 0, + indexed_vector_chunks: int | None = None, + ) -> None: self.status = status + self.total_chunks = total_chunks + self.indexed_vector_chunks = ( + total_chunks if indexed_vector_chunks is None else indexed_vector_chunks + ) async def get_stats(self) -> dict: return { "total_documents": 0, - "total_chunks": 0, + "total_chunks": self.total_chunks, + "indexed_vector_chunks": self.indexed_vector_chunks, "backend": self.status["vector_backend"], "storage": self.status, } @@ -23,6 +33,15 @@ async def get_stats(self) -> dict: def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample_config_file): monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True} + ) + ), + ) _RUNTIME_MODEL_OVERRIDES.clear() get_settings.cache_clear() DependencyContainer.reset_runtime_caches() @@ -104,6 +123,28 @@ def close_sync(self) -> None: assert DependencyContainer._knowledge_repository is None +def test_runtime_cache_reset_can_preserve_open_repository(): + class ClosableRepository: + def __init__(self) -> None: + self.close_calls = 0 + + def close_sync(self) -> None: + self.close_calls += 1 + + repository = ClosableRepository() + DependencyContainer._knowledge_repository = repository + DependencyContainer._vector_store = object() + DependencyContainer._slide_deck_service = object() + + DependencyContainer.reset_runtime_caches(preserve_repository=True) + + assert repository.close_calls == 0 + assert DependencyContainer._knowledge_repository is repository + assert DependencyContainer._vector_store is None + assert DependencyContainer._slide_deck_service is None + DependencyContainer.reset_runtime_caches() + + def test_force_new_repository_closes_old_repository_and_invalidates_dependents(monkeypatch): class ClosableRepository: def __init__(self, *_args, **_kwargs) -> None: @@ -189,6 +230,74 @@ def test_runtime_config_exposes_sqlite_fallback_status(monkeypatch, sample_confi assert storage["native_available"] is False +def test_runtime_config_rejects_embedding_identity_change_with_indexed_chunks( + monkeypatch, + sample_config_file, +): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True}, + total_chunks=3, + ) + ), + ) + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + client = TestClient(app) + + response = client.post( + "/api/config", + json={"models": {"embedding_model": {"model": "different-embedding-model"}}}, + ) + + assert response.status_code == 409 + assert "re-index" in response.json()["detail"] + assert get_settings().api.models.embedding_model.model == "test-embedding-model" + + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + +def test_runtime_config_allows_embedding_identity_change_without_indexed_vectors( + monkeypatch, + sample_config_file, +): + monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) + monkeypatch.setattr( + DependencyContainer, + "get_vector_store", + staticmethod( + lambda settings=None: FakeVectorStore( + {"vector_backend": "seekdb", "native_available": True}, + total_chunks=3, + indexed_vector_chunks=0, + ) + ), + ) + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + app = FastAPI() + app.include_router(config_router, prefix="/api") + response = TestClient(app).post( + "/api/config", + json={"models": {"embedding_model": {"model": "different-embedding-model"}}}, + ) + + assert response.status_code == 200 + assert response.json()["models"]["embedding_model"]["model"] == "different-embedding-model" + + _RUNTIME_MODEL_OVERRIDES.clear() + get_settings.cache_clear() + + def test_health_exposes_actual_vector_backend(monkeypatch): from backend import main as backend_main @@ -200,7 +309,8 @@ def fake_get_vector_store(): response = client.get("/health") - assert response.status_code == 200 + assert response.status_code == 503 + assert response.json()["status"] == "unhealthy" storage = response.json()["storage"] assert storage["actual_vector_backend"] == "unavailable" assert storage["native_available"] is False diff --git a/tests/test_seekdb_chunk_index.py b/tests/test_seekdb_chunk_index.py index 4254f93..29b4232 100644 --- a/tests/test_seekdb_chunk_index.py +++ b/tests/test_seekdb_chunk_index.py @@ -1,7 +1,4 @@ -import os -import subprocess import sys -import textwrap import types from pathlib import Path @@ -125,6 +122,7 @@ def __init__(self, path: str) -> None: self.collections = {} self.created = [] self.exit_calls = 0 + self.list_calls = 0 FakeClient.instances.append(self) def __exit__(self, exc_type, exc_value, traceback): @@ -145,12 +143,13 @@ def get_collection(self, name, embedding_function=None): raise KeyError(name) return self.collections[name] + def list_collections(self): + self.list_calls += 1 + return list(self.collections.values()) + @pytest.fixture(autouse=True) -def fake_pyseekdb(monkeypatch, request): - if request.node.name.startswith("test_real_pyseekdb"): - yield - return +def fake_pyseekdb(monkeypatch): FakeClient.instances.clear() module = types.SimpleNamespace(Client=FakeClient, HNSWConfiguration=FakeHNSWConfiguration) monkeypatch.setitem(sys.modules, "pyseekdb", module) @@ -198,6 +197,15 @@ def test_close_releases_native_client_once(tmp_path: Path): assert client.exit_calls == 1 +def test_probe_eagerly_opens_native_client(tmp_path: Path): + index = SeekDBChunkIndex(tmp_path / "native.seekdb") + client = FakeClient.instances[0] + + index.probe() + + assert client.list_calls == 1 + + def test_search_queries_seekdb_collections_and_reconstructs_chunks(tmp_path: Path): index = SeekDBChunkIndex(tmp_path / "native.seekdb") index.upsert_source_chunks("source-a", [chunk("chunk-a", "source-a", [0.1, 0.2, 0.3])]) @@ -567,217 +575,3 @@ def test_missing_pyseekdb_raises_without_silent_sqlite_vector_fallback(tmp_path: with pytest.raises(SeekDBUnavailableError): SeekDBChunkIndex(tmp_path / "native.seekdb") - - -def run_real_pyseekdb_subprocess(script: str) -> None: - repo_root = Path(__file__).resolve().parents[1] - env = os.environ.copy() - env["PYTHONPATH"] = os.pathsep.join( - value for value in [str(repo_root), env.get("PYTHONPATH", "")] if value - ) - subprocess_prelude = textwrap.dedent( - """ - try: - import pyseekdb # noqa: F401 - except Exception: - raise SystemExit(77) - - from backend.domain.source import KnowledgeChunk - from backend.infrastructure.vector_stores.seekdb_chunk_index import SeekDBChunkIndex - - - def chunk(chunk_id: str, source_id: str, embedding: list[float], metadata: dict | None = None): - return KnowledgeChunk( - id=chunk_id, - source_id=source_id, - content=f"{chunk_id} content", - chunk_index=0, - embedding=embedding, - metadata=metadata if metadata is not None else {"source_id": source_id}, - ) - """ - ) - subprocess_script = ( - subprocess_prelude - + "\nindex = None\ntry:\n" - + textwrap.indent(textwrap.dedent(script), " ") - + "\nfinally:\n if index is not None:\n index.close()\n" - ) - try: - completed = subprocess.run( - [sys.executable, "-c", subprocess_script], - cwd=repo_root, - env=env, - timeout=60, - check=False, - ) - except subprocess.TimeoutExpired: - pytest.skip("real pyseekdb subprocess timed out after 60 seconds") - if completed.returncode == 77: - pytest.skip("pyseekdb is unavailable in subprocess") - assert completed.returncode == 0 - - -def test_real_pyseekdb_upsert_refresh_and_query_round_trip(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks("real-source", [chunk("real-chunk", "real-source", [0.1, 0.2, 0.3])]) - results = index.search( - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=1, - ) - assert results[0]["chunk"].id == "real-chunk" - """ - ) - - -def test_real_pyseekdb_hybrid_search_round_trip(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks( - "real-source", - [ - chunk("real-lexical", "real-source", [0.9, 0.1, 0.1]), - chunk("real-vector", "real-source", [0.1, 0.2, 0.3]), - chunk("real-noise", "real-source", [0.8, 0.8, 0.8]), - ], - ) - results = index.hybrid_search( - query_text="real-lexical", - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=3, - ) - assert results - assert results[0]["chunk"].id == "real-lexical" - assert results[0]["score"] > 0 - """ - ) - - -def test_real_pyseekdb_reindex_replaces_stale_chunks(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks( - "real-source", - [ - chunk("real-a", "real-source", [0.1, 0.2, 0.3]), - chunk("real-b", "real-source", [0.3, 0.2, 0.1]), - ], - ) - index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) - results = index.search( - query_embedding=[0.9, 0.1, 0.1], - source_ids=["real-source"], - top_k=10, - ) - assert [item["chunk"].id for item in results] == ["real-c"] - """ - ) - - -def test_real_pyseekdb_conflicting_metadata_source_id_does_not_leave_stale_chunks( - tmp_path: Path, -): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks( - "real-source", - [ - chunk( - "real-a", - "real-source", - [0.1, 0.2, 0.3], - metadata={{"source_id": "wrong-source"}}, - ), - chunk( - "real-b", - "real-source", - [0.3, 0.2, 0.1], - metadata={{"source_id": "wrong-source"}}, - ), - ], - ) - index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) - results = index.search( - query_embedding=[0.9, 0.1, 0.1], - source_ids=["real-source"], - top_k=10, - ) - assert [item["chunk"].id for item in results] == ["real-c"] - """ - ) - - -def test_real_pyseekdb_empty_reindex_clears_existing_chunks(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) - index.upsert_source_chunks("real-source", []) - results = index.search( - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=10, - ) - assert results == [] - """ - ) - - -def test_real_pyseekdb_stale_bad_metadata_row_is_removed_by_collection_membership(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) - stale = chunk( - "real-stale", - "real-source", - [0.2, 0.2, 0.2], - metadata={{"source_id": "wrong-source"}}, - ) - collection = index._collection("real-source", dimension=3) - collection.upsert( - ids=[stale.id], - documents=[stale.content], - metadatas=[{{"source_id": "wrong-source", "payload": stale.model_dump_json()}}], - embeddings=[stale.embedding], - ) - collection.refresh_index() - - index.upsert_source_chunks("real-source", [chunk("real-c", "real-source", [0.9, 0.1, 0.1])]) - results = index.search( - query_embedding=[0.9, 0.1, 0.1], - source_ids=["real-source"], - top_k=10, - ) - assert [item["chunk"].id for item in results] == ["real-c"] - """ - ) - - -def test_real_pyseekdb_dimension_mismatch_preserves_existing_chunks(tmp_path: Path): - run_real_pyseekdb_subprocess( - f""" - index = SeekDBChunkIndex({str(tmp_path / "real.seekdb")!r}) - index.upsert_source_chunks("real-source", [chunk("real-a", "real-source", [0.1, 0.2, 0.3])]) - try: - index.upsert_source_chunks("real-source", [chunk("real-b", "real-source", [0.1, 0.2])]) - except ValueError: - pass - else: - raise AssertionError("dimension mismatch did not raise") - - results = index.search( - query_embedding=[0.1, 0.2, 0.3], - source_ids=["real-source"], - top_k=10, - ) - assert [item["chunk"].id for item in results] == ["real-a"] - """ - ) diff --git a/tests/test_slide_deck_domain_repository.py b/tests/test_slide_deck_domain_repository.py index 28c708b..450eee3 100644 --- a/tests/test_slide_deck_domain_repository.py +++ b/tests/test_slide_deck_domain_repository.py @@ -252,6 +252,9 @@ def __init__(self, path: str) -> None: calls.append(path) Path(path).mkdir(parents=True, exist_ok=True) + def list_collections(self): + return [] + monkeypatch.setitem(sys.modules, "pyseekdb", types.SimpleNamespace(Client=FakeClient)) db_path = tmp_path / "knowledge.db" diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index b7e4bcc..450d8a3 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -45,6 +45,10 @@ def test_repository_migrates_legacy_chunk_schema_with_vector_state(tmp_path: Pat columns = {row["name"] for row in repo._conn.execute("PRAGMA table_info(chunks)").fetchall()} assert "vector_state" in columns + state_columns = { + row["name"] for row in repo._conn.execute("PRAGMA table_info(chunk_index_state)").fetchall() + } + assert "embedding_profile" in state_columns @pytest.mark.asyncio @@ -210,6 +214,11 @@ def upsert_source_chunks(self, source_id, chunks): raise RuntimeError(self.message) +class ProbeFailingNativeIndex(RecordingNativeIndex): + def probe(self): + raise RuntimeError("native probe failed") + + class FailOnceNativeIndex(RecordingNativeIndex): def __init__(self, source_id: str, existing_chunks: list[KnowledgeChunk]) -> None: super().__init__() @@ -1134,6 +1143,20 @@ async def test_repository_backfill_requires_native_index_without_fallback(tmp_pa await repo.backfill_native_chunks() +def test_repository_storage_status_reports_failed_native_probe(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "probe-status.db", + native_chunk_index=ProbeFailingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + status = repo.storage_status() + + assert status["vector_backend"] == "unavailable" + assert status["native_available"] is False + assert "native probe failed" in status["error"] + + @pytest.mark.asyncio async def test_repository_backfill_clears_stale_native_for_fallback_rows_without_embeddings(tmp_path: Path): db_path = tmp_path / "knowledge.db" @@ -1232,6 +1255,8 @@ async def test_strict_backfill_keeps_unembedded_legacy_sources_lexically_searcha ).fetchone() assert state["storage_mode"] == "sqlite_legacy" assert repo._get_repository_meta("native_chunk_migration_v2") is None + chunks = await repo.get_chunks(source.id) + assert [chunk.id for chunk in chunks] == ["chunk-legacy-lexical"] @pytest.mark.asyncio @@ -1416,7 +1441,7 @@ async def test_repository_backfill_treats_empty_embedding_as_native_ineligible(t @pytest.mark.asyncio -async def test_repository_native_search_filters_sources_by_query_embedding_dimension( +async def test_repository_native_search_rejects_dimension_mismatched_sources( tmp_path: Path, ): db_path = tmp_path / "knowledge.db" @@ -1457,9 +1482,103 @@ async def test_repository_native_search_filters_sources_by_query_embedding_dimen repo = SeekDBRepository(db_path, native_chunk_index=native_index, allow_sqlite_vector_fallback=False) await repo.backfill_native_chunks() - await repo.search_chunks("dimension", source_ids=None, top_k=1, query_embedding=[0.1, 0.2, 0.3]) + with pytest.raises(RuntimeError, match="embedding dimension"): + await repo.search_chunks( + "dimension", + source_ids=None, + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + assert native_index.searches == [] + + +@pytest.mark.asyncio +async def test_repository_native_search_rejects_changed_embedding_profile(tmp_path: Path): + native_index = RecordingNativeIndex() + db_path = tmp_path / "knowledge.db" + source = KnowledgeSource(id="src-profile", kind=SourceKind.TEXT, title="Profile") + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + embedding_profile_id="profile-a", + ) + await writer.save_source(source) + await writer.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-profile", + source_id=source.id, + content="embedding profile content", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + ], + ) + await writer.close() + + reader = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + embedding_profile_id="profile-b", + ) + + with pytest.raises(RuntimeError, match="embedding profile"): + await reader.search_chunks( + "profile", + source_ids=[source.id], + top_k=1, + query_embedding=[0.1, 0.2, 0.3], + ) + + +@pytest.mark.asyncio +async def test_sqlite_fallback_does_not_compare_vectors_from_an_old_embedding_profile( + tmp_path: Path, +): + native_index = RecordingNativeIndex() + db_path = tmp_path / "fallback-profile.db" + source = KnowledgeSource(id="src-fallback-profile", kind=SourceKind.TEXT, title="Fallback") + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + embedding_profile_id="profile-a", + ) + await writer.save_source(source) + await writer.save_chunks( + source.id, + [ + KnowledgeChunk( + id="chunk-fallback-profile", + source_id=source.id, + content="unrelated content", + chunk_index=0, + embedding=[1.0, 0.0], + metadata={"source_id": source.id}, + ) + ], + ) + await writer.close() + + reader = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=True, + embedding_profile_id="profile-b", + ) + results = await reader.search_chunks( + "needle", + source_ids=[source.id], + top_k=1, + query_embedding=[1.0, 0.0], + ) - assert native_index.searches == [([0.1, 0.2, 0.3], [source_3d.id], 1)] + assert results == [] @pytest.mark.asyncio @@ -1750,6 +1869,82 @@ async def test_source_service_ingestion_and_vector_store_search_use_native_seekd assert results[0]["id"] == "native_chunk" +@pytest.mark.asyncio +async def test_source_service_marks_ready_only_after_chunk_commit(tmp_path: Path, monkeypatch): + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + tmp_path / "ingestion-order.db", + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + events: list[str] = [] + original_save_source = repo.save_source + original_save_chunks = repo.save_chunks + + async def record_source(source): + events.append(f"source:{source.status.value}") + return await original_save_source(source) + + async def record_chunks(source_id, chunks): + events.append("chunks") + return await original_save_chunks(source_id, chunks) + + monkeypatch.setattr(repo, "save_source", record_source) + monkeypatch.setattr(repo, "save_chunks", record_chunks) + service = SourceService( + repository=repo, + parser=DoclingParser(), + embedding_provider=DeterministicEmbeddingProvider(), + chunk_size=128, + chunk_overlap=0, + ) + + source = await service.create_text_source("Ordered", "Commit chunks before ready.") + + assert source.status.value == "ready" + assert events == ["source:processing", "source:processing", "chunks", "source:ready"] + + +@pytest.mark.asyncio +async def test_repository_recovers_processing_source_after_committed_chunks(tmp_path: Path): + db_path = tmp_path / "processing-recovery.db" + native_index = RecordingNativeIndex() + source = KnowledgeSource( + id="src-processing-recovery", + kind=SourceKind.TEXT, + title="Recovery", + text="Parsed content survived the interruption.", + chunk_count=1, + ) + chunk = KnowledgeChunk( + id="chunk-processing-recovery", + source_id=source.id, + content=source.text, + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + writer = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await writer.save_source(source) + await writer.save_chunks(source.id, [chunk]) + await writer.close() + + recovered = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + await recovered.initialize_storage() + + loaded = await recovered.get_source(source.id) + assert loaded is not None + assert loaded.status.value == "ready" + + @pytest.mark.asyncio async def test_text_source_is_chunked_with_citation_metadata(tmp_path: Path): repo = SeekDBRepository(tmp_path / "knowledge.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) @@ -1864,6 +2059,26 @@ def test_sources_api_text_upload_list_get_delete(tmp_path: Path): assert missing.status_code == 404 +def test_sources_api_rejects_strict_seekdb_ingestion_without_embeddings(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "strict-api.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) + app = FastAPI() + app.include_router(sources_router, prefix="/api") + app.dependency_overrides[get_source_service] = lambda: service + + response = TestClient(app).post( + "/api/sources/text", + json={"title": "Missing embeddings", "text": "This source requires vectors."}, + ) + + assert response.status_code == 422 + assert "require embeddings" in response.json()["detail"] + + def test_sources_api_file_upload_text_fallback(tmp_path: Path): repo = SeekDBRepository(tmp_path / "upload.db", native_chunk_index=None, allow_sqlite_vector_fallback=True) service = SourceService(repository=repo, parser=DoclingParser(), chunk_size=64, chunk_overlap=0) From 2cc6b1c5b0ca4acd9341fcf6cec929f679d6cd09 Mon Sep 17 00:00:00 2001 From: Lriver Date: Fri, 10 Jul 2026 15:16:15 +0800 Subject: [PATCH 22/22] fix(storage): address SeekDB review feedback --- .../repositories/seekdb_repository.py | 29 ++++++++++--- backend/main.py | 23 ++++++++-- tests/test_runtime_config_api.py | 31 +++++++++++++ tests/test_source_knowledge_base.py | 43 +++++++++++++++++++ 4 files changed, 115 insertions(+), 11 deletions(-) diff --git a/backend/infrastructure/repositories/seekdb_repository.py b/backend/infrastructure/repositories/seekdb_repository.py index f56f733..97e2682 100644 --- a/backend/infrastructure/repositories/seekdb_repository.py +++ b/backend/infrastructure/repositories/seekdb_repository.py @@ -106,6 +106,7 @@ def __init__( ) self._conn = sqlite3.connect(self.db_path, check_same_thread=False) self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") self._init_schema() self._storage_initialized = False self._migration_has_unresolved_sources = False @@ -466,7 +467,28 @@ def _backfill_native_chunks_sync(self) -> int: (SourceStatus.DELETED.value,), ).fetchall() sources = [KnowledgeSource.model_validate_json(row["payload"]) for row in source_rows] + reconciled_states = {"seekdb"} + if self.allow_sqlite_vector_fallback: + reconciled_states.add("sqlite_fallback_reconciled") + reconciled_placeholders = ",".join("?" for _ in reconciled_states) for source in sources: + state_row = self._conn.execute( + "SELECT source_id FROM chunk_index_state WHERE source_id = ?", + (source.id,), + ).fetchone() + if state_row: + unreconciled_row = self._conn.execute( + f""" + SELECT 1 + FROM chunks + WHERE source_id = ? + AND vector_state NOT IN ({reconciled_placeholders}) + LIMIT 1 + """, + (source.id, *reconciled_states), + ).fetchone() + if unreconciled_row is None: + continue sqlite_rows = self._conn.execute( "SELECT payload, vector_state FROM chunks WHERE source_id = ? ORDER BY chunk_index ASC", (source.id,), @@ -481,13 +503,6 @@ def _backfill_native_chunks_sync(self) -> int: storage_mode="seekdb", ) continue - reconciled_states = {"seekdb"} - if self.allow_sqlite_vector_fallback: - reconciled_states.add("sqlite_fallback_reconciled") - state_row = self._conn.execute( - "SELECT source_id FROM chunk_index_state WHERE source_id = ?", - (source.id,), - ).fetchone() if state_row and all(row["vector_state"] in reconciled_states for row in sqlite_rows): continue if state_row and all(row["vector_state"] == "legacy_needs_embedding" for row in sqlite_rows): diff --git a/backend/main.py b/backend/main.py index ffde0e3..73505f9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -82,12 +82,27 @@ async def health_check(): from .dependencies import get_vector_store current_settings = get_settings() - vector_store = get_vector_store() - stats = await vector_store.get_stats() - storage = stats.get("storage", {}) - native_available = storage.get("native_available", False) embedding_profile = current_settings.api.models.embedding_model embedding_configured = bool(embedding_profile.model and embedding_profile.api_key) + try: + vector_store = get_vector_store() + stats = await vector_store.get_stats() + storage = stats.get("storage", {}) + native_available = storage.get("native_available", False) + except Exception as exc: + return JSONResponse( + status_code=503, + content={ + "status": "unhealthy", + "version": current_settings.app_version, + "storage": { + "actual_vector_backend": "unavailable", + "native_available": False, + "embedding_configured": embedding_configured, + "error": str(exc), + }, + }, + ) strict_seekdb = ( current_settings.vector_store_type == "seekdb" and not current_settings.seekdb_allow_sqlite_fallback diff --git a/tests/test_runtime_config_api.py b/tests/test_runtime_config_api.py index 681b6be..ecb11aa 100644 --- a/tests/test_runtime_config_api.py +++ b/tests/test_runtime_config_api.py @@ -31,6 +31,11 @@ async def get_stats(self) -> dict: } +class FailingStatsVectorStore: + async def get_stats(self) -> dict: + raise RuntimeError("storage unavailable") + + def test_runtime_config_is_redacted_and_preserves_blank_keys(monkeypatch, sample_config_file): monkeypatch.setenv("NOTEBOOKLM_CONFIG_FILE", str(sample_config_file)) monkeypatch.setattr( @@ -314,3 +319,29 @@ def fake_get_vector_store(): storage = response.json()["storage"] assert storage["actual_vector_backend"] == "unavailable" assert storage["native_available"] is False + + +def test_health_returns_structured_503_when_storage_stats_fail(monkeypatch): + from backend import main as backend_main + + monkeypatch.setattr( + "backend.dependencies.get_vector_store", + lambda: FailingStatsVectorStore(), + ) + + response = TestClient(backend_main.app).get("/health") + + assert response.status_code == 503 + assert response.json() == { + "status": "unhealthy", + "version": get_settings().app_version, + "storage": { + "actual_vector_backend": "unavailable", + "native_available": False, + "embedding_configured": bool( + get_settings().api.models.embedding_model.model + and get_settings().api.models.embedding_model.api_key + ), + "error": "storage unavailable", + }, + } diff --git a/tests/test_source_knowledge_base.py b/tests/test_source_knowledge_base.py index 450d8a3..e1eaf65 100644 --- a/tests/test_source_knowledge_base.py +++ b/tests/test_source_knowledge_base.py @@ -51,6 +51,18 @@ def test_repository_migrates_legacy_chunk_schema_with_vector_state(tmp_path: Pat assert "embedding_profile" in state_columns +def test_repository_enables_sqlite_wal_mode(tmp_path: Path): + repo = SeekDBRepository( + tmp_path / "wal.db", + native_chunk_index=RecordingNativeIndex(), + allow_sqlite_vector_fallback=False, + ) + + journal_mode = repo._conn.execute("PRAGMA journal_mode").fetchone()[0] + + assert journal_mode.lower() == "wal" + + @pytest.mark.asyncio async def test_backfill_recognizes_pre_vector_state_strict_native_rows(tmp_path: Path): db_path = tmp_path / "legacy-strict.db" @@ -1088,6 +1100,37 @@ async def test_repository_backfills_existing_sqlite_chunks_to_native_index(tmp_p assert '"embedding":null' in sqlite_row["payload"] +@pytest.mark.asyncio +async def test_backfill_skips_payload_deserialization_for_reconciled_sources(tmp_path: Path): + db_path = tmp_path / "reconciled.db" + native_index = RecordingNativeIndex() + repo = SeekDBRepository( + db_path, + native_chunk_index=native_index, + allow_sqlite_vector_fallback=False, + ) + source = KnowledgeSource(id="src-reconciled", kind=SourceKind.TEXT, title="Reconciled") + chunk = KnowledgeChunk( + id="chunk-reconciled", + source_id=source.id, + content="already reconciled", + chunk_index=0, + embedding=[0.1, 0.2, 0.3], + metadata={"source_id": source.id}, + ) + await repo.save_source(source) + await repo.save_chunks(source.id, [chunk]) + repo._conn.execute( + "UPDATE chunks SET payload = 'not-json' WHERE source_id = ?", + (source.id,), + ) + repo._conn.commit() + + written = repo._backfill_native_chunks_sync() + + assert written == 0 + + @pytest.mark.asyncio async def test_backfill_restores_native_chunks_when_sqlite_scrub_fails(tmp_path: Path): db_path = tmp_path / "knowledge.db"