diff --git a/BACKLOG.md b/BACKLOG.md index 0b338f8..f0b5a75 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -18,6 +18,12 @@ hardening, or docs, not defects._ `_weight_generated`, `_weight_superseded`, and `_owners` now share one map built once per index `generation` and cached per process, like the packed vector matrix. Query cost tracks the fused candidate set instead of the vault size. +- [x] **A `SCHEMA_VERSION` bump that adds a column wedged an existing search index** ([#210](https://github.com/CryptoJones/omind/issues/210)) — _bug (retrieval)_ — + shipped in 6.6.0 and caught while setting up the #193 eval gate: the baseline read + `recall@1 = 0%`. `_wipe` deleted rows but never dropped tables, so a new column never + materialised and every ingest failed silently forever. Retrieval fell back to the + substring scan, so it degraded quietly instead of erroring. `_wipe` now drops and + recreates; a test exercises the upgrade path from the previous shape. - [x] **Compliance-log rotation silently never fired on Windows** ([#202](https://github.com/CryptoJones/omind/issues/202)) — _bug (enforcement)_ — found by CI on the #188 PR, before it shipped. The rotation renamed the log while this process still held its fd; Windows refuses that, and the `PermissionError` was swallowed diff --git a/CHANGELOG.md b/CHANGELOG.md index 771af39..491ba15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,23 @@ _The last of the 2026-08-01 review and 2026-08-02 comparison backlogs, released together rather than as another run of point versions._ ### Fixed +- **A `SCHEMA_VERSION` bump that adds a column no longer wedges an existing + search index** ([#210](https://github.com/CryptoJones/omind/issues/210)). + `_wipe` cleared the index with `DELETE FROM`, rows only — but `_SCHEMA` is + entirely `CREATE TABLE IF NOT EXISTS` and runs *before* the wipe, so on an + existing index file the **old column shape survived**. 6.6.0 added two columns + to `notes` and bumped the schema 4 → 5, after which every ingest INSERT failed + with `no such column: confidence`, `refresh()` and `search()` returned `None` + on every call, and the only repair was a manual `omind reindex --rebuild`. + + Quiet by construction: retrieval fell back to the pre-index substring scan + (invariant 2 held), so the symptom was silently worse recall rather than an + error. Measured on a live 784-note vault: **recall@1 0% wedged → 60% rebuilt**, + MRR **0.00 → 0.64**. `_wipe` now drops the tables and re-runs `_SCHEMA`. + + Latent since `SCHEMA_VERSION` was introduced — earlier bumps happened to be + shape-compatible. `SCHEMA_VERSION` promised a migration it never performed, + and nothing tested the upgrade path from a previous shape; now something does. - **One failing `PostToolUse` side effect no longer cancels the rest** ([#204](https://github.com/CryptoJones/omind/issues/204), found while writing the #189 test). `hooks.run_hook` ran four independent subsystems — the loop diff --git a/src/omind/searchindex.py b/src/omind/searchindex.py index 457d613..2088f11 100644 --- a/src/omind/searchindex.py +++ b/src/omind/searchindex.py @@ -541,11 +541,22 @@ def _meta(db: sqlite3.Connection, key: str) -> str: return str(row["value"]) if row else "" def _wipe(self, db: sqlite3.Connection) -> None: - """Drop everything: the schema or the embedding model changed, so every - stored vector and every FTS row is suspect.""" + """Drop and recreate everything: the schema or the embedding model + changed, so every stored vector and every FTS row is suspect. + + This must ``DROP``, not ``DELETE``. ``_SCHEMA`` is all + ``CREATE TABLE IF NOT EXISTS``, so deleting rows leaves the *old column + shape* in place — and a ``SCHEMA_VERSION`` bump that adds a column then + made every INSERT fail with "no such column" forever. The index stayed + empty, `refresh()` and `search()` returned ``None`` on every call, and + the only repair was a manual `omind reindex --rebuild`. Retrieval fell + back to the pre-index substring scan (invariant 2 held), so the symptom + was silently worse results rather than an error. + """ for table in ("notes", "note_tags", "chunks", "chunks_fts", "vectors", "links", "meta"): with contextlib.suppress(sqlite3.Error): - db.execute(f"DELETE FROM {table}") + db.execute(f"DROP TABLE IF EXISTS {table}") + db.executescript(_SCHEMA) db.execute( "INSERT OR REPLACE INTO meta(key, value) VALUES ('schema', ?), ('model', ?)", (str(SCHEMA_VERSION), self.model), diff --git a/tests/test_searchindex.py b/tests/test_searchindex.py index 3921878..47a40cd 100644 --- a/tests/test_searchindex.py +++ b/tests/test_searchindex.py @@ -10,6 +10,7 @@ from __future__ import annotations +import sqlite3 from collections.abc import Iterator from pathlib import Path @@ -507,3 +508,34 @@ def test_a_conflict_is_surfaced_on_both_notes(omi: Path) -> None: by_name = {hit.filename: hit for hit in hits} assert by_name["Claim A.md"].conflicts_with == "Claim B.md" assert by_name["Claim B.md"].conflicts_with == "Claim A.md" # symmetric + + +def test_a_schema_bump_that_adds_a_column_rebuilds_the_index(omi: Path) -> None: + """A SCHEMA_VERSION bump must survive an existing index file. + + `_SCHEMA` is all CREATE TABLE IF NOT EXISTS, so wiping by DELETE left the + old *column shape* in place: every INSERT then failed with "no such column" + and the index stayed empty forever — refresh() and search() returning None + on every call, repairable only by a manual `omind reindex --rebuild`. + Retrieval fell back to the substring scan, so the symptom was quietly worse + results rather than an error. + """ + _note(omi, "Handbook", "curated operations", ["ops"], details="zebracorn rollback") + index = searchindex.SearchIndex(omi) + assert index.refresh() is not None + path = index.path() + index.close() + + # Age the file into the shape a previous release wrote: two fewer columns, + # and the schema number that shipped with them. + db = sqlite3.connect(path) + db.execute("ALTER TABLE notes DROP COLUMN confidence") + db.execute("ALTER TABLE notes DROP COLUMN conflicts_with") + db.execute("INSERT OR REPLACE INTO meta(key, value) VALUES ('schema', '4')") + db.commit() + db.close() + + upgraded = searchindex.SearchIndex(omi) + stats = upgraded.refresh() + assert stats is not None and stats.notes == 1 # rebuilt, not wedged + assert [hit.filename for hit in (upgraded.search("zebracorn") or [])] == ["Handbook.md"]