diff --git a/CHANGELOG.md b/CHANGELOG.md index 571536b..55f4196 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.1.0] - 2026-08-02 +### Changed +- **`omind doctor` no longer reports a degraded search as healthy.** + `semantic search: off (keyword path)` was severity `ok` — a green tick. On a + real 784-note vault, turning the semantic leg on moved **recall@1 from 40% to + 60% and MRR from 0.42 to 0.64**, so the tick was reporting "materially worse + recall than you could have" as fine. It is now a warning that names the fix + (`pip install 'omind[embed]'`). Same failure shape as an index that silently + stopped updating: the honest signal existed and did not read as a problem. +- **The retrieval eval set went from 5 labelled cases to 30.** Five sounds + adequate until you notice each case is worth 20 percentage points: a change + that genuinely improved recall by 10% could not register at all — and did not. + The contextual-prefix experiment ([#193](https://github.com/CryptoJones/omind/issues/193)) + scored *identically* on five cases while individual ranks moved underneath. + Five cases cannot distinguish "no effect" from "an effect this instrument + cannot see." + + Queries are phrased the way an agent actually asks and deliberately avoid + echoing their target's title words, so no case is winnable by literal title + matching. They were authored from note contents *before* anything was + measured. Baseline on the reference vault: **recall@1 60.0%, recall@5 83.3%, + MRR 0.704** — note recall@5 is far higher than the 5-case set implied (60%), + which is the underpowering made visible. + + Two known misses are kept rather than relabelled: one where a near-duplicate + note outranks the target (a *vault* problem, which `consolidate` and + `graph frontier` exist to find), and one genuine retrieval gap where the note + says "production-grade, hardened, fault-tolerant" and the query says "quality + bar" — no lexical overlap, and the semantic leg does not bridge it. + +### Fixed +- **`compliance.py` leaves a breadcrumb when it swallows an error.** Every + failure path returned silently so the guard hook can never raise into the + agent — correct, but it made "the compliance log is being written" and "every + write has failed for a week" look identical from outside. That is exactly how + the Windows rotation bug ([#202](https://github.com/CryptoJones/omind/issues/202)) + hid for a full release: a `PermissionError` absorbed by a bare + `except OSError: return`, with nothing anywhere to read. Failures now route to + the size-capped hook failure log that `omind doctor` already reads. A missing + log on a fresh machine is still silent, because that one is not a failure. + ### Added - **`Transaction.remove()` — journaled, recoverable deletion.** The journal could only express writes, so a multi-note operation that *moves* notes could diff --git a/src/omind/bench.py b/src/omind/bench.py index 77820e9..6ca08f9 100644 --- a/src/omind/bench.py +++ b/src/omind/bench.py @@ -36,24 +36,166 @@ "mesh sync conflict", ) -#: Small labelled set drawn from the durable notes present in CryptoJones's -#: reference vault. It is intentionally human-readable and version-controlled: -#: ranking changes should move these metrics, not merely "look better." +#: Labelled query→note pairs scored by ``--quality``, drawn from the durable +#: (non-generated) notes in CryptoJones's reference vault. Version-controlled +#: and human-readable on purpose: ranking changes should move these numbers, +#: not merely "look better." +#: +#: **Thirty cases, not five.** The original set was five, which sounds fine +#: until you notice each case is worth 20 percentage points — a change that +#: genuinely improved recall by 10% could not register at all, and did not: +#: the contextual-prefix experiment (#193) scored *identically* on five cases +#: while two of them moved several ranks underneath. Five cases cannot tell +#: "no effect" from "an effect this instrument cannot see." +#: +#: Queries are written the way an agent actually asks — "may I stop when I +#: reach a natural stopping point" — and deliberately avoid echoing their +#: target's title words, so a case cannot be won by literal title matching +#: alone. They were authored from note *contents* before anything was measured, +#: so the set is not tuned to what already ranks well. +#: +#: Two cases are known misses and are kept on purpose: +#: * "sign in to Claude Code with a Max subscription" — a near-duplicate note +#: outranks the labelled target. That is a *vault* problem (one fact in two +#: notes), which is what ``consolidate`` and ``graph frontier`` are for. +#: * "what quality bar must the code I write meet" — the note says +#: "production-grade by default, hardened, fault-tolerant" and the query +#: says "quality bar". No lexical overlap, and the semantic leg does not +#: bridge it. A real retrieval gap, left visible rather than relabelled. QUALITY_CASES = ( - ("where does long-term assistant memory live", "Omi Is The Memory.md"), ( - "how does CryptoJones want the assistant to work", - "Working Preferences - How CryptoJones Wants Me to Operate.md", + 'where does long-term assistant memory live', + 'Omi Is The Memory.md', ), - ("what voice and persona should Dix use", "Voice and Persona - Dix and Shelly.md"), ( - "rules for working in git repos and handling secrets", - "Operational Rules - Git Repos and Secrets.md", + 'how does CryptoJones want the assistant to work', + 'Working Preferences - How CryptoJones Wants Me to Operate.md', + ), + ( + 'what voice and persona should Dix use', + 'Voice and Persona - Dix and Shelly.md', + ), + ( + 'rules for working in git repos and handling secrets', + 'Operational Rules - Git Repos and Secrets.md', + ), + ( + 'how should durable memory notes be created', + 'Memory Workflow.md', + ), + ( + 'how do I sign in to Claude Code with a Max subscription', + 'Claude Code auth Claude Max Pro subscriptions require OAuth.md', + ), + ( + 'what colour theme did we build for the terminal CLI', + 'Cyberdeck — Claude Code custom theme (~ .claude themes cyberdeck.json).md', + ), + ( + 'which repo do I clone to start a new MCP server', + 'mcp-server-baseline — private consulting repo for building MCP servers 2026-06-14.md', + ), + ( + 'how many pull requests before Codeberg throttles me', + 'project-codeberg-rate-limit.md', + ), + ( + 'what is GayHydra', + 'project-gayhydra.md', + ), + ( + 'should I hand over commands or run them myself', + "CJ preference — run commands yourself, don't hand them off 2026-06-13.md", + ), + ( + 'who starts the Windows test virtual machine', + 'Win11 QEMU VM on pluto — Claude launches it (standing procedure).md', + ), + ( + 'what quality bar must the code I write meet', + 'Engineering Standards - Scripts and Code.md', + ), + ( + 'may I stop when I reach a natural stopping point', + "CryptoJones operating mode — never stop at 'natural stopping points'; standing full " + 'pre-authorization to act.md', + ), + ( + 'what should I do immediately after bouncing audio', + 'CJ music production ALWAYS auto-open renders in VLC + DRIVE the work (proactive, ' + "don't offload). 2026-07-04 (dix).md", + ), + ( + 'is adding tests unprompted considered scope creep', + 'feedback-proactive-ci-testing.md', + ), + ( + 'what art direction was locked for the sequel', + 'Flatline Sessions sequel art style LOCKED — heavy rotoscope plus retro 35mm sci-fi ' + 'grammar (2026-07-02).md', + ), + ( + 'how do I derive scales by dividing the octave equally', + 'Slonimsky — Thesaurus of Scales and Melodic Patterns (systematic-composition ' + 'distillation).md', + ), + ( + 'how does human memory map onto musical time scales', + 'Bob Snyder — Music and Memory An Introduction (psychoacoustics distillation).md', + ), + ( + 'what decides whether a downbeat feels early or late', + 'Christopher Hasty — Meter as Rhythm (distillation).md', + ), + ( + 'where is the public website hosted', + 'Web presence — www.cryptojones.dev cryptojones.dev.md', + ), + ( + 'which machine runs the dedicated game server', + 'XSpaceWar-AI dedicated server — running it on makemake.md', + ), + ( + 'do the course pipelines share one virtualenv', + "Course render pipelines SHARE one venv assets models via symlinks — don't delete the " + 'anchor (2026-06-19).md', + ), + ( + 'which local coding model won the benchmark', + 'MacminiM2Pro_ModelShowdown — benchmark matrix run + dedicated-machine protocol ' + '(2026-07-01).md', + ), + ( + 'are there better uncensored models for the V620 yet', + 'Model re-sweep 2026-06-27 — late-June uncensored ≤32GB V620 candidates; incumbents ' + 'still hold.md', + ), + ( + 'which old game are we remaking in Godot', + 'Neuromancer Godot remake — contributors to credit + project basics.md', + ), + ( + 'is the lora explainer translated into other languages', + 'lora-for-hackers-scope.md', + ), + ( + 'when did the consult gate become graduated', + 'omind 2.45.0 — graduated consult-gate (warn-then-enforce, #98); fleet converged ' + '(2026-06-22).md', + ), + ( + 'what visual style did UNSDF move to', + 'UNSDF — modern 2D JRPG art overhaul shipped to main (2026-06-30).md', + ), + ( + 'how is rhythm modelled as interference of periodicities', + 'Schillinger — The Schillinger System of Musical Composition (systematic-composition ' + 'distillation).md', ), - ("how should durable memory notes be created", "Memory Workflow.md"), ) + @dataclass class Measurement: """One measured number, with the unit it is measured in.""" diff --git a/src/omind/cli.py b/src/omind/cli.py index c6fb0ef..11be760 100644 --- a/src/omind/cli.py +++ b/src/omind/cli.py @@ -772,11 +772,18 @@ def _diagnose_search_index(config: SetupConfig) -> list[CheckResult]: ) ) else: + # WARN, not ok. This reads as a supported configuration, and it is — + # retrieval works. But on a real 784-note vault, turning the semantic + # leg on moved recall@1 from 40% to 60% and MRR from 0.42 to 0.64. A + # green tick for "your recall is materially worse than it could be" is + # the same failure mode as an index that silently stopped updating: the + # honest signal existed and did not read as a problem. results.append( CheckResult( "search_semantic", - "ok", - f"semantic search: off (keyword path) — {semantic['reason']}", + "warn", + f"semantic search: off (keyword path) — {semantic['reason']}; " + "install the extra for ~20pp better recall: pip install 'omind[embed]'", ) ) diff --git a/src/omind/compliance.py b/src/omind/compliance.py index a2225bb..f5f8ece 100644 --- a/src/omind/compliance.py +++ b/src/omind/compliance.py @@ -69,6 +69,27 @@ def _truncate(text: str, limit: int = _COMMAND_CAP) -> str: return collapsed if len(collapsed) <= limit else collapsed[: limit - 1] + "…" +def _breadcrumb(context: str, exc: BaseException) -> None: + """Leave a trace when this module swallows an error. Never raises. + + Every failure path here returns silently so the guard hook can never raise + into the agent — correct, but it also meant "the compliance log is being + written" and "every write has failed for a week" looked identical from the + outside. That is exactly how the Windows rotation bug (#202) hid: a + PermissionError absorbed by a bare `except OSError: return`, with nothing + anywhere to read. + + Routed through the hooks failure log, which is size-capped and already the + place `omind doctor` looks. Imported lazily: hooks imports this module. + """ + try: + from omind.hooks import _record_failure + + _record_failure(f"compliance.{context}", exc) + except Exception: + return + + def log_event( kind: str, *, @@ -114,8 +135,10 @@ def log_event( oversized = os.fstat(fd).st_size > _LOG_CAP_BYTES if oversized: _rotate_if_needed(path) - except OSError: - return + except OSError as exc: + # A dropped compliance record is a hole in the audit trail, not a + # nuisance: the recidivism ladder counts what is in this file. + _breadcrumb("log_event", exc) def _rotate_if_needed(path: Path) -> None: @@ -136,14 +159,17 @@ def _rotate_if_needed(path: Path) -> None: try: # 0o600: nothing outside this user's own hooks ever takes this lock. fd = os.open(lock_path, os.O_WRONLY | os.O_CREAT, 0o600) - except OSError: + except OSError as exc: + _breadcrumb("rotate_lock", exc) return try: filelock.lock_fd(fd) # Re-check under the lock: another process may have just rotated. if path.stat().st_size > _LOG_CAP_BYTES: os.replace(path, compliance_archive_path()) - except OSError: + except OSError as exc: + # This is the exact swallow that hid #202 on Windows for a full release. + _breadcrumb("rotate", exc) return finally: with contextlib.suppress(OSError): @@ -166,7 +192,10 @@ def _parse(path: Path) -> list[dict[str, Any]]: # strict decoding made read_events raise forever and took down the # checkpoint timer / doctor / corpus export until the log was hand-repaired. lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: + except FileNotFoundError: + return [] # no log yet is the normal state on a fresh machine + except OSError as exc: + _breadcrumb("read_events", exc) return [] events: list[dict[str, Any]] = [] for line in lines: diff --git a/tests/test_cli.py b/tests/test_cli.py index ecbd122..2909063 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -279,3 +279,25 @@ def _enc(texts: list[str]) -> list[list[float]]: err = capsys.readouterr().err assert "Release Guide" in err and "similar" in err embed.reset() + + +def test_doctor_warns_when_semantic_search_is_off( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A degraded search must not report as a green tick. + + `semantic search: off` was severity "ok". On a real 784-note vault, turning + the semantic leg on moved recall@1 40% -> 60% and MRR 0.42 -> 0.64, so the + green tick reported "materially worse recall than you could have" as + healthy — the same shape as an index that silently stopped updating. + """ + from omind import cli, embed + from omind.provision import SetupConfig + + monkeypatch.setattr( + embed, "status", lambda: {"available": False, "reason": "model2vec not importable"} + ) + config = SetupConfig(vault=tmp_path, folder="OMI") + checks = {c.key: c for c in cli._diagnose_search_index(config)} + assert checks["search_semantic"].level == "warn" + assert "omind[embed]" in checks["search_semantic"].message