From 234542e74129506c54f8732c375339ce971a7955 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark" Date: Sun, 2 Aug 2026 15:39:52 -0500 Subject: [PATCH] fix(observability): stop reporting degraded and failed states as fine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things that were quietly invisible. doctor reported `semantic search: off` as severity "ok" — a green tick. On the 784-note reference vault, turning the semantic leg on moved recall@1 from 40% to 60% and MRR from 0.42 to 0.64, so the tick was calling "materially worse recall than you could have" healthy. Now a warning that names the fix. compliance.py swallowed every OSError with a bare `return`, so "the log is being written" and "every write has failed for a week" looked identical from outside. That is how the Windows rotation bug (#202) hid for a full release. Failures now leave a labelled breadcrumb in the hook failure log doctor already reads. A missing log on a fresh machine stays silent — that is not a failure. The retrieval eval set goes from 5 labelled cases to 30. Five sounds adequate until you notice each case is worth 20 percentage points: a real 10% gain could not register, and did not — #193 scored identically on five cases while individual ranks moved underneath it. Queries are phrased the way an agent asks, avoid echoing their target's title words, and were authored from note contents before anything was measured. Baseline: recall@1 60.0%, recall@5 83.3%, MRR 0.704 — recall@5 far above what the 5-case set implied, which is the underpowering made visible. Two known misses are kept rather than relabelled: a near-duplicate note outranking its target (a vault problem for consolidate/frontier), and a genuine gap where the note says "production-grade, hardened" and the query says "quality bar". Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 40 ++++++++++ src/omind/bench.py | 162 +++++++++++++++++++++++++++++++++++++--- src/omind/cli.py | 11 ++- src/omind/compliance.py | 39 ++++++++-- tests/test_cli.py | 22 ++++++ 5 files changed, 257 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 571536b..55f4196 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.1.0] - 2026-08-02 +### Changed +- **`omind doctor` no longer reports a degraded search as healthy.** + `semantic search: off (keyword path)` was severity `ok` — a green tick. On a + real 784-note vault, turning the semantic leg on moved **recall@1 from 40% to + 60% and MRR from 0.42 to 0.64**, so the tick was reporting "materially worse + recall than you could have" as fine. It is now a warning that names the fix + (`pip install 'omind[embed]'`). Same failure shape as an index that silently + stopped updating: the honest signal existed and did not read as a problem. +- **The retrieval eval set went from 5 labelled cases to 30.** Five sounds + adequate until you notice each case is worth 20 percentage points: a change + that genuinely improved recall by 10% could not register at all — and did not. + The contextual-prefix experiment ([#193](https://github.com/CryptoJones/omind/issues/193)) + scored *identically* on five cases while individual ranks moved underneath. + Five cases cannot distinguish "no effect" from "an effect this instrument + cannot see." + + Queries are phrased the way an agent actually asks and deliberately avoid + echoing their target's title words, so no case is winnable by literal title + matching. They were authored from note contents *before* anything was + measured. Baseline on the reference vault: **recall@1 60.0%, recall@5 83.3%, + MRR 0.704** — note recall@5 is far higher than the 5-case set implied (60%), + which is the underpowering made visible. + + Two known misses are kept rather than relabelled: one where a near-duplicate + note outranks the target (a *vault* problem, which `consolidate` and + `graph frontier` exist to find), and one genuine retrieval gap where the note + says "production-grade, hardened, fault-tolerant" and the query says "quality + bar" — no lexical overlap, and the semantic leg does not bridge it. + +### Fixed +- **`compliance.py` leaves a breadcrumb when it swallows an error.** Every + failure path returned silently so the guard hook can never raise into the + agent — correct, but it made "the compliance log is being written" and "every + write has failed for a week" look identical from outside. That is exactly how + the Windows rotation bug ([#202](https://github.com/CryptoJones/omind/issues/202)) + hid for a full release: a `PermissionError` absorbed by a bare + `except OSError: return`, with nothing anywhere to read. Failures now route to + the size-capped hook failure log that `omind doctor` already reads. A missing + log on a fresh machine is still silent, because that one is not a failure. + ### Added - **`Transaction.remove()` — journaled, recoverable deletion.** The journal could only express writes, so a multi-note operation that *moves* notes could diff --git a/src/omind/bench.py b/src/omind/bench.py index 77820e9..6ca08f9 100644 --- a/src/omind/bench.py +++ b/src/omind/bench.py @@ -36,24 +36,166 @@ "mesh sync conflict", ) -#: Small labelled set drawn from the durable notes present in CryptoJones's -#: reference vault. It is intentionally human-readable and version-controlled: -#: ranking changes should move these metrics, not merely "look better." +#: Labelled query→note pairs scored by ``--quality``, drawn from the durable +#: (non-generated) notes in CryptoJones's reference vault. Version-controlled +#: and human-readable on purpose: ranking changes should move these numbers, +#: not merely "look better." +#: +#: **Thirty cases, not five.** The original set was five, which sounds fine +#: until you notice each case is worth 20 percentage points — a change that +#: genuinely improved recall by 10% could not register at all, and did not: +#: the contextual-prefix experiment (#193) scored *identically* on five cases +#: while two of them moved several ranks underneath. Five cases cannot tell +#: "no effect" from "an effect this instrument cannot see." +#: +#: Queries are written the way an agent actually asks — "may I stop when I +#: reach a natural stopping point" — and deliberately avoid echoing their +#: target's title words, so a case cannot be won by literal title matching +#: alone. They were authored from note *contents* before anything was measured, +#: so the set is not tuned to what already ranks well. +#: +#: Two cases are known misses and are kept on purpose: +#: * "sign in to Claude Code with a Max subscription" — a near-duplicate note +#: outranks the labelled target. That is a *vault* problem (one fact in two +#: notes), which is what ``consolidate`` and ``graph frontier`` are for. +#: * "what quality bar must the code I write meet" — the note says +#: "production-grade by default, hardened, fault-tolerant" and the query +#: says "quality bar". No lexical overlap, and the semantic leg does not +#: bridge it. A real retrieval gap, left visible rather than relabelled. QUALITY_CASES = ( - ("where does long-term assistant memory live", "Omi Is The Memory.md"), ( - "how does CryptoJones want the assistant to work", - "Working Preferences - How CryptoJones Wants Me to Operate.md", + 'where does long-term assistant memory live', + 'Omi Is The Memory.md', ), - ("what voice and persona should Dix use", "Voice and Persona - Dix and Shelly.md"), ( - "rules for working in git repos and handling secrets", - "Operational Rules - Git Repos and Secrets.md", + 'how does CryptoJones want the assistant to work', + 'Working Preferences - How CryptoJones Wants Me to Operate.md', + ), + ( + 'what voice and persona should Dix use', + 'Voice and Persona - Dix and Shelly.md', + ), + ( + 'rules for working in git repos and handling secrets', + 'Operational Rules - Git Repos and Secrets.md', + ), + ( + 'how should durable memory notes be created', + 'Memory Workflow.md', + ), + ( + 'how do I sign in to Claude Code with a Max subscription', + 'Claude Code auth Claude Max Pro subscriptions require OAuth.md', + ), + ( + 'what colour theme did we build for the terminal CLI', + 'Cyberdeck — Claude Code custom theme (~ .claude themes cyberdeck.json).md', + ), + ( + 'which repo do I clone to start a new MCP server', + 'mcp-server-baseline — private consulting repo for building MCP servers 2026-06-14.md', + ), + ( + 'how many pull requests before Codeberg throttles me', + 'project-codeberg-rate-limit.md', + ), + ( + 'what is GayHydra', + 'project-gayhydra.md', + ), + ( + 'should I hand over commands or run them myself', + "CJ preference — run commands yourself, don't hand them off 2026-06-13.md", + ), + ( + 'who starts the Windows test virtual machine', + 'Win11 QEMU VM on pluto — Claude launches it (standing procedure).md', + ), + ( + 'what quality bar must the code I write meet', + 'Engineering Standards - Scripts and Code.md', + ), + ( + 'may I stop when I reach a natural stopping point', + "CryptoJones operating mode — never stop at 'natural stopping points'; standing full " + 'pre-authorization to act.md', + ), + ( + 'what should I do immediately after bouncing audio', + 'CJ music production ALWAYS auto-open renders in VLC + DRIVE the work (proactive, ' + "don't offload). 2026-07-04 (dix).md", + ), + ( + 'is adding tests unprompted considered scope creep', + 'feedback-proactive-ci-testing.md', + ), + ( + 'what art direction was locked for the sequel', + 'Flatline Sessions sequel art style LOCKED — heavy rotoscope plus retro 35mm sci-fi ' + 'grammar (2026-07-02).md', + ), + ( + 'how do I derive scales by dividing the octave equally', + 'Slonimsky — Thesaurus of Scales and Melodic Patterns (systematic-composition ' + 'distillation).md', + ), + ( + 'how does human memory map onto musical time scales', + 'Bob Snyder — Music and Memory An Introduction (psychoacoustics distillation).md', + ), + ( + 'what decides whether a downbeat feels early or late', + 'Christopher Hasty — Meter as Rhythm (distillation).md', + ), + ( + 'where is the public website hosted', + 'Web presence — www.cryptojones.dev cryptojones.dev.md', + ), + ( + 'which machine runs the dedicated game server', + 'XSpaceWar-AI dedicated server — running it on makemake.md', + ), + ( + 'do the course pipelines share one virtualenv', + "Course render pipelines SHARE one venv assets models via symlinks — don't delete the " + 'anchor (2026-06-19).md', + ), + ( + 'which local coding model won the benchmark', + 'MacminiM2Pro_ModelShowdown — benchmark matrix run + dedicated-machine protocol ' + '(2026-07-01).md', + ), + ( + 'are there better uncensored models for the V620 yet', + 'Model re-sweep 2026-06-27 — late-June uncensored ≤32GB V620 candidates; incumbents ' + 'still hold.md', + ), + ( + 'which old game are we remaking in Godot', + 'Neuromancer Godot remake — contributors to credit + project basics.md', + ), + ( + 'is the lora explainer translated into other languages', + 'lora-for-hackers-scope.md', + ), + ( + 'when did the consult gate become graduated', + 'omind 2.45.0 — graduated consult-gate (warn-then-enforce, #98); fleet converged ' + '(2026-06-22).md', + ), + ( + 'what visual style did UNSDF move to', + 'UNSDF — modern 2D JRPG art overhaul shipped to main (2026-06-30).md', + ), + ( + 'how is rhythm modelled as interference of periodicities', + 'Schillinger — The Schillinger System of Musical Composition (systematic-composition ' + 'distillation).md', ), - ("how should durable memory notes be created", "Memory Workflow.md"), ) + @dataclass class Measurement: """One measured number, with the unit it is measured in.""" diff --git a/src/omind/cli.py b/src/omind/cli.py index c6fb0ef..11be760 100644 --- a/src/omind/cli.py +++ b/src/omind/cli.py @@ -772,11 +772,18 @@ def _diagnose_search_index(config: SetupConfig) -> list[CheckResult]: ) ) else: + # WARN, not ok. This reads as a supported configuration, and it is — + # retrieval works. But on a real 784-note vault, turning the semantic + # leg on moved recall@1 from 40% to 60% and MRR from 0.42 to 0.64. A + # green tick for "your recall is materially worse than it could be" is + # the same failure mode as an index that silently stopped updating: the + # honest signal existed and did not read as a problem. results.append( CheckResult( "search_semantic", - "ok", - f"semantic search: off (keyword path) — {semantic['reason']}", + "warn", + f"semantic search: off (keyword path) — {semantic['reason']}; " + "install the extra for ~20pp better recall: pip install 'omind[embed]'", ) ) diff --git a/src/omind/compliance.py b/src/omind/compliance.py index a2225bb..f5f8ece 100644 --- a/src/omind/compliance.py +++ b/src/omind/compliance.py @@ -69,6 +69,27 @@ def _truncate(text: str, limit: int = _COMMAND_CAP) -> str: return collapsed if len(collapsed) <= limit else collapsed[: limit - 1] + "…" +def _breadcrumb(context: str, exc: BaseException) -> None: + """Leave a trace when this module swallows an error. Never raises. + + Every failure path here returns silently so the guard hook can never raise + into the agent — correct, but it also meant "the compliance log is being + written" and "every write has failed for a week" looked identical from the + outside. That is exactly how the Windows rotation bug (#202) hid: a + PermissionError absorbed by a bare `except OSError: return`, with nothing + anywhere to read. + + Routed through the hooks failure log, which is size-capped and already the + place `omind doctor` looks. Imported lazily: hooks imports this module. + """ + try: + from omind.hooks import _record_failure + + _record_failure(f"compliance.{context}", exc) + except Exception: + return + + def log_event( kind: str, *, @@ -114,8 +135,10 @@ def log_event( oversized = os.fstat(fd).st_size > _LOG_CAP_BYTES if oversized: _rotate_if_needed(path) - except OSError: - return + except OSError as exc: + # A dropped compliance record is a hole in the audit trail, not a + # nuisance: the recidivism ladder counts what is in this file. + _breadcrumb("log_event", exc) def _rotate_if_needed(path: Path) -> None: @@ -136,14 +159,17 @@ def _rotate_if_needed(path: Path) -> None: try: # 0o600: nothing outside this user's own hooks ever takes this lock. fd = os.open(lock_path, os.O_WRONLY | os.O_CREAT, 0o600) - except OSError: + except OSError as exc: + _breadcrumb("rotate_lock", exc) return try: filelock.lock_fd(fd) # Re-check under the lock: another process may have just rotated. if path.stat().st_size > _LOG_CAP_BYTES: os.replace(path, compliance_archive_path()) - except OSError: + except OSError as exc: + # This is the exact swallow that hid #202 on Windows for a full release. + _breadcrumb("rotate", exc) return finally: with contextlib.suppress(OSError): @@ -166,7 +192,10 @@ def _parse(path: Path) -> list[dict[str, Any]]: # strict decoding made read_events raise forever and took down the # checkpoint timer / doctor / corpus export until the log was hand-repaired. lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: + except FileNotFoundError: + return [] # no log yet is the normal state on a fresh machine + except OSError as exc: + _breadcrumb("read_events", exc) return [] events: list[dict[str, Any]] = [] for line in lines: diff --git a/tests/test_cli.py b/tests/test_cli.py index ecbd122..2909063 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -279,3 +279,25 @@ def _enc(texts: list[str]) -> list[list[float]]: err = capsys.readouterr().err assert "Release Guide" in err and "similar" in err embed.reset() + + +def test_doctor_warns_when_semantic_search_is_off( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A degraded search must not report as a green tick. + + `semantic search: off` was severity "ok". On a real 784-note vault, turning + the semantic leg on moved recall@1 40% -> 60% and MRR 0.42 -> 0.64, so the + green tick reported "materially worse recall than you could have" as + healthy — the same shape as an index that silently stopped updating. + """ + from omind import cli, embed + from omind.provision import SetupConfig + + monkeypatch.setattr( + embed, "status", lambda: {"available": False, "reason": "model2vec not importable"} + ) + config = SetupConfig(vault=tmp_path, folder="OMI") + checks = {c.key: c for c in cli._diagnose_search_index(config)} + assert checks["search_semantic"].level == "warn" + assert "omind[embed]" in checks["search_semantic"].message