diff --git a/CHANGELOG.md b/CHANGELOG.md index e00503a..d471f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,76 @@ # Changelog +## [0.7.0rc2] - 2026-05-22 + +### Features + +- **Contradiction detection rebuilt with NLI (no LLM dep).** The + `memory_check_contradictions` MCP tool now uses a Natural Language + Inference cross-encoder (`cross-encoder/nli-deberta-v3-xsmall`, + 22M params, ~87 MB INT8 ONNX) instead of an LLM classifier. NLI + is the canonical non-LLM ML primitive for this exact task — + entailment / contradiction / neutral classification on a sentence + pair — and ships through the same FastEmbed-style ONNX runtime + path already in mnemon. **Zero new dependencies** (onnxruntime + + tokenizers + huggingface_hub all transitively required by + FastEmbed already). Replaces the prior LLM-based path that + couldn't work on Fly (`[server]` extras don't install + `llama-cpp-python` per the 2026-05-21 "mnemon is LLM-free by + design" decision, so the LLM path was effectively broken since + the original `[server]`/`[llm]` split). + - **Bidirectional classification.** Each candidate pair is run + through the cross-encoder twice (premise→hypothesis + + hypothesis→premise, ~10-20ms total on CPU INT8). The two + directions disambiguate the mnemon taxonomy: both entail → + `same`; new entails old but not vice versa → `update`; + contradiction in either direction → `contradiction`; both + neutral → `unrelated`. + - **Cosine gate preserved.** Existing + `CONTRADICTION_OVERLAP_THRESHOLD=0.7` still filters candidates + before NLI — protects against the rare NLI false-positive on + obviously-unrelated pairs. + - **Model baked into Fly image.** Dockerfile downloads the 87 MB + quantized ONNX model + tokenizer at build time, mirroring the + existing FastEmbed bake. Cold start adds the NLI load to the + pre-warm path (~5-8 seconds total vs 3-5 seconds prior). Health + check start period bumped 30s → 45s. + - **Clean error surface.** When NLI isn't loadable (e.g., model + download fails on a fresh local install without network), the + MCP tool returns a clear "skipped — NLI classifier unavailable" + message instead of an opaque "Error occurred during tool + execution" envelope. Fail-loud per + `feedback_no_silent_fails`. Composes with the recalled + `feedback-mnemon-pypi-upload-claude-is-authorized` mental model: + surface failure causes specifically, never the generic envelope. + +- **`dry_run` parameter on `memory_check_contradictions`.** When + `dry_run=True`, the tool reports what WOULD have decayed without + applying any mutations (no confidence changes, no relations + inserted). Closes the read/command-separation violation in the + prior `check_*` naming; useful for operator audit before + committing destructive changes (the 2026-05-22 standing-tier + promotion incident — operator review of three contradictory + liquidity figures — would have benefited from this). + +### Internal + +- **New module `src/mnemon/nli.py`** mirroring `embedder.py`: + lazy-loaded singleton, `prewarm()` for lifespan startup, + `classify_pair()` for single-direction, `classify_pair_bidirectional()` + for the mnemon-taxonomy mapping, `is_available()` probe, + `NLIUnavailableError` named exception. Operator override: + `MNEMON_NLI_ONNX_VARIANT` env var to swap between FP32 / FP16 / + INT8 variants (default INT8 AVX-512 for x86 Fly). +- **`contradiction.py` refactored**: LLM imports + prompt + construction removed; vector gate + NLI classify pipeline now + explicit in the docstring. Return shape gains `nli_unavailable` + and `dry_run` flags for caller-side handling. +- **Tests**: `tests/test_nli.py` (11 new) covers bidirectional + label mapping, error surfacing, availability probe. + `tests/test_contradiction.py` refactored to mock the NLI layer + instead of `mnemon.llm.generate`; adds dry-run mutation-skip + test + nli-unavailable clean-flag test. Suite 836 → 847 passing. + ## [0.7.0rc1] - 2026-05-22 ### Fixes diff --git a/Dockerfile b/Dockerfile index 6761d6c..9dea6f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,17 @@ RUN pip install --no-cache-dir ".[server]" ENV FASTEMBED_CACHE_DIR=/app/.cache/fastembed RUN python -c "from fastembed import TextEmbedding; TextEmbedding(model_name='BAAI/bge-small-en-v1.5', cache_dir='/app/.cache/fastembed')" +# Bake the NLI cross-encoder (~87 MB INT8 ONNX) for +# memory_check_contradictions — same rationale as the FastEmbed bake. +# Without this, the first contradiction check pays a 5-15 second +# download cost AND risks Anthropic's MCP-proxy timeout on the call. +# Model lives in /app/.cache/huggingface (default HF cache root). +ENV HF_HOME=/app/.cache/huggingface +RUN python -c "from huggingface_hub import hf_hub_download; \ + hf_hub_download(repo_id='cross-encoder/nli-deberta-v3-xsmall', filename='onnx/model_qint8_avx512.onnx'); \ + hf_hub_download(repo_id='cross-encoder/nli-deberta-v3-xsmall', filename='tokenizer.json'); \ + hf_hub_download(repo_id='cross-encoder/nli-deberta-v3-xsmall', filename='config.json')" + # Vault data persists in /data (mount a Fly volume here) ENV MNEMON_VAULT_DIR=/data RUN mkdir -p /data @@ -25,10 +36,12 @@ ENV PORT=8080 EXPOSE 8080 # Health check has a generous start period because the server pre-loads -# the embedding model on startup (see server_remote.py) — uvicorn does -# not bind the port until that load completes (~3-5 seconds on warm -# disk, longer on first-ever boot if the model isn't yet cached). -HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ +# BOTH the embedding model and the NLI classifier on startup (see +# server_remote.py) — uvicorn does not bind the port until both loads +# complete (~5-8 seconds on warm disk, longer on first-ever boot if +# models aren't yet cached). Start period bumped to 45s for the dual +# pre-warm. +HEALTHCHECK --interval=30s --timeout=5s --start-period=45s --retries=3 \ CMD python -c "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/health', timeout=3).status == 200 else 1)" CMD ["mnemon", "serve-remote"] diff --git a/pyproject.toml b/pyproject.toml index 106d86a..bd4b548 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "mnemon-memory" -version = "0.7.0rc1" +version = "0.7.0rc2" description = "Universal long-term memory layer for AI agents via MCP" readme = "README.md" license = "MIT" diff --git a/src/mnemon/__init__.py b/src/mnemon/__init__.py index 8f33084..07bd873 100644 --- a/src/mnemon/__init__.py +++ b/src/mnemon/__init__.py @@ -1,3 +1,3 @@ """mnemon — Universal long-term memory layer for AI agents via MCP.""" -__version__ = "0.7.0rc1" +__version__ = "0.7.0rc2" diff --git a/src/mnemon/contradiction.py b/src/mnemon/contradiction.py index 963e530..8aa6d12 100644 --- a/src/mnemon/contradiction.py +++ b/src/mnemon/contradiction.py @@ -1,19 +1,34 @@ """Contradiction detection — finds and resolves conflicting memories. -When a new memory is saved, searches for existing memories on the same -topic. Uses the local LLM to classify the relationship: - - same: identical fact, no action (adds "related" relation) - - update: new supersedes old, decay old confidence - - contradiction: direct conflict, decay old confidence more aggressively - - unrelated: different topics, no action +When a new memory is saved, searches for existing memories on the +same topic. Uses **NLI** (Natural Language Inference) to classify +the relationship between each candidate pair: + + - ``same`` : semantic equivalence, no action (adds ``related`` relation) + - ``update`` : new supersedes old, decay old confidence + - ``contradiction``: direct conflict, decay old confidence more aggressively + - ``unrelated`` : different topics, no action + +Two-stage pipeline: + 1. Cosine similarity gate (``CONTRADICTION_OVERLAP_THRESHOLD``) — + cheap filter; unrelated pairs never reach the classifier + 2. NLI cross-encoder bidirectional classification — outputs the + mnemon taxonomy label + +Replaces the prior LLM-based classifier (2026-05-22 — see +``private/mnemon-salience-tier-plan-260521.md``) per the standing +"mnemon is LLM-free by design" decision. NLI is the SOTA non-LLM +ML primitive for this exact task; the embedded cross-encoder model +(``cross-encoder/nli-deberta-v3-xsmall``, ~87 MB INT8) ships through +the same FastEmbed-style ONNX path that already powers embeddings — +zero new deps. Also provides time-based confidence decay with access reinforcement. - -Phase 3: LLM-based contradiction detection + confidence decay. """ from __future__ import annotations +import logging import math from typing import TYPE_CHECKING @@ -27,20 +42,12 @@ if TYPE_CHECKING: from .store import Store +logger = logging.getLogger(__name__) + UPDATE_DECAY = 0.15 # confidence reduction for superseded memories CONTRADICTION_DECAY = 0.25 CONFIDENCE_FLOOR = 0.2 -CLASSIFY_SYSTEM_PROMPT = ( - "You classify the relationship between two memories. " - "Given an existing memory and a new memory, respond with exactly one word:\n\n" - "- same: they express the same fact or decision\n" - "- update: the new memory supersedes or refines the old one\n" - "- contradiction: they directly conflict\n" - "- unrelated: different topics\n\n" - "Respond with ONLY the classification word, nothing else." -) - VALID_CLASSIFICATIONS = {"same", "update", "contradiction", "unrelated"} @@ -49,90 +56,164 @@ def check_contradictions( new_title: str, new_content: str, new_doc_id: int, + *, + dry_run: bool = False, ) -> dict: """Check a new memory against existing memories for contradictions. - Returns {decayed: int, relationships: [{doc_id, title, relationship}]}. + Two-stage pipeline: + 1. Vector-similarity gate filters to genuinely overlapping + candidates (``CONTRADICTION_OVERLAP_THRESHOLD``). + 2. NLI bidirectional classify maps each candidate to mnemon's + taxonomy (``same`` / ``update`` / ``contradiction`` / + ``unrelated``). + + Side effects on classification (skipped when ``dry_run=True``): + - ``update`` : decay old confidence by ``UPDATE_DECAY``, + insert ``'supersedes'`` relation + - ``contradiction`` : decay old confidence by ``CONTRADICTION_DECAY``, + insert ``'contradicts'`` relation + - ``same`` : insert ``'related'`` relation, no decay + - ``unrelated`` : no action + + Returns: + { + "decayed": int, # # of confidence decays applied + "relationships": [ + { + "doc_id": int, + "title": str, + "relationship": "same" | "update" | "contradiction" | "unrelated", + "probs": {"contradiction": float, "entailment": float, "neutral": float}, # NLI a→b + }, + ... + ], + "nli_unavailable": bool, # True iff NLI couldn't load (downgrades to cosine-only) + "dry_run": bool, # echoes the input flag + } """ relationships: list[dict] = [] decayed = 0 - # Find overlapping memories via vector similarity + # Stage 1 — vector similarity gate try: from .embedder import embed query_emb = embed(f"title: {new_title} | text: {new_content}") overlapping = store.search_vector(query_emb, 5) - except Exception: - return {"decayed": 0, "relationships": []} + except Exception as e: + logger.warning("contradiction: embed/search failed (%s); skipping check", e) + return { + "decayed": 0, "relationships": [], + "nli_unavailable": False, "dry_run": dry_run, + } - # Filter to genuinely overlapping results (exclude self) candidates = [ r for r in overlapping if r.doc_id != new_doc_id and r.score >= CONTRADICTION_OVERLAP_THRESHOLD ] if not candidates: - return {"decayed": 0, "relationships": []} + return { + "decayed": 0, "relationships": [], + "nli_unavailable": False, "dry_run": dry_run, + } - # Classify each relationship via LLM + # Stage 2 — NLI classify (bidirectional) try: - from .llm import generate - except ImportError: - return {"decayed": 0, "relationships": []} + from .nli import NLIUnavailableError, classify_pair_bidirectional + except ImportError as e: + logger.warning("contradiction: NLI module import failed: %s", e) + return { + "decayed": 0, "relationships": [], + "nli_unavailable": True, "dry_run": dry_run, + } for candidate in candidates: try: - prompt = ( - f"Existing memory:\nTitle: {candidate.title}\n" - f"Content: {candidate.content[:CONTRADICTION_CONTEXT_MAX_CHARS]}\n\n" - f"New memory:\nTitle: {new_title}\n" - f"Content: {new_content[:CONTRADICTION_CONTEXT_MAX_CHARS]}" + premise = ( + f"title: {candidate.title} | " + f"text: {candidate.content[:CONTRADICTION_CONTEXT_MAX_CHARS]}" + ) + hypothesis = ( + f"title: {new_title} | " + f"text: {new_content[:CONTRADICTION_CONTEXT_MAX_CHARS]}" + ) + result = classify_pair_bidirectional(premise, hypothesis) + classification = result.mnemon_label + except NLIUnavailableError as e: + # First candidate's NLI failure → bail out entirely with the + # named-error path; subsequent candidates would fail + # identically (singleton model load). Surfaces a clear + # "nli unavailable" flag for the caller to communicate. + logger.warning("contradiction: NLI unavailable: %s", e) + return { + "decayed": decayed, + "relationships": relationships, + "nli_unavailable": True, + "dry_run": dry_run, + } + except Exception as e: + # Per-candidate failure (tokenization edge case, etc.) — + # log + skip this candidate, continue with others. + logger.warning( + "contradiction: classify failed for candidate #%d: %s", + candidate.doc_id, e, + ) + continue + + if classification not in VALID_CLASSIFICATIONS: + logger.warning( + "contradiction: unexpected classification %r for #%d; skipping", + classification, candidate.doc_id, ) + continue - response = generate(CLASSIFY_SYSTEM_PROMPT, prompt, max_tokens=10) - classification = response.strip().lower() - - if classification not in VALID_CLASSIFICATIONS: - continue - - relationships.append({ - "doc_id": candidate.doc_id, - "title": candidate.title, - "relationship": classification, - }) - - # Apply confidence decay - if classification == "update": - doc = store.get(candidate.doc_id) - if doc: - new_confidence = max(CONFIDENCE_FLOOR, doc.confidence - UPDATE_DECAY) - store.db.execute( - "UPDATE documents SET confidence = ?, updated_at = datetime('now') WHERE id = ?", - (new_confidence, candidate.doc_id), - ) - store.db.commit() - store.add_relation(new_doc_id, candidate.doc_id, "supersedes", 0.8) - decayed += 1 - - elif classification == "contradiction": - doc = store.get(candidate.doc_id) - if doc: - new_confidence = max(CONFIDENCE_FLOOR, doc.confidence - CONTRADICTION_DECAY) - store.db.execute( - "UPDATE documents SET confidence = ?, updated_at = datetime('now') WHERE id = ?", - (new_confidence, candidate.doc_id), - ) - store.db.commit() - store.add_relation(new_doc_id, candidate.doc_id, "contradicts", 0.9) - decayed += 1 - - elif classification == "same": - store.add_relation(new_doc_id, candidate.doc_id, "related", 1.0) - - except Exception: + relationships.append({ + "doc_id": candidate.doc_id, + "title": candidate.title, + "relationship": classification, + "probs": result.b_implies_a.probs, + }) + + # Side effects — skipped under dry_run + if dry_run: + if classification in ("update", "contradiction"): + decayed += 1 # would-decay count continue - return {"decayed": decayed, "relationships": relationships} + if classification == "update": + doc = store.get(candidate.doc_id) + if doc: + new_confidence = max(CONFIDENCE_FLOOR, doc.confidence - UPDATE_DECAY) + store.db.execute( + "UPDATE documents SET confidence = ?, updated_at = datetime('now') WHERE id = ?", + (new_confidence, candidate.doc_id), + ) + store.db.commit() + store.add_relation(new_doc_id, candidate.doc_id, "supersedes", 0.8) + decayed += 1 + + elif classification == "contradiction": + doc = store.get(candidate.doc_id) + if doc: + new_confidence = max(CONFIDENCE_FLOOR, doc.confidence - CONTRADICTION_DECAY) + store.db.execute( + "UPDATE documents SET confidence = ?, updated_at = datetime('now') WHERE id = ?", + (new_confidence, candidate.doc_id), + ) + store.db.commit() + store.add_relation(new_doc_id, candidate.doc_id, "contradicts", 0.9) + decayed += 1 + + elif classification == "same": + store.add_relation(new_doc_id, candidate.doc_id, "related", 1.0) + + return { + "decayed": decayed, + "relationships": relationships, + "nli_unavailable": False, + "dry_run": dry_run, + } # ── Confidence Decay ──────────────────────────────────────────────────────── diff --git a/src/mnemon/nli.py b/src/mnemon/nli.py new file mode 100644 index 0000000..a3f9a37 --- /dev/null +++ b/src/mnemon/nli.py @@ -0,0 +1,286 @@ +"""NLI (Natural Language Inference) classifier for pair-wise memory +relationships — used by contradiction detection. + +Replaces the prior LLM-based classifier (``llm.generate``) for the +``check_contradictions`` path. Same operational shape as FastEmbed: +ONNX runtime + tokenizer, lazy-loaded singleton, pre-warm at lifespan +startup, no PyTorch dependency. All transitive deps (``onnxruntime``, +``tokenizers``, ``huggingface_hub``) ship with FastEmbed already, so +this module adds zero new requirements on the Fly image. + +Model: ``cross-encoder/nli-deberta-v3-xsmall`` (22M params, ~87 MB +INT8 quantized). Trained on MNLI / SNLI / FEVER. Outputs three +labels: ``contradiction``, ``entailment``, ``neutral``. + +Per Brian's 2026-05-21 "no LLM in mnemon" decision: this is the +embedded-ML primitive for contradiction detection. Public-release +operators get it out of the box (auto-downloads on first use OR +baked into the Fly Docker image). Composes with the same SOTA-for- +public-release-constraint reasoning that drove the embedding-based +exemplar scorer in ``scripts/build_standing_set.py``. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Any, NamedTuple + +import numpy as np + +logger = logging.getLogger(__name__) + +# Model identity. INT8 quantized variant is the operational default — +# ~87 MB, x86 AVX-512 optimized (Fly hosts are x86), ~10ms / pair on +# CPU inference. FP32 (``onnx/model.onnx``) is larger but available +# via the ``MNEMON_NLI_ONNX_VARIANT`` env override. +MODEL_REPO = "cross-encoder/nli-deberta-v3-xsmall" +ONNX_FILE_DEFAULT = "onnx/model_qint8_avx512.onnx" +TOKENIZER_FILE = "tokenizer.json" +CONFIG_FILE = "config.json" + +# Label canonical from model config; verified at load. +EXPECTED_LABELS = {"contradiction", "entailment", "neutral"} + + +class NLIResult(NamedTuple): + """Single-pair classification result. + + ``label`` is the argmax of the three probabilities. ``probs`` is a + dict keyed by canonical label so callers don't depend on the + model's internal index order. + """ + label: str + probs: dict[str, float] + + +_session: Any = None +_tokenizer: Any = None +_id2label: dict[int, str] | None = None +_init_lock: Any = None + + +def _model_dir() -> Path: + """Where downloaded NLI files live. Honors ``MNEMON_NLI_MODEL_DIR`` + so an operator can point at a baked-in location (e.g., the Fly + image's ``/app/.cache/huggingface``).""" + return Path(os.environ.get( + "MNEMON_NLI_MODEL_DIR", + Path.home() / ".cache" / "huggingface" / "hub", + )) + + +def _ensure_loaded() -> None: + """Lazy-load model + tokenizer (singleton). Auto-downloads from + HuggingFace on first use if not already cached. + + Raises ``NLIUnavailableError`` if the model can't be loaded — + surface to callers rather than silently degrading. + """ + global _session, _tokenizer, _id2label, _init_lock + + if _session is not None: + return + + if _init_lock is None: + import threading + _init_lock = threading.Lock() + + with _init_lock: + if _session is not None: + return + + try: + from huggingface_hub import hf_hub_download + except ImportError as e: + raise NLIUnavailableError( + f"huggingface_hub not installed — should ship with FastEmbed: {e}" + ) from e + + variant = os.environ.get("MNEMON_NLI_ONNX_VARIANT", ONNX_FILE_DEFAULT) + + try: + onnx_path = hf_hub_download(repo_id=MODEL_REPO, filename=variant) + tokenizer_path = hf_hub_download(repo_id=MODEL_REPO, filename=TOKENIZER_FILE) + config_path = hf_hub_download(repo_id=MODEL_REPO, filename=CONFIG_FILE) + except Exception as e: + raise NLIUnavailableError( + f"NLI model download failed ({MODEL_REPO}): {e}" + ) from e + + # Verify the model's label space matches our expectations. If + # someone overrides MODEL_REPO to a model with a different + # label set, we want a fast, named failure here — not a silent + # mis-classification downstream. + import json + with open(config_path) as f: + cfg = json.load(f) + raw_id2label = cfg.get("id2label", {}) + id2label = {int(k): v for k, v in raw_id2label.items()} + if set(id2label.values()) != EXPECTED_LABELS: + raise NLIUnavailableError( + f"unexpected label set in {MODEL_REPO}: {id2label.values()}; " + f"expected {EXPECTED_LABELS}" + ) + + try: + import onnxruntime as ort + from tokenizers import Tokenizer + except ImportError as e: + raise NLIUnavailableError( + f"onnxruntime or tokenizers missing — should ship with FastEmbed: {e}" + ) from e + + try: + _session = ort.InferenceSession(onnx_path) + _tokenizer = Tokenizer.from_file(tokenizer_path) + _id2label = id2label + logger.info( + "nli: loaded %s (%s) — labels=%s", + MODEL_REPO, variant, list(id2label.values()), + ) + except Exception as e: + raise NLIUnavailableError( + f"NLI model load failed: {e}" + ) from e + + +class NLIUnavailableError(RuntimeError): + """Raised when NLI inference can't run — typically a download / + load failure on first use. Callers in best-effort paths + (``check_contradictions``) catch + degrade with a clear message; + fail-loud per ``feedback_no_silent_fails``.""" + + +def classify_pair(premise: str, hypothesis: str) -> NLIResult: + """Classify a single premise / hypothesis pair. + + Returns ``NLIResult(label, probs)`` with the argmax label and a + canonical-keyed probability dict. + + Raises ``NLIUnavailableError`` on download / load failure. + """ + _ensure_loaded() + assert _session is not None and _tokenizer is not None and _id2label is not None + + enc = _tokenizer.encode(premise, hypothesis) + input_ids = np.array([enc.ids], dtype=np.int64) + attention_mask = np.array([enc.attention_mask], dtype=np.int64) + + inputs: dict[str, np.ndarray] = { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + # token_type_ids only required by some architectures (BERT-family); + # DeBERTa-v3-xsmall doesn't take it, but include defensively for + # the operator-override case where MODEL_REPO points at a BERT-style + # model. The session check makes this a no-op for DeBERTa. + input_names = {i.name for i in _session.get_inputs()} + if "token_type_ids" in input_names: + inputs["token_type_ids"] = np.array([enc.type_ids], dtype=np.int64) + + logits = _session.run(None, inputs)[0] + # Softmax for interpretable probabilities + exp = np.exp(logits[0] - np.max(logits[0])) + probs_arr = exp / exp.sum() + probs = {_id2label[i]: float(probs_arr[i]) for i in range(len(probs_arr))} + label = max(probs, key=probs.__getitem__) + return NLIResult(label=label, probs=probs) + + +class BidirectionalResult(NamedTuple): + """Result of running NLI in both directions on a pair (a, b). + + Disambiguates ``same`` from ``update`` for mnemon's + contradiction-detection taxonomy: + - both directions entail → ``same`` (semantic equivalence) + - one direction entails, other neutral → ``update`` + (the entailing direction "supersedes" the other) + - either direction is contradiction → ``contradiction`` + - both neutral → ``unrelated`` + """ + mnemon_label: str # "same" | "update" | "contradiction" | "unrelated" + a_implies_b: NLIResult + b_implies_a: NLIResult + + +# Mnemon's contradiction taxonomy maps from NLI by combining both +# directions of the pair. Bidirectional inference is cheap — same +# model, two inferences, ~10-20ms total on CPU INT8. +def classify_pair_bidirectional(a: str, b: str) -> BidirectionalResult: + """Classify the relationship between two memories in both directions. + + Conventions: ``a`` is treated as the "existing" memory, ``b`` as + the "new" one. ``b_implies_a == entailment`` while + ``a_implies_b != entailment`` means ``b`` is a stronger / + more-detailed restatement → ``update`` (the new supersedes the + old). ``a_implies_b == contradiction`` (either direction) means + direct conflict. + """ + a_to_b = classify_pair(a, b) + b_to_a = classify_pair(b, a) + + # Contradiction in either direction is contradiction + if a_to_b.label == "contradiction" or b_to_a.label == "contradiction": + mnemon = "contradiction" + elif a_to_b.label == "entailment" and b_to_a.label == "entailment": + # Both directions entail → semantic equivalence + mnemon = "same" + elif b_to_a.label == "entailment": + # New entails old (b → a) but not vice versa → new is stronger + # / more-detailed → update + mnemon = "update" + elif a_to_b.label == "entailment": + # Old entails new (a → b) but not vice versa → new is a + # weaker subset / less-detailed restatement → still "same" + # rather than "update"; the existing memory dominates + mnemon = "same" + else: + # Both neutral → unrelated + mnemon = "unrelated" + + return BidirectionalResult( + mnemon_label=mnemon, + a_implies_b=a_to_b, + b_implies_a=b_to_a, + ) + + +def is_available() -> bool: + """Check if NLI inference can run. Returns True iff the model is + already loaded OR the dependencies + network needed to download it + are present. Cheap probe — doesn't actually load if unloaded; + callers that need a guaranteed working model should call + ``_ensure_loaded()`` directly and catch ``NLIUnavailableError``. + """ + if _session is not None: + return True + try: + import onnxruntime # noqa: F401 + from huggingface_hub import hf_hub_download # noqa: F401 + from tokenizers import Tokenizer # noqa: F401 + return True + except ImportError: + return False + + +def prewarm() -> None: + """Pre-load the model at lifespan startup so the first + classification doesn't pay the cold-load cost. Mirrors FastEmbed's + pre-warm pattern in ``server_remote.py``. Safe to call multiple + times — idempotent via the singleton check in ``_ensure_loaded``. + + Swallowed errors: pre-warm is best-effort. Failures here just + mean the first real classification will see the cold-load cost + (and surface any persistent failure as ``NLIUnavailableError`` + there). + """ + try: + _ensure_loaded() + except NLIUnavailableError as e: + # Pre-warm is secondary observability — primary deliverable + # (the server) survives without it; first real call will + # surface the named exception. Acceptable swallow per + # feedback_no_silent_fails category (b). + logger.warning("nli: pre-warm failed: %s; will retry on first call", e) diff --git a/src/mnemon/server.py b/src/mnemon/server.py index a2833d9..304aebe 100644 --- a/src/mnemon/server.py +++ b/src/mnemon/server.py @@ -501,11 +501,18 @@ def profile_update(title: str, content: str) -> str: @mcp.tool() -def memory_check_contradictions(id: int) -> str: +def memory_check_contradictions(id: int, dry_run: bool = False) -> str: """Check a memory for contradictions against existing memories. - Uses vector similarity + LLM classification to find conflicts. - Automatically decays confidence of superseded or contradicting memories. + Uses vector-similarity gate (>= overlap threshold) then NLI + bidirectional classification (no LLM required) to find conflicts. + Automatically decays confidence of superseded or contradicting + memories — unless ``dry_run=True``, in which case mutations are + skipped and the response indicates what WOULD have changed. + + Returns a human-readable summary including the NLI classification + per pair. On NLI unavailability (model not loadable), returns a + clear "skipped" message rather than an opaque error. """ store = _get_store() doc = store.get(id) @@ -513,7 +520,17 @@ def memory_check_contradictions(id: int) -> str: return f"Memory #{id} not found." from .contradiction import check_contradictions - result = check_contradictions(store, doc.title, doc.content, id) + result = check_contradictions( + store, doc.title, doc.content, id, dry_run=dry_run, + ) + + if result.get("nli_unavailable"): + return ( + f'Contradiction check for #{id} skipped — NLI classifier ' + f'unavailable on this server (model load failed). Try again ' + f'after a server restart, or run locally with the NLI model ' + f'pre-cached. No vault state was modified.' + ) if not result["relationships"]: return f"No contradictions found for memory #{id}." @@ -523,10 +540,18 @@ def memory_check_contradictions(id: int) -> str: for r in result["relationships"] ] + if dry_run: + action = ( + f'\n\n[dry-run] {result["decayed"]} memories WOULD have had ' + f'their confidence decayed. No mutations applied.' + ) + else: + action = f'\n\n{result["decayed"]} memories had their confidence decayed.' + return ( f'Contradiction check for #{id} "{doc.title}":\n' + "\n".join(lines) - + f'\n\n{result["decayed"]} memories had their confidence decayed.' + + action ) diff --git a/src/mnemon/server_remote.py b/src/mnemon/server_remote.py index 5db1f88..2a1322c 100644 --- a/src/mnemon/server_remote.py +++ b/src/mnemon/server_remote.py @@ -76,6 +76,25 @@ def run_remote() -> None: file=sys.stderr, ) + # Eager NLI init for contradiction detection — non-fatal if it + # fails. ~87 MB INT8 ONNX model; load on first call would block + # the calling MCP tool for several seconds. Pre-loading here + # shifts that cost to server startup, identical pattern to + # embedder above. + try: + from .nli import prewarm as nli_prewarm + + print("Pre-loading NLI classifier...", file=sys.stderr) + nli_prewarm() + print("NLI classifier ready.", file=sys.stderr) + except Exception as e: # noqa: BLE001 + print( + f"WARN: failed to pre-load NLI classifier " + f"({type(e).__name__}: {e}); first memory_check_contradictions " + "will pay the load cost lazily", + file=sys.stderr, + ) + config = OAuthConfig.from_env() # Self-hosted Authorization Server. When MNEMON_AS_ENABLED=true, the diff --git a/tests/test_contradiction.py b/tests/test_contradiction.py index 07336f9..f62f760 100644 --- a/tests/test_contradiction.py +++ b/tests/test_contradiction.py @@ -1,4 +1,5 @@ -"""Tests for contradiction detection and confidence decay.""" +"""Tests for contradiction detection (NLI-based, 2026-05-22 rebuild) +and confidence decay.""" import math import os @@ -17,6 +18,7 @@ check_contradictions, apply_confidence_decay, ) +from mnemon.nli import BidirectionalResult, NLIResult, NLIUnavailableError @pytest.fixture @@ -39,12 +41,29 @@ def _mock_embedding(): return np.zeros(384, dtype=np.float32) +def _bidir(mnemon_label: str) -> BidirectionalResult: + """Build a BidirectionalResult stub with the given mnemon label. + Sub-result probabilities are placeholders — only the + ``mnemon_label`` field is used by ``check_contradictions``.""" + placeholder = NLIResult( + label="neutral", + probs={"contradiction": 0.1, "entailment": 0.1, "neutral": 0.8}, + ) + return BidirectionalResult( + mnemon_label=mnemon_label, + a_implies_b=placeholder, + b_implies_a=placeholder, + ) + + class TestCheckContradictions: def test_returns_empty_when_no_vectors(self, store): doc_id = store.save(title="Test", content="Some content") result = check_contradictions(store, "New title", "New content", doc_id) assert result["decayed"] == 0 assert result["relationships"] == [] + assert result["nli_unavailable"] is False + assert result["dry_run"] is False def test_classifies_update_and_decays(self, store): id1 = store.save(title="DB choice", content="We use PostgreSQL for storage") @@ -62,7 +81,7 @@ def test_classifies_update_and_decays(self, store): with patch.object(store, "search_vector", return_value=mock_results), \ patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ - patch("mnemon.llm.generate", return_value="update"): + patch("mnemon.nli.classify_pair_bidirectional", return_value=_bidir("update")): result = check_contradictions(store, "DB migration", "Migrating to MySQL", id2) assert result["decayed"] == 1 @@ -87,12 +106,17 @@ def test_classifies_contradiction_and_decays_more(self, store): with patch.object(store, "search_vector", return_value=mock_results), \ patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ - patch("mnemon.llm.generate", return_value="contradiction"): + patch("mnemon.nli.classify_pair_bidirectional", return_value=_bidir("contradiction")): result = check_contradictions(store, "Auth method v2", "Never use JWT", id2) assert result["decayed"] == 1 assert result["relationships"][0]["relationship"] == "contradiction" + doc1_after = store.get(id1) + # contradiction decay (0.25) > update decay (0.15) — confirm the steeper one applied + expected = max(CONFIDENCE_FLOOR, original_confidence - CONTRADICTION_DECAY) + assert abs(doc1_after.confidence - expected) < 1e-6 + def test_same_classification_adds_relation(self, store): id1 = store.save(title="Deploy step", content="Deploy via Lambda") id2 = store.save(title="Deploy step copy", content="We deploy using Lambda") @@ -107,7 +131,7 @@ def test_same_classification_adds_relation(self, store): with patch.object(store, "search_vector", return_value=mock_results), \ patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ - patch("mnemon.llm.generate", return_value="same"): + patch("mnemon.nli.classify_pair_bidirectional", return_value=_bidir("same")): result = check_contradictions(store, "Deploy step copy", "We deploy using Lambda", id2) assert result["decayed"] == 0 @@ -117,7 +141,7 @@ def test_same_classification_adds_relation(self, store): assert len(related) > 0 def test_skips_self_in_vector_results(self, store): - """Ensure a document doesn't conflict with itself.""" + """A document doesn't conflict with itself.""" id1 = store.save(title="Test", content="Test content") mock_results = [ @@ -136,7 +160,7 @@ def test_skips_self_in_vector_results(self, store): assert result["relationships"] == [] def test_filters_below_overlap_threshold(self, store): - """Memories with low vector similarity should be skipped.""" + """Memories with low vector similarity should be skipped before NLI.""" id1 = store.save(title="A", content="Apples") id2 = store.save(title="B", content="Bananas") @@ -148,14 +172,101 @@ def test_filters_below_overlap_threshold(self, store): ) ] + # Even if NLI would (wrongly) say "contradiction" on every call, + # below-threshold pairs never reach it. with patch.object(store, "search_vector", return_value=mock_results), \ - patch("mnemon.embedder.embed", return_value=_mock_embedding()): + patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ + patch("mnemon.nli.classify_pair_bidirectional", + return_value=_bidir("contradiction")) as nli_mock: result = check_contradictions(store, "B", "Bananas", id2) assert result["relationships"] == [] + assert nli_mock.call_count == 0 # cosine gate intercepted + + def test_unrelated_does_not_decay(self, store): + id1 = store.save(title="A", content="Apples") + id2 = store.save(title="B", content="Bananas") + doc1 = store.get(id1) + original_confidence = doc1.confidence + + mock_results = [ + SearchResult( + doc_id=id1, title="A", content="Apples", + content_type="note", memory_type="semantic", confidence=original_confidence, + created_at="2026-01-01", score=0.8, source="vector", + ) + ] + + with patch.object(store, "search_vector", return_value=mock_results), \ + patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ + patch("mnemon.nli.classify_pair_bidirectional", + return_value=_bidir("unrelated")): + result = check_contradictions(store, "B", "Bananas", id2) + + # Unrelated → relationship recorded but no decay + assert result["decayed"] == 0 + # Confidence on the older memory unchanged + doc1_after = store.get(id1) + assert abs(doc1_after.confidence - original_confidence) < 1e-6 + + def test_dry_run_skips_mutations(self, store): + id1 = store.save(title="DB choice", content="We use PostgreSQL") + id2 = store.save(title="DB migration", content="Migrating to MySQL") + doc1 = store.get(id1) + original_confidence = doc1.confidence + + mock_results = [ + SearchResult( + doc_id=id1, title="DB choice", content="We use PostgreSQL", + content_type="note", memory_type="semantic", confidence=original_confidence, + created_at="2026-01-01", score=0.85, source="vector", + ) + ] + + with patch.object(store, "search_vector", return_value=mock_results), \ + patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ + patch("mnemon.nli.classify_pair_bidirectional", return_value=_bidir("update")): + result = check_contradictions(store, "DB migration", "Migrating to MySQL", id2, + dry_run=True) + + # Reported as a would-have-decayed for operator visibility + assert result["decayed"] == 1 + assert result["dry_run"] is True + # But the actual confidence is unchanged + doc1_after = store.get(id1) + assert abs(doc1_after.confidence - original_confidence) < 1e-6 + # And no 'supersedes' relation was inserted + rels = store.db.execute( + "SELECT COUNT(*) AS c FROM relations WHERE relation_type = 'supersedes'" + ).fetchone() + assert rels["c"] == 0 + + def test_nli_unavailable_returns_clear_flag(self, store): + id1 = store.save(title="A", content="Content A") + id2 = store.save(title="B", content="Content B") + + mock_results = [ + SearchResult( + doc_id=id1, title="A", content="Content A", + content_type="note", memory_type="semantic", confidence=0.5, + created_at="2026-01-01", score=0.85, source="vector", + ) + ] + + with patch.object(store, "search_vector", return_value=mock_results), \ + patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ + patch("mnemon.nli.classify_pair_bidirectional", + side_effect=NLIUnavailableError("model load failed")): + result = check_contradictions(store, "B", "Content B", id2) + + assert result["nli_unavailable"] is True + assert result["decayed"] == 0 + # No mutations applied + rels = store.db.execute("SELECT COUNT(*) AS c FROM relations").fetchone() + assert rels["c"] == 0 def test_invalid_classification_skipped(self, store): - """LLM returning garbage should be skipped.""" + """NLI returning an out-of-taxonomy label should be skipped, not crash.""" id1 = store.save(title="A", content="Content A") id2 = store.save(title="B", content="Content B") @@ -169,7 +280,8 @@ def test_invalid_classification_skipped(self, store): with patch.object(store, "search_vector", return_value=mock_results), \ patch("mnemon.embedder.embed", return_value=_mock_embedding()), \ - patch("mnemon.llm.generate", return_value="i dont know"): + patch("mnemon.nli.classify_pair_bidirectional", + return_value=_bidir("garbage_label")): result = check_contradictions(store, "B", "Content B", id2) assert result["decayed"] == 0 @@ -232,17 +344,5 @@ def test_access_reinforcement_slows_decay(self, store): doc1 = store.get(id1) doc2 = store.get(id2) + # High-access memory should retain more confidence than low-access at same age assert doc2.confidence > doc1.confidence - - def test_confidence_floor(self, store): - doc_id = store.save(title="Ancient", content="Very old observation", content_type="handoff") - - store.db.execute( - "UPDATE documents SET updated_at = datetime('now', '-365 days') WHERE id = ?", - (doc_id,), - ) - store.db.commit() - - apply_confidence_decay(store) - doc = store.get(doc_id) - assert doc.confidence >= CONFIDENCE_FLOOR diff --git a/tests/test_nli.py b/tests/test_nli.py new file mode 100644 index 0000000..a6e6957 --- /dev/null +++ b/tests/test_nli.py @@ -0,0 +1,123 @@ +"""Tests for the NLI cross-encoder module — model layer + label mapping. + +Real inference is exercised by ``tests/fixtures/nli_real_inference_pairs.py`` +(skipped by default; opt-in via env var). These tests cover the mnemon- +side logic (bidirectional → taxonomy mapping, error handling) with the +classifier mocked so they run fast and offline. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from mnemon.nli import ( + BidirectionalResult, + NLIResult, + NLIUnavailableError, + classify_pair_bidirectional, + is_available, +) + + +def _result(label: str, **probs) -> NLIResult: + """Build an NLIResult with explicit probs (defaults sum to 1.0).""" + full = {"contradiction": 0.0, "entailment": 0.0, "neutral": 0.0} + full.update(probs) + full[label] = max(full[label], 1.0 - sum(v for k, v in full.items() if k != label)) + return NLIResult(label=label, probs=full) + + +class TestBidirectionalMapping: + """The mnemon taxonomy is derived from the pair of unidirectional + NLI classifications. These tests lock the mapping logic.""" + + def _patch(self, a_to_b: str, b_to_a: str): + """Return a context manager that stubs classify_pair to + produce a_to_b on the first call and b_to_a on the second.""" + results = iter([_result(a_to_b), _result(b_to_a)]) + return patch("mnemon.nli.classify_pair", + side_effect=lambda p, h: next(results)) + + def test_contradiction_in_either_direction_wins(self): + # a→b contradiction + with self._patch("contradiction", "neutral"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "contradiction" + # b→a contradiction + with self._patch("neutral", "contradiction"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "contradiction" + # both directions contradict + with self._patch("contradiction", "contradiction"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "contradiction" + + def test_both_entail_is_same(self): + with self._patch("entailment", "entailment"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "same" + + def test_b_entails_a_only_is_update(self): + """New (b) entails old (a) but not vice versa → new is the + stronger / more-detailed statement → 'update'.""" + with self._patch("neutral", "entailment"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "update" + + def test_a_entails_b_only_is_same_not_update(self): + """Old (a) entails new (b) but not vice versa → new is a + weaker subset → existing memory dominates → 'same', not + 'update'. Composes with the salience-tier 'don't crowd with + weaker restatements' invariant.""" + with self._patch("entailment", "neutral"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "same" + + def test_both_neutral_is_unrelated(self): + with self._patch("neutral", "neutral"): + r = classify_pair_bidirectional("a", "b") + assert r.mnemon_label == "unrelated" + + def test_result_carries_subdirections(self): + """The BidirectionalResult preserves both NLI sub-results + for caller-side inspection (logging, observability).""" + with self._patch("contradiction", "entailment"): + r = classify_pair_bidirectional("a", "b") + assert r.a_implies_b.label == "contradiction" + assert r.b_implies_a.label == "entailment" + + +class TestAvailability: + def test_is_available_when_deps_present(self): + # In a normal test env all FastEmbed transitive deps are present + assert is_available() is True + + def test_is_available_false_when_onnxruntime_missing(self): + # Simulate missing onnxruntime + import builtins + original_import = builtins.__import__ + + def raise_import(name, *args, **kwargs): + if name == "onnxruntime": + raise ImportError("simulated missing") + return original_import(name, *args, **kwargs) + + # Reset the session singleton so is_available actually probes + import mnemon.nli + original_session = mnemon.nli._session + mnemon.nli._session = None + try: + with patch("builtins.__import__", side_effect=raise_import): + assert is_available() is False + finally: + mnemon.nli._session = original_session + + +class TestErrorSurfacing: + def test_unavailable_error_has_descriptive_message(self): + e = NLIUnavailableError("model load failed: connection refused") + # The message survives string conversion (needed for the MCP + # tool's clear-error path). + assert "model load failed" in str(e)