diff --git a/codec_claim_check.py b/codec_claim_check.py index 6939976..eee3384 100644 --- a/codec_claim_check.py +++ b/codec_claim_check.py @@ -143,15 +143,57 @@ def _sentences(text: str) -> Iterable[str]: yield part -def find_unbacked_claims(reply: str, actions_taken: Optional[Set[str]] = None) -> List[Claim]: +# ── The user ASKED for persistence ──────────────────────────────────────────── +# Matching the reply is whack-a-mole: the model has unlimited ways to say "I +# saved it" — "I have ingested…", "the rules are active", "I've logged this in +# your preferences", "Memorized." Each new phrasing needed a new pattern. +# +# The REQUEST side is small and stable. "Remember this for every future session" +# is a persistence request however the model answers it. So: if the user asked +# to persist and nothing persisted, the turn is unbacked — regardless of wording. +# This can't be evaded by rephrasing, which the reply patterns provably could. +_PERSIST_REQUEST = re.compile( + r"\b(?:" + r"remember\s+(?:this|that|it|my|me)\b[^.?!]{0,60}\b(?:future|always|from now|forever|every (?:session|time|chat))" + r"|from now on\b[^.?!]{0,50}\b(?:remember|always|use|call|treat|answer|reply)" + r"|(?:for|in)\s+(?:all|every)\s+(?:future\s+)?(?:session|conversation|chat|interaction)s?" + # "always remember" is unambiguously cross-session. Bare "always use X" + # is not — it usually means "for this conversation", so it stays out. + r"|always\s+remember\b" + r"|(?:save|store|keep)\s+(?:this|that|it)\b[^.?!]{0,40}\b(?:permanently|forever|for good|as a (?:rule|preference))" + r"|add\s+(?:a\s+)?standing\s+rule" + r")", re.I) + +# Reply already admits it can't — don't pile a correction on an honest answer. +_ALREADY_HONEST = re.compile( + r"\b(?:I\s+(?:can(?:no|')t|am unable to|do(?:n't| not) have|have no)\b" + r"|no mechanism|not able to (?:persist|remember|save)" + r"|won't (?:persist|survive|carry))", re.I) + + +def find_unbacked_claims(reply: str, actions_taken: Optional[Set[str]] = None, + user_request: Optional[str] = None) -> List[Claim]: """Claims in `reply` that nothing in this turn actually backs. `actions_taken` is the set of skill names that ran during the turn. An empty set means the turn was pure text generation, so every action claim is unbacked. + + `user_request` is the message being answered. When the user explicitly asked + for something to persist and nothing did, the turn is flagged no matter how + the reply is worded — the phrasing-independent half of the check. """ if not reply: return [] + + done_now = {a.lower() for a in (actions_taken or set())} + if (user_request + and _PERSIST_REQUEST.search(user_request) + and not (done_now & _PERSISTENCE_SKILLS) + and not _ALREADY_HONEST.search(reply)): + return [Claim("impossible", + "You asked for this to persist across sessions.", + "standing instructions")] done = {a.lower() for a in (actions_taken or set())} found: List[Claim] = [] diff --git a/routes/chat.py b/routes/chat.py index 60231e9..bf72d11 100644 --- a/routes/chat.py +++ b/routes/chat.py @@ -1259,7 +1259,8 @@ def _resolve_skill_tag(raw_tag): try: import codec_claim_check _unbacked = codec_claim_check.find_unbacked_claims( - "".join(_visible), actions_taken=_stream_actions) + "".join(_visible), actions_taken=_stream_actions, + user_request=last_user_text) if _unbacked: _note = codec_claim_check.correction_note(_unbacked) yield f"data: {json.dumps({'token': _note})}\n\n" @@ -1342,7 +1343,8 @@ def _resolve_skill_tag(raw_tag): try: import codec_claim_check _unbacked = codec_claim_check.find_unbacked_claims( - answer, actions_taken=_turn_actions) + answer, actions_taken=_turn_actions, + user_request=last_user_text) if _unbacked: answer += codec_claim_check.correction_note(_unbacked) log_event( diff --git a/tests/test_claim_check.py b/tests/test_claim_check.py index ec6cddd..7713e98 100644 --- a/tests/test_claim_check.py +++ b/tests/test_claim_check.py @@ -204,3 +204,63 @@ def test_persistence_claims_caught(reply): ]) def test_persistence_false_positive_guard(reply): assert cc.find_unbacked_claims(reply, actions_taken=set()) == [], reply + + +# ── request-intent detection (the phrasing-independent half) ────────────────── +# Matching the REPLY is whack-a-mole. Across three live tests the model produced +# "I have ingested the instruction set", "The instruction set is active", +# "I've logged this in your persistent preferences", and simply "Memorized." — +# each needing a new pattern. The REQUEST side is small and stable, so a +# persistence ask with nothing persisted is flagged however the reply is worded. + +_PERSIST_REQ = "From now on remember my favourite colour is orange, for every future session." + + +@pytest.mark.parametrize("reply", [ + "Memorized. Orange is the brand color for future sessions.", + "Noted. Orange is now your default.", + "Confirmed. I have ingested that.", + "Got it!", + "Understood.", + "", # even an empty-ish affirmation + "Sure thing 🧡", +]) +def test_persistence_request_flagged_whatever_the_reply(reply): + if not reply: + pytest.skip("empty reply short-circuits earlier by design") + assert cc.find_unbacked_claims(reply, actions_taken=set(), + user_request=_PERSIST_REQ), reply + + +def test_not_flagged_when_actually_persisted(): + assert cc.find_unbacked_claims("Saved as standing rule 1.", + actions_taken={"standing_rules"}, + user_request=_PERSIST_REQ) == [] + + +def test_not_flagged_when_reply_is_already_honest(): + """Never pile a correction onto an answer that already says it can't.""" + for honest in ["I can't persist that across sessions.", + "I don't have cross-session memory.", + "There's no mechanism for that — it won't survive a restart."]: + assert cc.find_unbacked_claims(honest, actions_taken=set(), + user_request=_PERSIST_REQ) == [], honest + + +@pytest.mark.parametrize("request_text", [ + "What's the weather?", + "Remember to buy milk", # a to-do, not cross-session persistence + "Can you always use metric?", # ambiguous — usually means this chat + "always use tabs not spaces", + "Summarise this document", + "What did I do 20 minutes ago?", +]) +def test_ordinary_requests_are_not_persistence_asks(request_text): + assert cc.find_unbacked_claims("Sure, done.", actions_taken=set(), + user_request=request_text) == [], request_text + + +def test_no_user_request_falls_back_to_reply_patterns(): + """Callers that don't pass the request still get the reply-side checks.""" + assert cc.find_unbacked_claims("I have ingested the instruction set.", + actions_taken=set())