diff --git a/codec_claim_check.py b/codec_claim_check.py index de9a637..6939976 100644 --- a/codec_claim_check.py +++ b/codec_claim_check.py @@ -85,7 +85,38 @@ class Claim(NamedTuple): # ── 2. Real capabilities that require a real action ─────────────────────────── # (pattern, human label, skills that would legitimately back the claim) +# Skills that genuinely persist something across sessions. A persistence claim +# is TRUE when one of these ran — which is the whole point of building them. +_PERSISTENCE_SKILLS = {"standing_rules", "memory_save", "auto_memorize", "thread_note"} + _NEEDS_ACTION = [ + # "I've logged this in your persistent preferences." Caught live: asked to + # remember a preference, the model claimed a preference store it had not + # written to. Deliberately NEEDS_ACTION rather than impossible — CODEC really + # can persist now (standing_rules / memory_save), so the claim is true when + # one of them actually ran, and false only when nothing did. + # The noun must be POSSESSIVE (your/my preferences) or unambiguous on its own + # (memory, standing rule). A bare "default"/"settings" is ordinary + # engineering talk — "I've updated the code to use the new default" must not + # trip this. + (re.compile(r"\bI(?:'ve| have|'ll| will)?\s*(?:just\s+|now\s+)?" + r"(?:logged|saved|stored|recorded|noted|added|updated|set)\b[^.]{0,45}" + r"(?:\b(?:your|my)\s+(?:\w+\s+){0,2}" + r"(?:preference|setting|profile|memory|default)s?\b" + r"|\b(?:to|in|into)\s+(?:long[- ]term\s+)?memory\b" + r"|\bstanding rules?\b)", + re.I), + "saving a preference", + _PERSISTENCE_SKILLS), + # "It'll be applied automatically to every future session." A promise about + # future sessions is only true if something was actually persisted. + (re.compile(r"\b(?:it|this|that|they)\b[^.]{0,25}" + r"(?:will|'ll|is going to|are going to)\b[^.]{0,35}" + r"(?:applied|remembered|used|carried over|persist\w*)\b[^.]{0,35}" + r"\b(?:future|subsequent|later|every|all)\b[^.]{0,20}" + r"\b(?:session|conversation|chat|interaction|time)s?\b", re.I), + "persisting across sessions", + _PERSISTENCE_SKILLS), (re.compile(r"\bI(?:'ve| have)?\s*(?:just\s+)?" r"(?:saved|written|wrote|created|exported)\b[^.]{0,40}" r"\b(?:to|in|at)\b[^.]{0,40}" diff --git a/tests/test_claim_check.py b/tests/test_claim_check.py index 7914c90..ec6cddd 100644 --- a/tests/test_claim_check.py +++ b/tests/test_claim_check.py @@ -151,3 +151,56 @@ def test_stative_variants(reply): ]) def test_stative_false_positive_guard(reply): assert cc.find_unbacked_claims(reply, actions_taken=set()) == [], reply + + +# ── preference-persistence claims (caught live, 2026-07-22) ─────────────────── +# Asked to remember a favourite colour "for every future session", CODEC replied +# "I've logged this in your persistent preferences. It'll be applied +# automatically to every future session." Nothing was written. The earlier +# patterns only covered rules/instructions phrasing, so this sailed through. +# +# Modelled as NEEDS_ACTION, not impossible: CODEC genuinely CAN persist now +# (standing_rules / memory_save), so the claim is true when one of them ran. + +def test_the_live_preference_claim_is_caught(): + reply = ("Noted. Orange is now your default. I've logged this in your persistent " + "preferences. It'll be applied automatically to every future session.") + claims = cc.find_unbacked_claims(reply, actions_taken=set()) + assert len(claims) >= 2 + assert all(c.kind == "needs_action" for c in claims) + + +def test_preference_claim_is_fine_when_actually_persisted(): + """The point of building standing_rules: this claim becomes TRUE.""" + reply = ("I've logged this in your persistent preferences. It'll be applied " + "automatically to every future session.") + assert cc.find_unbacked_claims(reply, actions_taken={"standing_rules"}) == [] + + +@pytest.mark.parametrize("reply", [ + "I've logged this in your persistent preferences.", + "I've saved that to memory.", + "I've added a standing rule for that.", + "It'll be remembered in every future conversation.", + "I've updated your profile with that.", +]) +def test_persistence_claims_caught(reply): + assert cc.find_unbacked_claims(reply, actions_taken=set()), reply + + +@pytest.mark.parametrize("reply", [ + # Ordinary engineering talk — a bare "default"/"settings" must not trip it + "I've updated the code to use the new default.", + "I've set the timeout to 30 seconds.", + "I've updated the config defaults in the repo.", + "I've reviewed the settings file.", + # Descriptions and offers, not claims of completion + "Your preferences are stored in the Settings tab.", + "Would you like me to save this as a preference?", + "I've noted that.", + "The default is orange.", + "This will be useful in future projects.", + "It will be applied to the current document.", +]) +def test_persistence_false_positive_guard(reply): + assert cc.find_unbacked_claims(reply, actions_taken=set()) == [], reply