From 0d94433b506f4b84fe6b7843f4b796bfec2f5351 Mon Sep 17 00:00:00 2001 From: Mickael Farina Date: Fri, 24 Jul 2026 18:33:07 +0200 Subject: [PATCH] =?UTF-8?q?feat(standing=5Frules):=20inject=5Fcount=20+=20?= =?UTF-8?q?last=5Finjected=20=E2=80=94=20named=20for=20what=20it=20measure?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deliberately NOT fired_count. A standing rule's entire lifecycle is prompt_block() -> string -> appended to sys_prompt -> sent to the LLM. Nothing downstream reports whether the model consulted the rule, obeyed it, or ignored it. Injection is the only observable event in the pipeline. Measuring real influence would need ablation (run the turn with and without the rule and diff — doubles cost, and non-determinism makes one diff weak evidence) or asking the model which rules it used, which is precisely the unverifiable self-report codec_claim_check exists to refuse. A field called fired_count would be a fabricated metric wearing an authoritative name, in the module whose whole job is refusing that. What the number is good for: inject_count == 0 PROVES dead weight — the rule never even reached the model. A high count proves nothing about usefulness, only that you're paying for the rule on every message. The listing says exactly this rather than implying the model used anything. record=True is passed ONLY by the live chat path; listing, preview and tests default to False so the metric can't measure itself. Migration is additive: existing rules keep id/text/added and gain inject_count 0 / last_injected null. The store is never reset. Co-Authored-By: Claude Opus 4.8 --- codec_standing_rules.py | 76 +++++++++++++++++++++++++++++++++--- routes/chat.py | 3 +- skills/.manifest.json | 2 +- skills/standing_rules.py | 19 ++++++++- tests/test_standing_rules.py | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 9 deletions(-) diff --git a/codec_standing_rules.py b/codec_standing_rules.py index 6400410..9d5e6dc 100644 --- a/codec_standing_rules.py +++ b/codec_standing_rules.py @@ -62,7 +62,11 @@ def _save(data: Dict) -> None: def list_rules() -> List[Dict]: - return load().get("rules", []) + """Rules with telemetry fields guaranteed present (older stores are + backfilled in-memory; the on-disk backfill happens on the next write).""" + data = load() + _migrate(data) + return data.get("rules", []) def add_rule(text: str) -> Dict: @@ -85,7 +89,10 @@ def add_rule(text: str) -> Dict: f"you never invoke are noise you pay for on every message."} rule = {"id": f"r{len(rules) + 1}_{int(datetime.now(timezone.utc).timestamp())}", "text": text, - "added": datetime.now(timezone.utc).isoformat(timespec="seconds")} + "added": datetime.now(timezone.utc).isoformat(timespec="seconds"), + # Telemetry — see prompt_block() for exactly what this counts. + "inject_count": 0, + "last_injected": None} rules.append(rule) _save(data) return {"ok": True, "rule": rule, @@ -121,9 +128,68 @@ def clear_rules() -> Dict: return {"ok": True, "message": f"Cleared {n} standing rule(s)."} -def prompt_block() -> str: - """The block appended to the chat system prompt. "" when there are none.""" - rules = list_rules() +def _migrate(data: Dict) -> bool: + """Backfill telemetry fields on rules written before they existed. + + Additive only — an existing rule keeps its id, text and added date, and + starts at inject_count 0 / last_injected None. Never resets the store. + Returns True when something changed (so the caller can persist). + """ + changed = False + for r in data.get("rules", []): + if "inject_count" not in r: + r["inject_count"] = 0 + changed = True + if "last_injected" not in r: + r["last_injected"] = None + changed = True + return changed + + +def prompt_block(record: bool = False) -> str: + """The block appended to the chat system prompt. "" when there are none. + + `record=True` counts this as an INJECTION — see the honesty note below. + Only the live chat path passes it; listing, tests and previews must not, + or the count would measure itself. + + WHAT inject_count MEANS, AND DOES NOT + ------------------------------------- + It counts how many times a rule was placed into the system prompt. That is + the ONLY thing this pipeline can observe. A standing rule's entire lifecycle + is: prompt_block() -> string -> appended to sys_prompt -> sent to the LLM. + Nothing downstream reports whether the model consulted the rule, obeyed it, + or ignored it. + + So this is deliberately NOT called fired_count. Measuring influence would + need ablation (run the turn with and without the rule and diff — doubles + cost, and non-determinism makes a single diff weak evidence) or asking the + model which rules it used — which is exactly the unverifiable self-report + this codebase exists to refuse. A field named fired_count would be a + fabricated metric wearing an authoritative name. + + Practical consequence: inject_count == 0 proves a rule is dead weight + (it never even reached the model). A high inject_count proves nothing about + usefulness — only that the rule is being paid for on every message. + """ + if not record: + rules = list_rules() + else: + # Read-modify-write under the same lock the writers use, so a concurrent + # add/remove can't lose the increment. + data = load() + _migrate(data) + rules = data.get("rules", []) + if rules: + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + for r in rules: + r["inject_count"] = int(r.get("inject_count") or 0) + 1 + r["last_injected"] = now + try: + _save(data) + except Exception as e: # never break a reply over telemetry + log.warning("standing rules: inject-count write failed: %s", e) + if not rules: return "" lines = ["STANDING RULES — the user set these; they apply to every reply:"] diff --git a/routes/chat.py b/routes/chat.py index 7a6210a..ae66130 100644 --- a/routes/chat.py +++ b/routes/chat.py @@ -722,7 +722,8 @@ def _build_chat_system_prompt(config: dict, budget, has_attachment: bool, # or safety framing. This is what makes "saved" an honest answer. try: import codec_standing_rules - _sr = codec_standing_rules.prompt_block() + # record=True: this is the live turn, so it counts as an injection. + _sr = codec_standing_rules.prompt_block(record=True) if _sr: sys_prompt += "\n\n" + _sr except Exception as e: diff --git a/skills/.manifest.json b/skills/.manifest.json index bfdff7f..300d35d 100644 --- a/skills/.manifest.json +++ b/skills/.manifest.json @@ -79,7 +79,7 @@ "self_improve.py": "74c8b9cb0ce81f8600d35e72b73e543bdeccb3c8e5a4cdc79ddb4df5282c1846", "shift_report.py": "2c4bc669dec391f18329e0fe7b955c6b911322404e7e627e102776c22b448da2", "skill_forge.py": "07ea8fb2d026eca050154e5e60ea1968e77bc2d7786958d60f6e1a6d50f07182", - "standing_rules.py": "42bcea7ba808b84daf5bb7f242ea908a1da4eb20d26d6cf0e4751118d89721a6", + "standing_rules.py": "9f7f4a2af7cf8fbe11aaa11440c61eeab299e28da0e3bb0d7a52703787e22ee1", "stuck.py": "dfddfe4dfa1a9d5c017f53e53033a7eb33114a517cd6fa937d9d44e65083f1c9", "system_info.py": "4cc32e0ad9cb81f34309734ef3388f7cde63407de9b23bb0cd589571a4c43ecb", "terminal.py": "1fe6c1cd1241ea6d328401f1c0fb4c362482e3ec2a61a4f7592b2d4822cd51c1", diff --git a/skills/standing_rules.py b/skills/standing_rules.py index d82837d..5a4e07b 100644 --- a/skills/standing_rules.py +++ b/skills/standing_rules.py @@ -43,10 +43,25 @@ def _render(rules): if not rules: return ("You have no standing rules yet. Add one with: " "\"add a standing rule: \".") - lines = [f"You have {len(rules)} standing rule(s), applied to every reply:"] + lines = [f"You have {len(rules)} standing rule(s), added to every reply:"] + never = [] for i, r in enumerate(rules, 1): + n = int(r.get("inject_count") or 0) + last = r.get("last_injected") + when = f", last {str(last)[:10]}" if last else "" lines.append(f" {i}. {r.get('text', '')}") - lines.append("\nRemove one with: \"remove standing rule 2\".") + lines.append(f" sent to the model {n}x{when}" if n else + " never sent to the model yet") + if not n: + never.append(i) + lines.append("") + lines.append("\"sent to the model\" counts INJECTION into the prompt — not whether " + "the model actually used the rule. Nothing in the pipeline can see that, " + "so it isn't claimed.") + if never: + lines.append(f"Rules {', '.join(f'#{i}' for i in never)} have never even reached " + f"the model — those are safe to delete.") + lines.append("Remove one with: \"remove standing rule 2\".") return "\n".join(lines) diff --git a/tests/test_standing_rules.py b/tests/test_standing_rules.py index 43353f0..64e19f9 100644 --- a/tests/test_standing_rules.py +++ b/tests/test_standing_rules.py @@ -119,3 +119,71 @@ def test_skill_remove(monkeypatch, tmp_path): s.run("add a standing rule: be brief") assert "Removed" in s.run("remove standing rule 1") assert "no standing rules" in s.run("list standing rules").lower() + + +# ── inject_count telemetry ──────────────────────────────────────────────────── +# Named inject_count, NOT fired_count, deliberately. A standing rule's whole +# lifecycle is prompt_block() -> string -> sys_prompt -> LLM. Nothing downstream +# reports whether the model used the rule, so "fired" would be a fabricated +# metric wearing an authoritative name — in the module that exists to refuse +# exactly that. + +def test_new_rule_starts_at_zero(sr): + rule = sr.add_rule("be brief")["rule"] + assert rule["inject_count"] == 0 + assert rule["last_injected"] is None + + +def test_injection_increments_and_stamps(sr): + sr.add_rule("be brief") + sr.prompt_block(record=True) + r = sr.list_rules()[0] + assert r["inject_count"] == 1 + assert r["last_injected"] is not None + + +def test_preview_does_not_count_itself(sr): + """Listing/preview must never inflate the metric it reports.""" + sr.add_rule("be brief") + for _ in range(5): + sr.prompt_block() # record defaults to False + sr.list_rules() + assert sr.list_rules()[0]["inject_count"] == 0 + + +def test_count_survives_a_reload(sr): + """The counter is on disk, not in memory — a restart must not reset it.""" + sr.add_rule("be brief") + sr.prompt_block(record=True) + sr.prompt_block(record=True) + fresh = sr.load() # re-read from disk, no cached state + assert fresh["rules"][0]["inject_count"] == 2 + + +def test_migration_backfills_without_resetting(sr): + """An old store keeps its data and gains the fields at zero.""" + import json + sr.RULES_PATH.write_text(json.dumps({"schema": 1, "rules": [ + {"id": "old1", "text": "keep me", "added": "2026-07-01T10:00:00+00:00"}]})) + r = sr.list_rules()[0] + assert r["text"] == "keep me", "existing rule text must survive migration" + assert r["added"] == "2026-07-01T10:00:00+00:00", "added date must survive" + assert r["inject_count"] == 0 and r["last_injected"] is None + + +def test_no_rules_means_no_write_churn(sr): + """An empty store must not be rewritten on every turn.""" + sr.prompt_block(record=True) + assert not sr.RULES_PATH.exists() or sr.load()["rules"] == [] + + +def test_listing_labels_injection_honestly(monkeypatch, tmp_path): + """The user-facing text must not imply the model 'used' the rule.""" + s = _skill(monkeypatch, tmp_path) + s.run("add a standing rule: be brief") + import codec_standing_rules as mod + mod.prompt_block(record=True) + out = s.run("show my standing rules") + assert "sent to the model" in out + assert "counts INJECTION" in out + assert "fired" not in out.lower(), "must not claim the rule 'fired'"