diff --git a/codec_standing_rules.py b/codec_standing_rules.py index 6400410..9d5e6dc 100644 --- a/codec_standing_rules.py +++ b/codec_standing_rules.py @@ -62,7 +62,11 @@ def _save(data: Dict) -> None: def list_rules() -> List[Dict]: - return load().get("rules", []) + """Rules with telemetry fields guaranteed present (older stores are + backfilled in-memory; the on-disk backfill happens on the next write).""" + data = load() + _migrate(data) + return data.get("rules", []) def add_rule(text: str) -> Dict: @@ -85,7 +89,10 @@ def add_rule(text: str) -> Dict: f"you never invoke are noise you pay for on every message."} rule = {"id": f"r{len(rules) + 1}_{int(datetime.now(timezone.utc).timestamp())}", "text": text, - "added": datetime.now(timezone.utc).isoformat(timespec="seconds")} + "added": datetime.now(timezone.utc).isoformat(timespec="seconds"), + # Telemetry — see prompt_block() for exactly what this counts. + "inject_count": 0, + "last_injected": None} rules.append(rule) _save(data) return {"ok": True, "rule": rule, @@ -121,9 +128,68 @@ def clear_rules() -> Dict: return {"ok": True, "message": f"Cleared {n} standing rule(s)."} -def prompt_block() -> str: - """The block appended to the chat system prompt. "" when there are none.""" - rules = list_rules() +def _migrate(data: Dict) -> bool: + """Backfill telemetry fields on rules written before they existed. + + Additive only — an existing rule keeps its id, text and added date, and + starts at inject_count 0 / last_injected None. Never resets the store. + Returns True when something changed (so the caller can persist). + """ + changed = False + for r in data.get("rules", []): + if "inject_count" not in r: + r["inject_count"] = 0 + changed = True + if "last_injected" not in r: + r["last_injected"] = None + changed = True + return changed + + +def prompt_block(record: bool = False) -> str: + """The block appended to the chat system prompt. "" when there are none. + + `record=True` counts this as an INJECTION — see the honesty note below. + Only the live chat path passes it; listing, tests and previews must not, + or the count would measure itself. + + WHAT inject_count MEANS, AND DOES NOT + ------------------------------------- + It counts how many times a rule was placed into the system prompt. That is + the ONLY thing this pipeline can observe. A standing rule's entire lifecycle + is: prompt_block() -> string -> appended to sys_prompt -> sent to the LLM. + Nothing downstream reports whether the model consulted the rule, obeyed it, + or ignored it. + + So this is deliberately NOT called fired_count. Measuring influence would + need ablation (run the turn with and without the rule and diff — doubles + cost, and non-determinism makes a single diff weak evidence) or asking the + model which rules it used — which is exactly the unverifiable self-report + this codebase exists to refuse. A field named fired_count would be a + fabricated metric wearing an authoritative name. + + Practical consequence: inject_count == 0 proves a rule is dead weight + (it never even reached the model). A high inject_count proves nothing about + usefulness — only that the rule is being paid for on every message. + """ + if not record: + rules = list_rules() + else: + # Read-modify-write under the same lock the writers use, so a concurrent + # add/remove can't lose the increment. + data = load() + _migrate(data) + rules = data.get("rules", []) + if rules: + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + for r in rules: + r["inject_count"] = int(r.get("inject_count") or 0) + 1 + r["last_injected"] = now + try: + _save(data) + except Exception as e: # never break a reply over telemetry + log.warning("standing rules: inject-count write failed: %s", e) + if not rules: return "" lines = ["STANDING RULES — the user set these; they apply to every reply:"] diff --git a/routes/chat.py b/routes/chat.py index 7a6210a..ae66130 100644 --- a/routes/chat.py +++ b/routes/chat.py @@ -722,7 +722,8 @@ def _build_chat_system_prompt(config: dict, budget, has_attachment: bool, # or safety framing. This is what makes "saved" an honest answer. try: import codec_standing_rules - _sr = codec_standing_rules.prompt_block() + # record=True: this is the live turn, so it counts as an injection. + _sr = codec_standing_rules.prompt_block(record=True) if _sr: sys_prompt += "\n\n" + _sr except Exception as e: diff --git a/skills/.manifest.json b/skills/.manifest.json index bfdff7f..300d35d 100644 --- a/skills/.manifest.json +++ b/skills/.manifest.json @@ -79,7 +79,7 @@ "self_improve.py": "74c8b9cb0ce81f8600d35e72b73e543bdeccb3c8e5a4cdc79ddb4df5282c1846", "shift_report.py": "2c4bc669dec391f18329e0fe7b955c6b911322404e7e627e102776c22b448da2", "skill_forge.py": "07ea8fb2d026eca050154e5e60ea1968e77bc2d7786958d60f6e1a6d50f07182", - "standing_rules.py": "42bcea7ba808b84daf5bb7f242ea908a1da4eb20d26d6cf0e4751118d89721a6", + "standing_rules.py": "9f7f4a2af7cf8fbe11aaa11440c61eeab299e28da0e3bb0d7a52703787e22ee1", "stuck.py": "dfddfe4dfa1a9d5c017f53e53033a7eb33114a517cd6fa937d9d44e65083f1c9", "system_info.py": "4cc32e0ad9cb81f34309734ef3388f7cde63407de9b23bb0cd589571a4c43ecb", "terminal.py": "1fe6c1cd1241ea6d328401f1c0fb4c362482e3ec2a61a4f7592b2d4822cd51c1", diff --git a/skills/standing_rules.py b/skills/standing_rules.py index d82837d..5a4e07b 100644 --- a/skills/standing_rules.py +++ b/skills/standing_rules.py @@ -43,10 +43,25 @@ def _render(rules): if not rules: return ("You have no standing rules yet. Add one with: " "\"add a standing rule: \".") - lines = [f"You have {len(rules)} standing rule(s), applied to every reply:"] + lines = [f"You have {len(rules)} standing rule(s), added to every reply:"] + never = [] for i, r in enumerate(rules, 1): + n = int(r.get("inject_count") or 0) + last = r.get("last_injected") + when = f", last {str(last)[:10]}" if last else "" lines.append(f" {i}. {r.get('text', '')}") - lines.append("\nRemove one with: \"remove standing rule 2\".") + lines.append(f" sent to the model {n}x{when}" if n else + " never sent to the model yet") + if not n: + never.append(i) + lines.append("") + lines.append("\"sent to the model\" counts INJECTION into the prompt — not whether " + "the model actually used the rule. Nothing in the pipeline can see that, " + "so it isn't claimed.") + if never: + lines.append(f"Rules {', '.join(f'#{i}' for i in never)} have never even reached " + f"the model — those are safe to delete.") + lines.append("Remove one with: \"remove standing rule 2\".") return "\n".join(lines) diff --git a/tests/test_standing_rules.py b/tests/test_standing_rules.py index 43353f0..64e19f9 100644 --- a/tests/test_standing_rules.py +++ b/tests/test_standing_rules.py @@ -119,3 +119,71 @@ def test_skill_remove(monkeypatch, tmp_path): s.run("add a standing rule: be brief") assert "Removed" in s.run("remove standing rule 1") assert "no standing rules" in s.run("list standing rules").lower() + + +# ── inject_count telemetry ──────────────────────────────────────────────────── +# Named inject_count, NOT fired_count, deliberately. A standing rule's whole +# lifecycle is prompt_block() -> string -> sys_prompt -> LLM. Nothing downstream +# reports whether the model used the rule, so "fired" would be a fabricated +# metric wearing an authoritative name — in the module that exists to refuse +# exactly that. + +def test_new_rule_starts_at_zero(sr): + rule = sr.add_rule("be brief")["rule"] + assert rule["inject_count"] == 0 + assert rule["last_injected"] is None + + +def test_injection_increments_and_stamps(sr): + sr.add_rule("be brief") + sr.prompt_block(record=True) + r = sr.list_rules()[0] + assert r["inject_count"] == 1 + assert r["last_injected"] is not None + + +def test_preview_does_not_count_itself(sr): + """Listing/preview must never inflate the metric it reports.""" + sr.add_rule("be brief") + for _ in range(5): + sr.prompt_block() # record defaults to False + sr.list_rules() + assert sr.list_rules()[0]["inject_count"] == 0 + + +def test_count_survives_a_reload(sr): + """The counter is on disk, not in memory — a restart must not reset it.""" + sr.add_rule("be brief") + sr.prompt_block(record=True) + sr.prompt_block(record=True) + fresh = sr.load() # re-read from disk, no cached state + assert fresh["rules"][0]["inject_count"] == 2 + + +def test_migration_backfills_without_resetting(sr): + """An old store keeps its data and gains the fields at zero.""" + import json + sr.RULES_PATH.write_text(json.dumps({"schema": 1, "rules": [ + {"id": "old1", "text": "keep me", "added": "2026-07-01T10:00:00+00:00"}]})) + r = sr.list_rules()[0] + assert r["text"] == "keep me", "existing rule text must survive migration" + assert r["added"] == "2026-07-01T10:00:00+00:00", "added date must survive" + assert r["inject_count"] == 0 and r["last_injected"] is None + + +def test_no_rules_means_no_write_churn(sr): + """An empty store must not be rewritten on every turn.""" + sr.prompt_block(record=True) + assert not sr.RULES_PATH.exists() or sr.load()["rules"] == [] + + +def test_listing_labels_injection_honestly(monkeypatch, tmp_path): + """The user-facing text must not imply the model 'used' the rule.""" + s = _skill(monkeypatch, tmp_path) + s.run("add a standing rule: be brief") + import codec_standing_rules as mod + mod.prompt_block(record=True) + out = s.run("show my standing rules") + assert "sent to the model" in out + assert "counts INJECTION" in out + assert "fired" not in out.lower(), "must not claim the rule 'fired'"