diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index fe9361c..6b0f1ba 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -2,6 +2,7 @@ id: hosted-worker paths: - "worker/**" + - "scripts/usefulness.py" entry: - "worker/src/index.ts" - "worker/src/memory.ts" @@ -15,6 +16,7 @@ tests: - "worker/test/readonly.test.ts" - "worker/test/feedback.test.ts" - "worker/test/confirm.test.ts" + - "tests/test_usefulness.py" --- # Hosted Worker @@ -27,6 +29,11 @@ defrag pass (`worker/src/defrag.ts`: file unsorted claims, then consolidate). It TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and static `KYPP_S1_TOKENS`. ## Gotchas +- Every shown `claim_usages` row keeps where its claim sat: `metadata` `{position, section, rank}`, + in the order the reply renders. `recordUsage` takes the reply's sections in that order, so a new + briefing section is passed there in render order too. `scripts/usefulness.py` scores each rating + against the used rate of its slot (read-only, one SELECT on production D1); act on its `lift`, not + on raw `used` counts, which favour whatever the briefing shows first. - Recall, briefing reads and `get` select `CLAIM_COLUMNS` (`worker/src/store.ts`), not `*`, so `metadata` and `file_tried_at` never reach a hydrated `Claim`. A field read off a claim on those paths has to be added to that list. diff --git a/kypp/store.py b/kypp/store.py index d3f2b4d..e4994ae 100644 --- a/kypp/store.py +++ b/kypp/store.py @@ -477,7 +477,10 @@ def record_usage(self, consumer: str, claims, *, surface: str, project: str | No self._write([( "INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,metadata,created_at)" " VALUES(?,?,?,?,?,?,?,?,?)", - (_uid(), consumer, cid, project, scope, surface, query, "{}", now)) for cid in ids]) + # Where each claim sat in what was shown, as the hosted store keeps it: a rating is read + # against what that slot gets anyway. + (_uid(), consumer, cid, project, scope, surface, query, + json.dumps({"position": i + 1, "section": surface, "rank": i + 1}), now)) for i, cid in enumerate(ids)]) return len(ids) def usages_for(self, consumer: str) -> list[dict]: diff --git a/scripts/usefulness.py b/scripts/usefulness.py new file mode 100644 index 0000000..93c8838 --- /dev/null +++ b/scripts/usefulness.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""How useful each claim is, against what its slot gets anyway. + + python3 scripts/usefulness.py [--since YYYY-MM-DD] [--min N] [--from rows.json] + +A `used` rating means more for a claim shown at position 9 of a recall than for one at the top of +every briefing: the top slots are read and acted on whatever sits there. So each rated showing is +scored against the used rate of its slot (surface, section, rank band) over every claim, and a claim's +`lift` is its used rate minus that expectation. Positive: it helps more than its placement explains. + +Read-only. Without --from it runs one SELECT on the production D1 database through wrangler (run it +from a session with tumbler, as `wrangler` already is), from the worker directory. Only a showing the +session rated counts; an unrated one says nothing either way, and the share rated is printed. +Showings recorded before rows kept their slot (`metadata.position`) fall in an "unplaced" band. +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +WORKER = Path(__file__).resolve().parent.parent / "worker" +VERDICTS = {1: "used", 0: "not_used", -1: "wrong"} + + +def sql(since: str) -> str: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", since): + raise SystemExit(f"usefulness: --since must be YYYY-MM-DD, got {since!r}") + return ("SELECT u.consumer, u.claim_id, u.surface, u.score, u.metadata, u.created_at, c.subject" + " FROM claim_usages u LEFT JOIN memory_claims c ON c.id = u.claim_id" + f" WHERE u.created_at >= '{since}' ORDER BY u.created_at, u.rowid") + + +def fetch(since: str) -> list[dict]: + out = subprocess.run(["npx", "wrangler", "d1", "execute", "DB", "--remote", "--env", "production", + "--json", "--command", sql(since)], cwd=WORKER, capture_output=True, text=True) + if out.returncode: + raise SystemExit(f"usefulness: wrangler failed ({out.returncode}): {out.stderr.strip()[-500:]}") + return json.loads(out.stdout)[0]["results"] + + +def band(rank: int | None) -> str: + if rank is None: + return "unplaced" + return "1-3" if rank <= 3 else "4-10" if rank <= 10 else "11+" + + +def slot(row: dict) -> tuple[str, str, str]: + meta = json.loads(row.get("metadata") or "{}") + return (row["surface"], meta.get("section", "?"), band(meta.get("rank"))) + + +def showings(rows: list[dict]) -> list[dict]: + """One record per (session, claim) the session was shown: the slot it was first shown in, and the + session's newest rating of it (None when it never rated it).""" + seen: dict[tuple[str, str], dict] = {} + for r in rows: # oldest first + key = (r["consumer"], r["claim_id"]) + if r["surface"] == "feedback": + if key in seen: # a rating with no showing in the window can't be placed + seen[key]["score"] = r["score"] + elif key not in seen: + seen[key] = {"claim": r["claim_id"], "subject": r.get("subject") or "", "slot": slot(r), "score": None} + return list(seen.values()) + + +def baselines(shown: list[dict]) -> dict[tuple, tuple[int, float]]: + """Per slot: rated showings and their used rate.""" + by = defaultdict(list) + for s in shown: + if s["score"] is not None: + by[s["slot"]].append(s["score"] == 1) + return {k: (len(v), sum(v) / len(v)) for k, v in by.items()} + + +def lifts(shown: list[dict], base: dict, min_rated: int) -> list[dict]: + by = defaultdict(list) + for s in shown: + if s["score"] is not None: + by[s["claim"]].append(s) + out = [] + for claim, rated in by.items(): + if len(rated) < min_rated: + continue + used = sum(s["score"] == 1 for s in rated) / len(rated) + expected = sum(base[s["slot"]][1] for s in rated) / len(rated) + out.append({"claim": claim, "subject": rated[0]["subject"], "n": len(rated), "used": used, + "expected": expected, "lift": used - expected, + "wrong": sum(s["score"] == -1 for s in rated)}) + return sorted(out, key=lambda r: (-r["lift"], r["claim"])) + + +def report(rows: list[dict], min_rated: int) -> str: + shown = showings(rows) + rated = [s for s in shown if s["score"] is not None] + base = baselines(shown) + lines = [f"{len(shown)} showings, {len(rated)} rated ({len(rated) / len(shown):.0%})" if shown else "no showings", + "", "Used rate by slot:", f" {'surface':9} {'section':9} {'rank':8} {'n':>4} {'used':>5}"] + lines += [f" {s:9} {sec:9} {b:8} {n:>4} {rate:>5.2f}" for (s, sec, b), (n, rate) in sorted(base.items())] + table = lifts(shown, base, min_rated) + lines += ["", f"Claims rated at least {min_rated} times, by lift (used rate minus its slots' rate):", + f" {'claim':10} {'n':>3} {'used':>5} {'expect':>6} {'lift':>6} {'wrong':>5} subject"] + lines += [f" {r['claim'][:8]:10} {r['n']:>3} {r['used']:>5.2f} {r['expected']:>6.2f} {r['lift']:>+6.2f} " + f"{r['wrong']:>5} {r['subject'][:60]}" for r in table] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--since", default="2026-11-07", help="YYYY-MM-DD; feedback began on 2026-11-07") + parser.add_argument("--min", type=int, default=3, help="ratings a claim needs to be listed") + parser.add_argument("--from", dest="source", type=Path, help="rows as JSON, instead of querying D1") + args = parser.parse_args(argv) + rows = json.loads(args.source.read_text()) if args.source else fetch(args.since) + print(report(rows, args.min)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_usefulness.py b/tests/test_usefulness.py new file mode 100644 index 0000000..97d0998 --- /dev/null +++ b/tests/test_usefulness.py @@ -0,0 +1,50 @@ +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import usefulness # noqa: E402 + + +def shown(consumer, claim, surface="briefing", section="settled", rank=1): + return {"consumer": consumer, "claim_id": claim, "surface": surface, "score": None, "subject": claim, + "metadata": json.dumps({"position": rank, "section": section, "rank": rank})} + + +def rated(consumer, claim, score): + return {"consumer": consumer, "claim_id": claim, "surface": "feedback", "score": score, + "metadata": json.dumps({"verdict": usefulness.VERDICTS[score]})} + + +class Lift(unittest.TestCase): + def test_a_claim_is_scored_against_its_slot(self): + rows = [] + # "top" sits first in every briefing and is used 2 of 4 times, as the top slot is on average. + # "deep" sits at rank 8 of recalls, where nothing else gets used, and is used 2 of 3 times. + for i in range(4): + rows += [shown(f"s{i}", "top"), rated(f"s{i}", "top", 1 if i < 2 else 0)] + for i in range(3): + rows += [shown(f"r{i}", "deep", "recall", "results", 8), rated(f"r{i}", "deep", 1 if i < 2 else -1)] + rows += [shown(f"r{i}", "noise", "recall", "results", 9), rated(f"r{i}", "noise", 0)] + table = {r["claim"]: r for r in usefulness.lifts(usefulness.showings(rows), usefulness.baselines(usefulness.showings(rows)), 3)} + self.assertAlmostEqual(table["top"]["lift"], 0.0) + self.assertGreater(table["deep"]["lift"], 0.3) + self.assertEqual(table["deep"]["wrong"], 1) + self.assertLess(table["noise"]["lift"], 0) + + def test_newest_rating_counts_and_unrated_or_unplaced_rows(self): + old = {**shown("a", "c"), "metadata": "{}"} # shown before rows kept their slot + rows = [old, rated("a", "c", 0), rated("a", "c", 1), shown("b", "c"), rated("z", "c", 1)] + got = usefulness.showings(rows) + self.assertEqual([(s["slot"], s["score"]) for s in got], + [(("briefing", "?", "unplaced"), 1), (("briefing", "settled", "1-3"), None)]) + self.assertIn("2 showings, 1 rated (50%)", usefulness.report(rows, 1)) + + def test_since_is_checked_before_it_reaches_sql(self): + with self.assertRaises(SystemExit): + usefulness.sql("2026-11-07' OR 1=1 --") + + +if __name__ == "__main__": + unittest.main() diff --git a/worker/src/mcp.ts b/worker/src/mcp.ts index 9c8bfa9..012a723 100644 --- a/worker/src/mcp.ts +++ b/worker/src/mcp.ts @@ -270,7 +270,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key ]); const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? ""); const claims = briefingOrder(settled.filter((c) => !isOrg(c)), limit); - await store.recordUsage(ctx.session, [...org, ...claims, ...yours], "briefing", null); + await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours]], "briefing", null); return (org.length ? `# how this org works\n${renderClaims(org)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims) : "") + (yours.length ? `\n# unconfirmed in this room\n${renderClaims(yours)}` : ""); } @@ -278,7 +278,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key const query = String(a.query ?? ""), limit = recallLimit(a.limit); const claims = await rankForQuery(ctx.s1, query, await store.recall(query, null, null, { room, candidates: a.include_candidates ? "all" : "origin", types: a.types, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit); - await store.recordUsage(ctx.session, claims, "recall", null, a.query || null); + await store.recordUsage(ctx.session, [["results", claims]], "recall", null, a.query || null); return renderClaims(claims); } case "claim": { @@ -311,7 +311,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key case "expand": { const c = await store.get(String(a.handle ?? ""), identity, ctx.session, room); if (!c) throw new Error(`unknown claim handle ${a.handle}`); - await store.recordUsage(ctx.session, [c], "expand", c.project); + await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); return JSON.stringify(claimDict(c), null, 2); } } @@ -397,7 +397,7 @@ async function pinnedTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, p const groups = new Set(ancestors.flat()); const visible = c && (c.scope === "project" ? c.project === null || names.includes(c.project) : c.scope === "group" ? groups.has(c.project ?? "") : true); if (!c || !visible) throw new Error(`unknown claim handle ${a.handle}`); - await store.recordUsage(ctx.session, [c], "expand", c.project); + await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); return JSON.stringify(claimDict(c), null, 2); } // Why: an unsorted claim from a pinned token could be filed anywhere, so it must name its project. @@ -426,7 +426,8 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, ]); const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? ""); const claims = briefingOrder(settled.filter((c) => !isOrg(c)), limit); - await store.recordUsage(ctx.session, [...org, ...claims, ...recent, ...yours], "briefing", project); + // In the order the reply renders them: yours comes before recent. + await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours], ["recent", recent]], "briefing", project); return note + (org.length ? `# how this org works\n${renderClaims(org)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims) : "") + (yours.length ? `\n# how you work, from your other sessions (unconfirmed)\n${renderClaims(yours)}` : "") + (recent.length ? `\n# recent from other agents (unconfirmed)\n${renderClaims(recent)}` : ""); @@ -435,7 +436,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, const query = String(a.query ?? ""), limit = recallLimit(a.limit); const claims = await rankForQuery(ctx.s1, query, await store.recall(query, project, user, { candidates: a.include_candidates ? "all" : "origin", groups, types: a.types, agent: a.agent, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit); - await store.recordUsage(ctx.session, claims, "recall", project, a.query || null); + await store.recordUsage(ctx.session, [["results", claims]], "recall", project, a.query || null); return note + renderClaims(claims); } case "claim": { @@ -482,7 +483,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, case "expand": { const c = await store.get(String(a.handle ?? ""), user, ctx.session); if (!c) throw new Error(`unknown claim handle ${a.handle}`); - await store.recordUsage(ctx.session, [c], "expand", c.project); + await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); return JSON.stringify(claimDict(c), null, 2); } case "correct": { diff --git a/worker/src/store.ts b/worker/src/store.ts index af2d0a4..55dde99 100644 --- a/worker/src/store.ts +++ b/worker/src/store.ts @@ -122,6 +122,8 @@ const laneSql = (at: Lane): [string, string] => "project" in at ? ["c.scope = 'p // Which unconfirmed (candidate) claims a read returns besides accepted memory: none (briefing), // other agents' claims in the caller's own project (the default), or candidates at every level. export type Candidates = "none" | "origin" | "all"; +// A read's reply as shown: [section label, its claims in order], in the order the reply renders them. +export type Shown = [section: string, claims: Claim[]][]; // What a caller sees: this project, the groups it rolls up to, their own user scope, the global // scope, and the unsorted claims their own session wrote; never superseded/rejected history. @@ -488,13 +490,17 @@ export class D1Store { return results.length ? hydrate(results[0]) : null; } - async recordUsage(consumer: string | null, claims: Claim[], surface: "recall" | "briefing" | "expand", + // What a read showed, in the order it rendered it: each section's label and claims. Every row keeps + // where its claim sat (`position` in the whole reply, `section`, `rank` within it), so a rating can + // be read against what that slot gets anyway: the top of a briefing is used more whatever is there. + async recordUsage(consumer: string | null, shown: Shown, surface: "recall" | "briefing" | "expand", project: string | null, query: string | null = null): Promise { - if (!consumer || !claims.length) return; + const rows = shown.flatMap(([section, claims]) => claims.map((c, i) => ({ c, section, rank: i + 1 }))); + if (!consumer || !rows.length) return; const ts = now(); - await this.db.batch(claims.map((c) => this.db.prepare( - "INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,created_at) VALUES(?,?,?,?,?,?,?,?)", - ).bind(uid(), consumer, c.id, project, c.scope, surface, query, ts))); + await this.db.batch(rows.map(({ c, section, rank }, i) => this.db.prepare( + "INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,metadata,created_at) VALUES(?,?,?,?,?,?,?,?,?)", + ).bind(uid(), consumer, c.id, project, c.scope, surface, query, JSON.stringify({ position: i + 1, section, rank }), ts))); } // The shown claim `handle` names for this session: one a read showed it, never a claim it only knows the id of. diff --git a/worker/test/feedback.test.ts b/worker/test/feedback.test.ts index baecdc3..3dbbc59 100644 --- a/worker/test/feedback.test.ts +++ b/worker/test/feedback.test.ts @@ -143,3 +143,25 @@ test("instructions name feedback for every caller that has it", async () => { assert.match(ro, /\n3\. BEFORE YOU FINISH — `feedback`/); assert.doesNotMatch(ro, /claim`/); }); + +test("every shown row keeps where its claim sat, so a rating can be read against its slot", async () => { + const db = await setup(); + const writer = agent(db, "w"); + const ids = []; + for (const n of [1, 2, 3]) ids.push((await writer("claim", { subject: `port ${n}`, content: `wrangler port lesson ${n}` })).text.split("\n")[0]); + const reader = agent(db, "r"); + await reader("recall", { query: "wrangler port", include_candidates: true }); + await reader("briefing"); + await reader("expand", { handle: ids[0] }); + const rows = (await db.prepare("SELECT claim_id, surface, metadata FROM claim_usages WHERE consumer = 'r' ORDER BY rowid").all()).results as any[]; + const recall = rows.filter((r) => r.surface === "recall").map((r) => JSON.parse(r.metadata)); + assert.deepEqual(recall, [1, 2, 3].map((n) => ({ position: n, section: "results", rank: n }))); + // The briefing renders others' unconfirmed claims under "recent", after any settled ones. + const briefing = rows.filter((r) => r.surface === "briefing").map((r) => JSON.parse(r.metadata)); + assert.ok(briefing.length > 0); + briefing.forEach((m, i) => assert.equal(m.position, i + 1)); + assert.ok(briefing.every((m) => m.section === "recent"), JSON.stringify(briefing)); + assert.deepEqual(briefing.map((m) => m.rank), briefing.map((_, i) => i + 1)); + const expand = rows.filter((r) => r.surface === "expand").map((r) => JSON.parse(r.metadata)); + assert.deepEqual(expand, [{ position: 1, section: "expand", rank: 1 }]); +}); diff --git a/worker/test/operator.test.ts b/worker/test/operator.test.ts index 1acb1c3..8c3192f 100644 --- a/worker/test/operator.test.ts +++ b/worker/test/operator.test.ts @@ -247,7 +247,7 @@ test("purge erases a claim's text from every place recall reads, keeps a tombsto const other = await store.claim({ type: "fact", subject: "deploys", content: "deploys go out on green", scope: "user", project: null, confidence: 0.7, sourceIds: [], codeRefs: [], accept: true, agent: "t", user: "someone" }); // One recall returned both claims, so its query (and the key) is copied onto both usage rows. - await store.recordUsage("s", [(await store.anyClaim(id))!, (await store.anyClaim(other))!], "recall", null, `which ${KEY}`); + await store.recordUsage("s", [["results", [(await store.anyClaim(id))!, (await store.anyClaim(other))!]]], "recall", null, `which ${KEY}`); // An earlier reject whose reason quotes the leaked text. await store.operatorReject(id, `wrong: ${KEY}`); assert.equal((await call({ op: "purge", handle: id.slice(0, 8) })).status, 400, "a reason is required");