Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/features/hosted-worker.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
id: hosted-worker
paths:
- "worker/**"
- "scripts/usefulness.py"
entry:
- "worker/src/index.ts"
- "worker/src/memory.ts"
Expand All @@ -15,6 +16,7 @@ tests:
- "worker/test/readonly.test.ts"
- "worker/test/feedback.test.ts"
- "worker/test/confirm.test.ts"
- "tests/test_usefulness.py"
---
# Hosted Worker

Expand All @@ -27,6 +29,11 @@ defrag pass (`worker/src/defrag.ts`: file unsorted claims, then consolidate). It
TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and static `KYPP_S1_TOKENS`.

## Gotchas
- Every shown `claim_usages` row keeps where its claim sat: `metadata` `{position, section, rank}`,
in the order the reply renders. `recordUsage` takes the reply's sections in that order, so a new
briefing section is passed there in render order too. `scripts/usefulness.py` scores each rating
against the used rate of its slot (read-only, one SELECT on production D1); act on its `lift`, not
on raw `used` counts, which favour whatever the briefing shows first.
- Recall, briefing reads and `get` select `CLAIM_COLUMNS` (`worker/src/store.ts`), not `*`, so
`metadata` and `file_tried_at` never reach a hydrated `Claim`. A field read off a claim on those
paths has to be added to that list.
Expand Down
5 changes: 4 additions & 1 deletion kypp/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,10 @@ def record_usage(self, consumer: str, claims, *, surface: str, project: str | No
self._write([(
"INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,metadata,created_at)"
" VALUES(?,?,?,?,?,?,?,?,?)",
(_uid(), consumer, cid, project, scope, surface, query, "{}", now)) for cid in ids])
# Where each claim sat in what was shown, as the hosted store keeps it: a rating is read
# against what that slot gets anyway.
(_uid(), consumer, cid, project, scope, surface, query,
json.dumps({"position": i + 1, "section": surface, "rank": i + 1}), now)) for i, cid in enumerate(ids)])
return len(ids)

def usages_for(self, consumer: str) -> list[dict]:
Expand Down
124 changes: 124 additions & 0 deletions scripts/usefulness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""How useful each claim is, against what its slot gets anyway.

python3 scripts/usefulness.py [--since YYYY-MM-DD] [--min N] [--from rows.json]

A `used` rating means more for a claim shown at position 9 of a recall than for one at the top of
every briefing: the top slots are read and acted on whatever sits there. So each rated showing is
scored against the used rate of its slot (surface, section, rank band) over every claim, and a claim's
`lift` is its used rate minus that expectation. Positive: it helps more than its placement explains.

Read-only. Without --from it runs one SELECT on the production D1 database through wrangler (run it
from a session with tumbler, as `wrangler` already is), from the worker directory. Only a showing the
session rated counts; an unrated one says nothing either way, and the share rated is printed.
Showings recorded before rows kept their slot (`metadata.position`) fall in an "unplaced" band.
"""
from __future__ import annotations

import argparse
import json
import re
import subprocess
import sys
from collections import defaultdict
from pathlib import Path

WORKER = Path(__file__).resolve().parent.parent / "worker"
VERDICTS = {1: "used", 0: "not_used", -1: "wrong"}


def sql(since: str) -> str:
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", since):
raise SystemExit(f"usefulness: --since must be YYYY-MM-DD, got {since!r}")
return ("SELECT u.consumer, u.claim_id, u.surface, u.score, u.metadata, u.created_at, c.subject"
" FROM claim_usages u LEFT JOIN memory_claims c ON c.id = u.claim_id"
f" WHERE u.created_at >= '{since}' ORDER BY u.created_at, u.rowid")


def fetch(since: str) -> list[dict]:
out = subprocess.run(["npx", "wrangler", "d1", "execute", "DB", "--remote", "--env", "production",
"--json", "--command", sql(since)], cwd=WORKER, capture_output=True, text=True)
if out.returncode:
raise SystemExit(f"usefulness: wrangler failed ({out.returncode}): {out.stderr.strip()[-500:]}")
return json.loads(out.stdout)[0]["results"]


def band(rank: int | None) -> str:
if rank is None:
return "unplaced"
return "1-3" if rank <= 3 else "4-10" if rank <= 10 else "11+"


def slot(row: dict) -> tuple[str, str, str]:
meta = json.loads(row.get("metadata") or "{}")
return (row["surface"], meta.get("section", "?"), band(meta.get("rank")))


def showings(rows: list[dict]) -> list[dict]:
"""One record per (session, claim) the session was shown: the slot it was first shown in, and the
session's newest rating of it (None when it never rated it)."""
seen: dict[tuple[str, str], dict] = {}
for r in rows: # oldest first
key = (r["consumer"], r["claim_id"])
if r["surface"] == "feedback":
if key in seen: # a rating with no showing in the window can't be placed
seen[key]["score"] = r["score"]
elif key not in seen:
seen[key] = {"claim": r["claim_id"], "subject": r.get("subject") or "", "slot": slot(r), "score": None}
return list(seen.values())


def baselines(shown: list[dict]) -> dict[tuple, tuple[int, float]]:
"""Per slot: rated showings and their used rate."""
by = defaultdict(list)
for s in shown:
if s["score"] is not None:
by[s["slot"]].append(s["score"] == 1)
return {k: (len(v), sum(v) / len(v)) for k, v in by.items()}


def lifts(shown: list[dict], base: dict, min_rated: int) -> list[dict]:
by = defaultdict(list)
for s in shown:
if s["score"] is not None:
by[s["claim"]].append(s)
out = []
for claim, rated in by.items():
if len(rated) < min_rated:
continue
used = sum(s["score"] == 1 for s in rated) / len(rated)
expected = sum(base[s["slot"]][1] for s in rated) / len(rated)
out.append({"claim": claim, "subject": rated[0]["subject"], "n": len(rated), "used": used,
"expected": expected, "lift": used - expected,
"wrong": sum(s["score"] == -1 for s in rated)})
return sorted(out, key=lambda r: (-r["lift"], r["claim"]))


def report(rows: list[dict], min_rated: int) -> str:
shown = showings(rows)
rated = [s for s in shown if s["score"] is not None]
base = baselines(shown)
lines = [f"{len(shown)} showings, {len(rated)} rated ({len(rated) / len(shown):.0%})" if shown else "no showings",
"", "Used rate by slot:", f" {'surface':9} {'section':9} {'rank':8} {'n':>4} {'used':>5}"]
lines += [f" {s:9} {sec:9} {b:8} {n:>4} {rate:>5.2f}" for (s, sec, b), (n, rate) in sorted(base.items())]
table = lifts(shown, base, min_rated)
lines += ["", f"Claims rated at least {min_rated} times, by lift (used rate minus its slots' rate):",
f" {'claim':10} {'n':>3} {'used':>5} {'expect':>6} {'lift':>6} {'wrong':>5} subject"]
lines += [f" {r['claim'][:8]:10} {r['n']:>3} {r['used']:>5.2f} {r['expected']:>6.2f} {r['lift']:>+6.2f} "
f"{r['wrong']:>5} {r['subject'][:60]}" for r in table]
return "\n".join(lines)


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--since", default="2026-11-07", help="YYYY-MM-DD; feedback began on 2026-11-07")
parser.add_argument("--min", type=int, default=3, help="ratings a claim needs to be listed")
parser.add_argument("--from", dest="source", type=Path, help="rows as JSON, instead of querying D1")
args = parser.parse_args(argv)
rows = json.loads(args.source.read_text()) if args.source else fetch(args.since)
print(report(rows, args.min))
return 0


if __name__ == "__main__":
sys.exit(main())
50 changes: 50 additions & 0 deletions tests/test_usefulness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import json
import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
import usefulness # noqa: E402


def shown(consumer, claim, surface="briefing", section="settled", rank=1):
return {"consumer": consumer, "claim_id": claim, "surface": surface, "score": None, "subject": claim,
"metadata": json.dumps({"position": rank, "section": section, "rank": rank})}


def rated(consumer, claim, score):
return {"consumer": consumer, "claim_id": claim, "surface": "feedback", "score": score,
"metadata": json.dumps({"verdict": usefulness.VERDICTS[score]})}


class Lift(unittest.TestCase):
def test_a_claim_is_scored_against_its_slot(self):
rows = []
# "top" sits first in every briefing and is used 2 of 4 times, as the top slot is on average.
# "deep" sits at rank 8 of recalls, where nothing else gets used, and is used 2 of 3 times.
for i in range(4):
rows += [shown(f"s{i}", "top"), rated(f"s{i}", "top", 1 if i < 2 else 0)]
for i in range(3):
rows += [shown(f"r{i}", "deep", "recall", "results", 8), rated(f"r{i}", "deep", 1 if i < 2 else -1)]
rows += [shown(f"r{i}", "noise", "recall", "results", 9), rated(f"r{i}", "noise", 0)]
table = {r["claim"]: r for r in usefulness.lifts(usefulness.showings(rows), usefulness.baselines(usefulness.showings(rows)), 3)}
self.assertAlmostEqual(table["top"]["lift"], 0.0)
self.assertGreater(table["deep"]["lift"], 0.3)
self.assertEqual(table["deep"]["wrong"], 1)
self.assertLess(table["noise"]["lift"], 0)

def test_newest_rating_counts_and_unrated_or_unplaced_rows(self):
old = {**shown("a", "c"), "metadata": "{}"} # shown before rows kept their slot
rows = [old, rated("a", "c", 0), rated("a", "c", 1), shown("b", "c"), rated("z", "c", 1)]
got = usefulness.showings(rows)
self.assertEqual([(s["slot"], s["score"]) for s in got],
[(("briefing", "?", "unplaced"), 1), (("briefing", "settled", "1-3"), None)])
self.assertIn("2 showings, 1 rated (50%)", usefulness.report(rows, 1))

def test_since_is_checked_before_it_reaches_sql(self):
with self.assertRaises(SystemExit):
usefulness.sql("2026-11-07' OR 1=1 --")


if __name__ == "__main__":
unittest.main()
15 changes: 8 additions & 7 deletions worker/src/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -270,15 +270,15 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key
]);
const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? "");
const claims = briefingOrder(settled.filter((c) => !isOrg(c)), limit);
await store.recordUsage(ctx.session, [...org, ...claims, ...yours], "briefing", null);
await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours]], "briefing", null);
return (org.length ? `# how this org works\n${renderClaims(org)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims) : "")
+ (yours.length ? `\n# unconfirmed in this room\n${renderClaims(yours)}` : "");
}
case "recall": {
const query = String(a.query ?? ""), limit = recallLimit(a.limit);
const claims = await rankForQuery(ctx.s1, query, await store.recall(query, null, null,
{ room, candidates: a.include_candidates ? "all" : "origin", types: a.types, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit);
await store.recordUsage(ctx.session, claims, "recall", null, a.query || null);
await store.recordUsage(ctx.session, [["results", claims]], "recall", null, a.query || null);
return renderClaims(claims);
}
case "claim": {
Expand Down Expand Up @@ -311,7 +311,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key
case "expand": {
const c = await store.get(String(a.handle ?? ""), identity, ctx.session, room);
if (!c) throw new Error(`unknown claim handle ${a.handle}`);
await store.recordUsage(ctx.session, [c], "expand", c.project);
await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project);
return JSON.stringify(claimDict(c), null, 2);
}
}
Expand Down Expand Up @@ -397,7 +397,7 @@ async function pinnedTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, p
const groups = new Set(ancestors.flat());
const visible = c && (c.scope === "project" ? c.project === null || names.includes(c.project) : c.scope === "group" ? groups.has(c.project ?? "") : true);
if (!c || !visible) throw new Error(`unknown claim handle ${a.handle}`);
await store.recordUsage(ctx.session, [c], "expand", c.project);
await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project);
return JSON.stringify(claimDict(c), null, 2);
}
// Why: an unsorted claim from a pinned token could be filed anywhere, so it must name its project.
Expand Down Expand Up @@ -426,7 +426,8 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx,
]);
const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? "");
const claims = briefingOrder(settled.filter((c) => !isOrg(c)), limit);
await store.recordUsage(ctx.session, [...org, ...claims, ...recent, ...yours], "briefing", project);
// In the order the reply renders them: yours comes before recent.
await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours], ["recent", recent]], "briefing", project);
return note + (org.length ? `# how this org works\n${renderClaims(org)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims) : "")
+ (yours.length ? `\n# how you work, from your other sessions (unconfirmed)\n${renderClaims(yours)}` : "")
+ (recent.length ? `\n# recent from other agents (unconfirmed)\n${renderClaims(recent)}` : "");
Expand All @@ -435,7 +436,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx,
const query = String(a.query ?? ""), limit = recallLimit(a.limit);
const claims = await rankForQuery(ctx.s1, query, await store.recall(query, project, user,
{ candidates: a.include_candidates ? "all" : "origin", groups, types: a.types, agent: a.agent, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit);
await store.recordUsage(ctx.session, claims, "recall", project, a.query || null);
await store.recordUsage(ctx.session, [["results", claims]], "recall", project, a.query || null);
return note + renderClaims(claims);
}
case "claim": {
Expand Down Expand Up @@ -482,7 +483,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx,
case "expand": {
const c = await store.get(String(a.handle ?? ""), user, ctx.session);
if (!c) throw new Error(`unknown claim handle ${a.handle}`);
await store.recordUsage(ctx.session, [c], "expand", c.project);
await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project);
return JSON.stringify(claimDict(c), null, 2);
}
case "correct": {
Expand Down
16 changes: 11 additions & 5 deletions worker/src/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ const laneSql = (at: Lane): [string, string] => "project" in at ? ["c.scope = 'p
// Which unconfirmed (candidate) claims a read returns besides accepted memory: none (briefing),
// other agents' claims in the caller's own project (the default), or candidates at every level.
export type Candidates = "none" | "origin" | "all";
// A read's reply as shown: [section label, its claims in order], in the order the reply renders them.
export type Shown = [section: string, claims: Claim[]][];

// What a caller sees: this project, the groups it rolls up to, their own user scope, the global
// scope, and the unsorted claims their own session wrote; never superseded/rejected history.
Expand Down Expand Up @@ -488,13 +490,17 @@ export class D1Store {
return results.length ? hydrate(results[0]) : null;
}

async recordUsage(consumer: string | null, claims: Claim[], surface: "recall" | "briefing" | "expand",
// What a read showed, in the order it rendered it: each section's label and claims. Every row keeps
// where its claim sat (`position` in the whole reply, `section`, `rank` within it), so a rating can
// be read against what that slot gets anyway: the top of a briefing is used more whatever is there.
async recordUsage(consumer: string | null, shown: Shown, surface: "recall" | "briefing" | "expand",
project: string | null, query: string | null = null): Promise<void> {
if (!consumer || !claims.length) return;
const rows = shown.flatMap(([section, claims]) => claims.map((c, i) => ({ c, section, rank: i + 1 })));
if (!consumer || !rows.length) return;
const ts = now();
await this.db.batch(claims.map((c) => this.db.prepare(
"INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,created_at) VALUES(?,?,?,?,?,?,?,?)",
).bind(uid(), consumer, c.id, project, c.scope, surface, query, ts)));
await this.db.batch(rows.map(({ c, section, rank }, i) => this.db.prepare(
"INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,query,metadata,created_at) VALUES(?,?,?,?,?,?,?,?,?)",
).bind(uid(), consumer, c.id, project, c.scope, surface, query, JSON.stringify({ position: i + 1, section, rank }), ts)));
}

// The shown claim `handle` names for this session: one a read showed it, never a claim it only knows the id of.
Expand Down
22 changes: 22 additions & 0 deletions worker/test/feedback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,3 +143,25 @@ test("instructions name feedback for every caller that has it", async () => {
assert.match(ro, /\n3\. BEFORE YOU FINISH — `feedback`/);
assert.doesNotMatch(ro, /claim`/);
});

test("every shown row keeps where its claim sat, so a rating can be read against its slot", async () => {
const db = await setup();
const writer = agent(db, "w");
const ids = [];
for (const n of [1, 2, 3]) ids.push((await writer("claim", { subject: `port ${n}`, content: `wrangler port lesson ${n}` })).text.split("\n")[0]);
const reader = agent(db, "r");
await reader("recall", { query: "wrangler port", include_candidates: true });
await reader("briefing");
await reader("expand", { handle: ids[0] });
const rows = (await db.prepare("SELECT claim_id, surface, metadata FROM claim_usages WHERE consumer = 'r' ORDER BY rowid").all()).results as any[];
const recall = rows.filter((r) => r.surface === "recall").map((r) => JSON.parse(r.metadata));
assert.deepEqual(recall, [1, 2, 3].map((n) => ({ position: n, section: "results", rank: n })));
// The briefing renders others' unconfirmed claims under "recent", after any settled ones.
const briefing = rows.filter((r) => r.surface === "briefing").map((r) => JSON.parse(r.metadata));
assert.ok(briefing.length > 0);
briefing.forEach((m, i) => assert.equal(m.position, i + 1));
assert.ok(briefing.every((m) => m.section === "recent"), JSON.stringify(briefing));
assert.deepEqual(briefing.map((m) => m.rank), briefing.map((_, i) => i + 1));
const expand = rows.filter((r) => r.surface === "expand").map((r) => JSON.parse(r.metadata));
assert.deepEqual(expand, [{ position: 1, section: "expand", rank: 1 }]);
});
2 changes: 1 addition & 1 deletion worker/test/operator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ test("purge erases a claim's text from every place recall reads, keeps a tombsto
const other = await store.claim({ type: "fact", subject: "deploys", content: "deploys go out on green", scope: "user", project: null,
confidence: 0.7, sourceIds: [], codeRefs: [], accept: true, agent: "t", user: "someone" });
// One recall returned both claims, so its query (and the key) is copied onto both usage rows.
await store.recordUsage("s", [(await store.anyClaim(id))!, (await store.anyClaim(other))!], "recall", null, `which ${KEY}`);
await store.recordUsage("s", [["results", [(await store.anyClaim(id))!, (await store.anyClaim(other))!]]], "recall", null, `which ${KEY}`);
// An earlier reject whose reason quotes the leaked text.
await store.operatorReject(id, `wrong: ${KEY}`);
assert.equal((await call({ op: "purge", handle: id.slice(0, 8) })).status, 400, "a reason is required");
Expand Down
Loading