diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index d43d2df..6c31c53 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -16,6 +16,7 @@ tests: - "worker/test/readonly.test.ts" - "worker/test/feedback.test.ts" - "worker/test/confirm.test.ts" + - "worker/test/check.test.ts" - "worker/test/briefing-budget.test.ts" - "worker/test/dispute.test.ts" - "worker/test/evidence.test.ts" @@ -234,6 +235,11 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and passed to `recordUsage`, so a slot's used rate is over what the reply showed. A new briefing section goes through `fitted` with the others, or it escapes the budget. Unconfirmed user-scope lines (`RECENT_USER`) are capped at 3 because they reach every repo's briefing. +- Agents are memory's recheck pass (a System One model returns probabilities, never text, so it can't + rewrite a claim). Each briefing moves up to `CHECK` claims into a "check these" section (`toCheck`): + disputed ones first (`recall(..., { disputed: true })`), then accepted ones shown at least `CHECK_SHOWN` times with no writable `used` in + `CHECK_DAYS`. A disputed claim reaches a briefing only there. They render after "yours" and before + "recent", inside the same budget, and are recorded under section `check`. - `initialize` instructions are built from `STEPS` in `mcp.ts`, keeping only the steps whose tool the caller can see (`visibleTools`), so a read-only or `KYPP_CORRECT=off` caller is never told to call a tool it lacks. diff --git a/worker/src/mcp.ts b/worker/src/mcp.ts index 853ec81..c25c794 100644 --- a/worker/src/mcp.ts +++ b/worker/src/mcp.ts @@ -191,6 +191,25 @@ function fitted(sections: Claim[][]) { return { keep, line, more }; } +// Memory is checked by the agents that read it: each briefing asks for a check of up to CHECK claims, +// disputed ones first, then accepted ones read at least CHECK_SHOWN times with no `used` in CHECK_DAYS. +// A disputed claim comes back only here, under a heading that says to check it. +const CHECK = 2, CHECK_SHOWN = 10, CHECK_DAYS = 30; +const CHECK_HEADING = "# check these if your work touches them, then rate them with `feedback`"; +async function toCheck(store: D1Store, claims: Claim[], disputed: Claim[], session: string | null): Promise { + for (const c of disputed) c.disputed = true; + const mine = (c: Claim) => !!session && c.session === session; // never ask a session to check its own claim + const seen = await store.checkedness(claims.map((c) => c.id)); + const since = new Date(Date.now() - CHECK_DAYS * 86_400_000).toISOString(); + const stale = (c: Claim) => { + const k = seen.get(c.id); + return c.status === "accepted" && !!k && k.shown >= CHECK_SHOWN && (k.lastUsed ?? "") < since; + }; + const byShown = (a: Claim, b: Claim) => (seen.get(b.id)?.shown ?? 0) - (seen.get(a.id)?.shown ?? 0); + const unique = [...new Map(claims.map((c) => [c.id, c])).values()].filter((c) => !mine(c) && stale(c)).sort(byShown); + return [...disputed.filter((c) => !mine(c)), ...unique].slice(0, CHECK); +} + async function expanded(store: D1Store, c: Claim): Promise { const d = (await store.disputes([c.id])).get(c.id); return JSON.stringify(claimDict(c, d), null, 2); @@ -295,16 +314,20 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key const limit = Math.min(a.limit ?? 12, 30); // Why a read of its own, as in a repo briefing: recall ranks the org last, so a busy room's // capped list would crowd its rules out. - const [settled, { orgs, org: allOrg }, allYours] = await Promise.all([ + const [settled, { orgs, org: allOrg }, allYours, disputed] = await Promise.all([ store.recall("", null, null, { room, candidates: "none", limit: limit * 3, undisputed: true }), orgRules(store, groups), store.recentUserCandidates(identity, ctx.session, RECENT_USER, "room"), + store.recall("", null, null, { room, candidates: "none", limit: CHECK, disputed: true }), ]); + const check = await toCheck(store, [...allOrg, ...settled], disputed, ctx.session); + const checking = new Set(check.map((c) => c.id)), shown = (cs: Claim[]) => cs.filter((c) => !checking.has(c.id)); const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? ""); - const { keep: [org, claims, yours], line, more } = fitted([allOrg, briefingOrder(settled.filter((c) => !isOrg(c)), limit), allYours]); - await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours]], "briefing", null); + const { keep: [org, claims, yours, chk], line, more } = fitted([shown(allOrg), briefingOrder(shown(settled).filter((c) => !isOrg(c)), limit), allYours, check]); + await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours], ["check", chk]], "briefing", null); return (org.length ? `# how this org works\n${renderClaims(org, undefined, line)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims, line) : "") - + (yours.length ? `\n# unconfirmed in this room\n${renderClaims(yours, undefined, line)}` : "") + more; + + (yours.length ? `\n# unconfirmed in this room\n${renderClaims(yours, undefined, line)}` : "") + + (chk.length ? `\n${CHECK_HEADING}\n${renderClaims(chk, undefined, line)}` : "") + more; } case "recall": { const query = String(a.query ?? ""), limit = recallLimit(a.limit); @@ -449,21 +472,25 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, case "briefing": { const limit = Math.min(a.limit ?? 12, 30); // An org's rules apply to every contributor, so they come first under their own heading. - const [settled, { orgs, org: allOrg }, allRecent, allYours] = await Promise.all([ + const [settled, { orgs, org: allOrg }, allRecent, allYours, disputed] = await Promise.all([ store.recall("", project, user, { candidates: "none", groups, limit: limit * 3, undisputed: true }), orgRules(store, groups), // Why: the briefing stays settled memory; others' unconfirmed claims get a short, labelled section. project ? store.recentCandidates(project, ctx.session, RECENT) : [], // User scope is how this person works, read from every repo; unconfirmed lessons stay labelled and apart. user ? store.recentUserCandidates(user, ctx.session, RECENT_USER) : [], + store.recall("", project, user, { candidates: "none", groups, limit: CHECK, disputed: true }), ]); + const check = await toCheck(store, [...allOrg, ...settled], disputed, ctx.session); + const checking = new Set(check.map((c) => c.id)), shown = (cs: Claim[]) => cs.filter((c) => !checking.has(c.id)); const isOrg = (c: Claim) => c.scope === "group" && orgs.has(c.project ?? ""); // In the order the reply renders them: yours comes before recent. - const { keep: [org, claims, yours, recent], line, more } = fitted([allOrg, - briefingOrder(settled.filter((c) => !isOrg(c)), limit), allYours, allRecent]); - await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours], ["recent", recent]], "briefing", project); + const { keep: [org, claims, yours, chk, recent], line, more } = fitted([shown(allOrg), + briefingOrder(shown(settled).filter((c) => !isOrg(c)), limit), allYours, check, allRecent]); + await store.recordUsage(ctx.session, [["org", org], ["settled", claims], ["yours", yours], ["check", chk], ["recent", recent]], "briefing", project); return note + (org.length ? `# how this org works\n${renderClaims(org, undefined, line)}\n` : "") + (claims.length || !org.length ? renderBriefing(claims, line) : "") + (yours.length ? `\n# how you work, from your other sessions (unconfirmed)\n${renderClaims(yours, undefined, line)}` : "") + + (chk.length ? `\n${CHECK_HEADING}\n${renderClaims(chk, undefined, line)}` : "") + (recent.length ? `\n# recent from other agents (unconfirmed)\n${renderClaims(recent, undefined, line)}` : "") + more; } case "recall": { diff --git a/worker/src/store.ts b/worker/src/store.ts index a02ae1a..4d92f45 100644 --- a/worker/src/store.ts +++ b/worker/src/store.ts @@ -393,7 +393,7 @@ export class D1Store { // The id is last so equal rows come back in the same order on every engine. async recall(match: string, project: string | null, user: string | null, opts: { candidates?: Candidates; groups?: string[]; types?: string[]; agent?: string; limit?: number; session?: string | null; room?: RoomScope; - undisputed?: boolean } = {}): Promise { + undisputed?: boolean; disputed?: boolean } = {}): Promise { match = match.trim(); const [where, params] = opts.room ? roomVisible("c", opts.room, opts.candidates ?? "origin") @@ -405,6 +405,7 @@ export class D1Store { if (opts.agent) { filter += " AND c.agent = ?"; fparams.push(opts.agent); } // Briefings leave disputed claims out in the query itself, so they can't use up its LIMIT. if (opts.undisputed) filter += ` AND NOT ${DISPUTED("c")}`; + if (opts.disputed) filter += ` AND ${DISPUTED("c")}`; // `groups` comes nearest first, so a parent's claim outranks a grandparent's. const groups = opts.room?.groups ?? opts.groups ?? []; const near = groups.length ? `1 + (CASE c.project ${groups.map((_, i) => `WHEN ? THEN ${i}`).join(" ")} END) * 0.01` : "1"; @@ -595,6 +596,17 @@ export class D1Store { return new Map(results.map((r) => [r.id as string, { at: r.at as string }])); } + // Per claim, how many times reads showed it and when a writable `used` last confirmed it held. + async checkedness(ids: string[]): Promise> { + if (!ids.length) return new Map(); + const { results } = await this.db.prepare( + "SELECT u.claim_id, SUM(CASE WHEN u.surface != 'feedback' THEN 1 ELSE 0 END) AS shown," + + " MAX(CASE WHEN u.surface = 'feedback' AND u.score = 1 AND json_extract(u.metadata, '$.writable') = 1 THEN u.created_at END) AS last_used" + + " FROM claim_usages u JOIN json_each(?) j ON j.value = u.claim_id GROUP BY u.claim_id", + ).bind(JSON.stringify([...new Set(ids)])).all(); + return new Map(results.map((r) => [r.claim_id as string, { shown: Number(r.shown), lastUsed: (r.last_used as string | null) ?? null }])); + } + // Every group or org `project` rolls up to, nearest first, through parents of parents up to three // levels (a cycle in the table just stops). The roll-up table is the only thing that decides what a // project can see. diff --git a/worker/test/check.test.ts b/worker/test/check.test.ts new file mode 100644 index 0000000..a53c558 --- /dev/null +++ b/worker/test/check.test.ts @@ -0,0 +1,54 @@ +// Agents check memory as they read it: a briefing asks for a check of an accepted claim that is read +// often and hasn't held up in anyone's work lately, and a writable `used` takes it back off the list. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { type Caller, handleMcp } from "../src/mcp.ts"; +import { memoryD1 } from "./d1.ts"; + +const OWNER: Caller = { user: "owner", agent: "claude" }; +const HEADING = "\n# check these if your work touches them, then rate them with `feedback`\n"; + +async function setup() { + const db = memoryD1(); + await db.prepare("INSERT INTO projects(name, description, created_at) VALUES('kypp', '', '2026-11-06T00:00:00Z')").run(); + return db; +} + +function agent(db: D1Database, session: string) { + return async (name: string, args: Record = {}) => { + const req = new Request("https://k/mcp", { method: "POST", headers: { "X-Kypp-Project": "kypp", "Mcp-Session-Id": session }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }) }); + return ((await (await handleMcp(req, db, OWNER, { s1: null })).json()) as any).result.content[0].text as string; + }; +} + +test("an often-read claim nobody has confirmed lately moves to the check section until a used", async () => { + const db = await setup(); + const w = agent(db, "w"); + const id = (await w("correct", { subject: "wrangler port", content: "wrangler dev listens on 8787", type: "pitfall" })).split("\n")[0].trim(); + await w("correct", { subject: "deploys", content: "the worker deploys on merge to main", type: "procedure" }); + for (let i = 0; i < 9; i++) await agent(db, `r${i}`)("briefing"); + + const before = await agent(db, "r9")("briefing"); + assert.ok(!before.includes(HEADING), "9 reads each is under the bar"); + const after = await agent(db, "r10")("briefing"); + const [settled, check] = after.split(HEADING); + assert.match(check, /wrangler port/); + assert.match(check, /deploys/); + assert.doesNotMatch(settled, /wrangler port|deploys/); + + await agent(db, "r10")("feedback", { ratings: [{ handle: id, verdict: "used", note: "ran wrangler dev; it answered on 8787" }] }); + const [settled2, check2] = (await agent(db, "r11")("briefing")).split(HEADING); + assert.match(settled2, /wrangler port/); + assert.doesNotMatch(check2, /wrangler port/); + assert.match(check2, /deploys/); +}); + +test("a briefing asks for at most two checks", async () => { + const db = await setup(); + const w = agent(db, "w"); + for (const s of ["a", "b", "c"]) await w("correct", { subject: `trap ${s}`, content: `lesson ${s}`, type: "pitfall" }); + for (let i = 0; i < 10; i++) await agent(db, `r${i}`)("briefing"); + const check = (await agent(db, "x")("briefing")).split(HEADING)[1]; + assert.equal(check.split("\n").filter((l) => /^[0-9a-f]{8} \[/.test(l)).length, 2); +}); diff --git a/worker/test/dispute.test.ts b/worker/test/dispute.test.ts index 5db2e7d..ca01586 100644 --- a/worker/test/dispute.test.ts +++ b/worker/test/dispute.test.ts @@ -27,6 +27,9 @@ async function accepted(db: D1Database, subject: string, content: string) { return (await agent(db, "w")("correct", { subject, content, type: "pitfall" })).split("\n")[0].trim(); } +// The briefing without its "check these" section, where a disputed claim comes back to be checked. +const settled = (briefing: string) => briefing.split("\n# check these")[0]; + const WRONG = { verdict: "wrong", note: "the port moved to 8788" }; const USED = { verdict: "used", note: "ran wrangler dev; it answered on 8787" }; @@ -38,7 +41,10 @@ test("a wrong rating takes the claim out of the briefing; recall and expand mark await r("feedback", { ratings: [{ handle: id, ...WRONG }] }); const next = agent(db, "n"); - assert.doesNotMatch(await next("briefing"), /wrangler port/); + const brief = await next("briefing"); + assert.doesNotMatch(settled(brief), /wrangler port/); + assert.match(brief, /\n# check these if your work touches them, then rate them with `feedback`\n.*wrangler port .*⚠ a session rated this wrong/, + "a disputed claim comes back only to be checked"); assert.match(await next("recall", { query: "wrangler port" }), /wrangler port .*⚠ a session rated this wrong; check it before relying on it$/); const full = JSON.parse(await next("expand", { handle: id })); assert.ok(full.rated_wrong.at); @@ -52,7 +58,7 @@ test("a later used from another session clears the dispute", async () => { const a = agent(db, "a"), b = agent(db, "b"); await a("briefing"); await b("briefing"); await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); - assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); + assert.doesNotMatch(settled(await agent(db, "x")("briefing")), /wrangler port/); await b("feedback", { ratings: [{ handle: id, ...USED }] }); assert.match(await agent(db, "y")("briefing"), /wrangler port/); assert.doesNotMatch(await agent(db, "y")("recall", { query: "wrangler port" }), /rated this wrong/); @@ -65,7 +71,7 @@ test("a used made before the wrong doesn't clear it, and the rater withdrawing d await a("briefing"); await b("briefing"); await b("feedback", { ratings: [{ handle: id, ...USED }] }); await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); - assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); + assert.doesNotMatch(settled(await agent(db, "x")("briefing")), /wrangler port/); await a("feedback", { ratings: [{ handle: id, verdict: "not_used" }] }); assert.match(await agent(db, "y")("briefing"), /wrangler port/); }); @@ -98,7 +104,7 @@ test("a later rating from the same session without write access doesn't lift its await a("briefing"); await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); await aRo("feedback", { ratings: [{ handle: id, ...USED }] }); - assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); + assert.doesNotMatch(settled(await agent(db, "x")("briefing")), /wrangler port/); }); test("disputed claims don't use up a section's limit: the briefing fills past them", async () => { @@ -129,5 +135,5 @@ test("the briefing finds undisputed memory past more disputed claims than one qu await a("feedback", { ratings: disputed.slice(50).map((handle) => ({ handle, ...WRONG })) }); const brief = await agent(db, "x")("briefing", { limit: 30 }); assert.match(brief, /a lesson that held/); - assert.doesNotMatch(brief, /trap \d/); + assert.doesNotMatch(settled(brief), /trap \d/); });