diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index 6b0f1ba..5d4d64d 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -16,6 +16,7 @@ tests: - "worker/test/readonly.test.ts" - "worker/test/feedback.test.ts" - "worker/test/confirm.test.ts" + - "worker/test/dispute.test.ts" - "tests/test_usefulness.py" --- # Hosted Worker @@ -212,6 +213,15 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and claim rows the version guard reads. This is the only way a session that was shown a candidate can help accept it: its re-claim is an echo, and the claim reply says so. The Python arbiter doesn't count ratings, so local and hosted promote differently here. +- A `wrong` rating disputes its claim. The rule is one SQL fragment, `DISPUTED` in `store.ts`: some + session's newest writable verdict is `wrong` and no newest writable `used` came after it, in + (`created_at`, `rowid`) order. Unmarked rows are ignored when picking a session's newest, so a session + id reused with a read-only token can't lift its own dispute. Briefing queries exclude disputed claims + in SQL (`recall(..., { undisputed: true })`, both `recent*Candidates`), so they never use up a LIMIT; + a new briefing read must do the same. Recall marks them `⚠ a session rated this wrong` and `expand` + returns `rated_wrong: {at}` (`store.disputes`). The note is never shown to agents: it skips the write + gate. Keyed on the claim id, so a new version under the subject starts clean; no status changes. The + local store has no feedback, so local briefings don't drop them. - `initialize` instructions are built from `STEPS` in `mcp.ts`, keeping only the steps whose tool the caller can see (`visibleTools`), so a read-only or `KYPP_CORRECT=off` caller is never told to call a tool it lacks. diff --git a/worker/src/mcp.ts b/worker/src/mcp.ts index 012a723..4f0ec14 100644 --- a/worker/src/mcp.ts +++ b/worker/src/mcp.ts @@ -5,7 +5,7 @@ import { type Claim, DEFAULT_POLICY, HUMAN_CORRECTION_CONFIDENCE, type Policy, T import { consolidate } from "./consolidate.ts"; import { fileClaim } from "./defrag.ts"; import { CONTROL_ABOVE, DROP_BELOW, type ProjectInfo, RERANK_POOL, type S1Client, judge, rankForQuery } from "./s1.ts"; -import { type Lane, type Renamed, type RoomScope, D1Store } from "./store.ts"; +import { type Dispute, type Lane, type Renamed, type RoomScope, D1Store } from "./store.ts"; import { roomIdentity } from "./rooms.ts"; export interface Caller { @@ -87,7 +87,7 @@ const TOOLS = [ }, { name: "feedback", - description: "Rate the claims this session was shown (by briefing, recall or expand), once, before you finish: `used` you acted on it and it held (say what you checked in `note`), `not_used` it was shown but didn't bear on the task, `wrong` it misled you (say how in `note`). Changes no claim; the ratings tune what memory shows. Rate again to change a verdict.", + description: "Rate the claims this session was shown (by briefing, recall or expand), once, before you finish: `used` you acted on it and it held (say what you checked in `note`), `not_used` it was shown but didn't bear on the task, `wrong` it misled you (say how in `note`). Changes no claim, but a `wrong` keeps it out of every briefing until another session rates it `used` or a new version replaces it. Rate again to change a verdict.", inputSchema: { type: "object", required: ["ratings"], properties: { @@ -167,9 +167,22 @@ const ROOM_TOOLS = [ type Args = Record; type Rpc = { jsonrpc: "2.0"; id?: string | number | null; method: string; params?: any }; -function claimDict(c: Claim) { +function claimDict(c: Claim, dispute?: Dispute) { const { id, type, subject, content, scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at } = c; - return { id, type, subject, content, scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at }; + return { id, type, subject, content, scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at, + ...(dispute ? { rated_wrong: dispute } : {}) }; +} + +// A claim a session rated `wrong` stays out of the briefing, which every session loads (its queries pass +// `undisputed`), until a later `used` clears it or a new version replaces it. Recall still shows it, marked. +async function markDisputed(store: D1Store, claims: Claim[]): Promise { + const d = await store.disputes(claims.map((c) => c.id)); + for (const c of claims) if (d.has(c.id)) c.disputed = true; +} + +async function expanded(store: D1Store, c: Claim): Promise { + const d = (await store.disputes([c.id])).get(c.id); + return JSON.stringify(claimDict(c, d), null, 2); } // Search pulls a wide pool and System One reranks it; ranking never changes what a caller may see. @@ -221,7 +234,7 @@ const NEAR_SUBJECTS = 3; async function orgRules(store: D1Store, groups: string[]): Promise<{ orgs: Set; org: Claim[] }> { const orgs = await store.orgs(groups); if (!orgs.size) return { orgs, org: [] }; - const org = await store.recall("", null, null, { candidates: "none", groups: groups.filter((g) => orgs.has(g)), limit: ORG_RULES }); + const org = await store.recall("", null, null, { candidates: "none", groups: groups.filter((g) => orgs.has(g)), limit: ORG_RULES, undisputed: true }); return { orgs, org: org.filter((c) => c.scope === "group" && orgs.has(c.project ?? "")) }; } @@ -264,7 +277,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key // Why a read of its own, as in a repo briefing: recall ranks the org last, so a busy room's // capped list would crowd its rules out. const [settled, { orgs, org }, yours] = await Promise.all([ - store.recall("", null, null, { room, candidates: "none", limit: limit * 3 }), + store.recall("", null, null, { room, candidates: "none", limit: limit * 3, undisputed: true }), orgRules(store, groups), store.recentUserCandidates(identity, ctx.session, RECENT_USER, "room"), ]); @@ -278,7 +291,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key const query = String(a.query ?? ""), limit = recallLimit(a.limit); const claims = await rankForQuery(ctx.s1, query, await store.recall(query, null, null, { room, candidates: a.include_candidates ? "all" : "origin", types: a.types, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit); - await store.recordUsage(ctx.session, [["results", claims]], "recall", null, a.query || null); + await Promise.all([markDisputed(store, claims), store.recordUsage(ctx.session, [["results", claims]], "recall", null, a.query || null)]); return renderClaims(claims); } case "claim": { @@ -312,7 +325,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key const c = await store.get(String(a.handle ?? ""), identity, ctx.session, room); if (!c) throw new Error(`unknown claim handle ${a.handle}`); await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); - return JSON.stringify(claimDict(c), null, 2); + return expanded(store, c); } } throw new Error(`unknown tool ${name}`); @@ -398,7 +411,7 @@ async function pinnedTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, p const visible = c && (c.scope === "project" ? c.project === null || names.includes(c.project) : c.scope === "group" ? groups.has(c.project ?? "") : true); if (!c || !visible) throw new Error(`unknown claim handle ${a.handle}`); await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); - return JSON.stringify(claimDict(c), null, 2); + return expanded(store, c); } // Why: an unsorted claim from a pinned token could be filed anywhere, so it must name its project. if (!project && name === "claim" && a.scope !== "user") throw new Error(`say which project this lesson is about: ${mayUse}`); @@ -417,7 +430,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, const limit = Math.min(a.limit ?? 12, 30); // An org's rules apply to every contributor, so they come first under their own heading. const [settled, { orgs, org }, recent, yours] = await Promise.all([ - store.recall("", project, user, { candidates: "none", groups, limit: limit * 3 }), + store.recall("", project, user, { candidates: "none", groups, limit: limit * 3, undisputed: true }), orgRules(store, groups), // Why: the briefing stays settled memory; others' unconfirmed claims get a short, labelled section. project ? store.recentCandidates(project, ctx.session, RECENT) : [], @@ -436,7 +449,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, const query = String(a.query ?? ""), limit = recallLimit(a.limit); const claims = await rankForQuery(ctx.s1, query, await store.recall(query, project, user, { candidates: a.include_candidates ? "all" : "origin", groups, types: a.types, agent: a.agent, limit: poolSize(ctx.s1, query, limit), session: ctx.session }), limit); - await store.recordUsage(ctx.session, [["results", claims]], "recall", project, a.query || null); + await Promise.all([markDisputed(store, claims), store.recordUsage(ctx.session, [["results", claims]], "recall", project, a.query || null)]); return note + renderClaims(claims); } case "claim": { @@ -484,7 +497,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, const c = await store.get(String(a.handle ?? ""), user, ctx.session); if (!c) throw new Error(`unknown claim handle ${a.handle}`); await store.recordUsage(ctx.session, [["expand", [c]]], "expand", c.project); - return JSON.stringify(claimDict(c), null, 2); + return expanded(store, c); } case "correct": { if (!ctx.correct) throw new Error("correct is turned off on this server; use claim instead"); diff --git a/worker/src/memory.ts b/worker/src/memory.ts index 1b74a76..ab36bb6 100644 --- a/worker/src/memory.ts +++ b/worker/src/memory.ts @@ -62,6 +62,7 @@ export interface Claim { verify?: string | null; // the operator's freshness command; set only through POST /v1/operator pointer?: boolean; // set by recall when a path, anchor or doc:// ref in the query matched the claim, not its words confirmations?: Confirmation[]; // set by consolidate: other sessions whose newest feedback on this claim is `used` + disputed?: boolean; // set by reads: a session rated it `wrong` and no later `used` cleared it } // A session that was shown the claim, acted on it, and rated it `used` with a note on what it checked. @@ -213,6 +214,7 @@ export function compactLine(c: Claim): string { if (body.length > CLIP) line += `… (expand ${c.id.slice(0, 8)} for full)`; const path = c.code_refs.find((r) => r && typeof r === "object" && typeof r.path === "string")?.path; if (path) line += ` → ${path}`; + if (c.disputed) line += " ⚠ a session rated this wrong; check it before relying on it"; return line; } diff --git a/worker/src/store.ts b/worker/src/store.ts index 55dde99..a4dafbb 100644 --- a/worker/src/store.ts +++ b/worker/src/store.ts @@ -112,9 +112,24 @@ const UNSORTED = "scope = 'project' AND project IS NULL"; // A session's verdict is its newest rating of the claim; it confirms only if that rating is `used` and marked writable. const CONFIRMING = "CASE WHEN json_extract(u.metadata, '$.writable') = 1 THEN u.score END"; +// A claim is disputed while some session's newest writable verdict on it is `wrong` and no session's +// newest writable verdict is a `used` made after that. Only writable ratings count, and unmarked rows +// are ignored when picking a session's newest, so a read-only rating can't lift a dispute. +// Verdict order is (created_at, rowid), so two ratings in one millisecond still have an order. +const RATED = (a: string) => `${a}.surface = 'feedback' AND json_extract(${a}.metadata, '$.writable') = 1`; +const LATER = (a: string, b: string) => `(${a}.created_at > ${b}.created_at OR (${a}.created_at = ${b}.created_at AND ${a}.rowid > ${b}.rowid))`; +const NEWEST = (a: string, n: string) => + `NOT EXISTS (SELECT 1 FROM claim_usages ${n} WHERE ${n}.claim_id = ${a}.claim_id AND ${n}.consumer = ${a}.consumer AND ${RATED(n)} AND ${LATER(n, a)})`; +const STANDING_WRONG = (t: string) => `SELECT w.created_at FROM claim_usages w WHERE w.claim_id = ${t}.id AND ${RATED("w")} AND w.score = -1` + + ` AND ${NEWEST("w", "wn")} AND NOT EXISTS (SELECT 1 FROM claim_usages u WHERE u.claim_id = w.claim_id AND ${RATED("u")}` + + ` AND u.score = 1 AND ${LATER("u", "w")} AND ${NEWEST("u", "un")})`; +const DISPUTED = (t: string) => `EXISTS (${STANDING_WRONG(t)})`; + /** A claim's level, keyed the way its group is: a project, a person's own scope, or a room's lane. */ export type Lane = { project: string } | { user: string } | { room: string }; /** An earlier claim of this session's with the same content under another subject, and its provenance. */ +/** A standing `wrong` rating on a claim: when it was made. */ +export interface Dispute { at: string } export interface Renamed { id: string; subject: string; stamped: boolean; echoOf: string[] } const laneSql = (at: Lane): [string, string] => "project" in at ? ["c.scope = 'project' AND c.project = ?", at.project] : "room" in at ? ["c.scope = 'room' AND c.user = ?", at.room] : ["c.scope = 'user' AND c.user = ?", at.user]; @@ -375,7 +390,8 @@ export class D1Store { // store.recall: accepted, nearer level (project, then its groups, then user, then global), confidence. // The id is last so equal rows come back in the same order on every engine. async recall(match: string, project: string | null, user: string | null, - opts: { candidates?: Candidates; groups?: string[]; types?: string[]; agent?: string; limit?: number; session?: string | null; room?: RoomScope } = {}): Promise { + opts: { candidates?: Candidates; groups?: string[]; types?: string[]; agent?: string; limit?: number; session?: string | null; room?: RoomScope; + undisputed?: boolean } = {}): Promise { match = match.trim(); const [where, params] = opts.room ? roomVisible("c", opts.room, opts.candidates ?? "origin") @@ -385,6 +401,8 @@ export class D1Store { let filter = screen + (types.length ? ` AND c.type IN (${types.map(() => "?").join(",")})` : ""); const fparams: unknown[] = [...sparams, ...types]; if (opts.agent) { filter += " AND c.agent = ?"; fparams.push(opts.agent); } + // Briefings leave disputed claims out in the query itself, so they can't use up its LIMIT. + if (opts.undisputed) filter += ` AND NOT ${DISPUTED("c")}`; // `groups` comes nearest first, so a parent's claim outranks a grandparent's. const groups = opts.room?.groups ?? opts.groups ?? []; const near = groups.length ? `1 + (CASE c.project ${groups.map((_, i) => `WHEN ? THEN ${i}`).join(" ")} END) * 0.01` : "1"; @@ -565,6 +583,16 @@ export class D1Store { return out; } + // Per claim, when its newest standing `wrong` was made, for the claims that are disputed. + async disputes(ids: string[]): Promise> { + if (!ids.length) return new Map(); + const { results } = await this.db.prepare( + `SELECT c.id, (SELECT MAX(created_at) FROM (${STANDING_WRONG("c")})) AS at` + + ` FROM memory_claims c JOIN json_each(?) j ON j.value = c.id WHERE ${DISPUTED("c")}`, + ).bind(JSON.stringify([...new Set(ids)])).all(); + return new Map(results.map((r) => [r.id as string, { at: r.at as string }])); + } + // Every group or org `project` rolls up to, nearest first, through parents of parents up to three // levels (a cycle in the table just stops). The roll-up table is the only thing that decides what a // project can see. @@ -594,7 +622,7 @@ export class D1Store { const [screen, sparams] = this.screen("memory_claims", session); const { results } = await this.db.prepare( `SELECT ${claimColumns()} FROM memory_claims WHERE status = 'candidate' AND scope = ? AND user = ? AND session IS NOT ?` - + screen + " ORDER BY created_at DESC LIMIT ?", + + screen + ` AND NOT ${DISPUTED("memory_claims")} ORDER BY created_at DESC LIMIT ?`, ).bind(scope, user, session, ...sparams, limit).all(); return results.map(hydrate); } @@ -604,7 +632,7 @@ export class D1Store { const [screen, sparams] = this.screen("memory_claims", session); const { results } = await this.db.prepare( `SELECT ${claimColumns()} FROM memory_claims WHERE status = 'candidate' AND scope = 'project' AND project = ? AND session IS NOT ?` - + screen + " ORDER BY created_at DESC LIMIT ?", + + screen + ` AND NOT ${DISPUTED("memory_claims")} ORDER BY created_at DESC LIMIT ?`, ).bind(project, session, ...sparams, limit).all(); return results.map(hydrate); } diff --git a/worker/test/dispute.test.ts b/worker/test/dispute.test.ts new file mode 100644 index 0000000..5db2e7d --- /dev/null +++ b/worker/test/dispute.test.ts @@ -0,0 +1,133 @@ +// A claim a writable session rated `wrong` leaves the briefing until a later `used` clears it or a new +// version replaces it; recall still shows it, marked. The rater's note is never shown to other agents. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { type Caller, handleMcp } from "../src/mcp.ts"; +import { memoryD1 } from "./d1.ts"; + +const OWNER: Caller = { user: "owner", agent: "claude" }; + +async function setup() { + const db = memoryD1(); + await db.prepare("INSERT INTO projects(name, description, created_at) VALUES('kypp', '', '2026-11-06T00:00:00Z')").run(); + return db; +} + +function agent(db: D1Database, session: string, caller: Caller = OWNER) { + return async (name: string, args: Record = {}) => { + const req = new Request("https://k/mcp", { method: "POST", headers: { "X-Kypp-Project": "kypp", "Mcp-Session-Id": session }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }) }); + const { result } = await (await handleMcp(req, db, caller, { s1: null })).json() as any; + return result.content[0].text as string; + }; +} + +// An accepted claim, as a human correction lands it. +async function accepted(db: D1Database, subject: string, content: string) { + return (await agent(db, "w")("correct", { subject, content, type: "pitfall" })).split("\n")[0].trim(); +} + +const WRONG = { verdict: "wrong", note: "the port moved to 8788" }; +const USED = { verdict: "used", note: "ran wrangler dev; it answered on 8787" }; + +test("a wrong rating takes the claim out of the briefing; recall and expand mark it without the note", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const r = agent(db, "r"); + assert.match(await r("briefing"), /wrangler port/); + await r("feedback", { ratings: [{ handle: id, ...WRONG }] }); + + const next = agent(db, "n"); + assert.doesNotMatch(await next("briefing"), /wrangler port/); + assert.match(await next("recall", { query: "wrangler port" }), /wrangler port .*⚠ a session rated this wrong; check it before relying on it$/); + const full = JSON.parse(await next("expand", { handle: id })); + assert.ok(full.rated_wrong.at); + assert.doesNotMatch(JSON.stringify(full), /8788/, "the note skipped the write gate, so no agent reads it"); + assert.equal(full.status, "accepted", "a rating changes no claim"); +}); + +test("a later used from another session clears the dispute", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const a = agent(db, "a"), b = agent(db, "b"); + await a("briefing"); await b("briefing"); + await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); + assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); + await b("feedback", { ratings: [{ handle: id, ...USED }] }); + assert.match(await agent(db, "y")("briefing"), /wrangler port/); + assert.doesNotMatch(await agent(db, "y")("recall", { query: "wrangler port" }), /rated this wrong/); +}); + +test("a used made before the wrong doesn't clear it, and the rater withdrawing does", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const a = agent(db, "a"), b = agent(db, "b"); + await a("briefing"); await b("briefing"); + await b("feedback", { ratings: [{ handle: id, ...USED }] }); + await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); + assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); + await a("feedback", { ratings: [{ handle: id, verdict: "not_used" }] }); + assert.match(await agent(db, "y")("briefing"), /wrangler port/); +}); + +test("a wrong from a caller that can't write at the claim's level hides nothing", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const ro = agent(db, "ro", { ...OWNER, readOnly: true }); + await ro("briefing"); + await ro("feedback", { ratings: [{ handle: id, ...WRONG }] }); + assert.match(await agent(db, "x")("briefing"), /wrangler port/); +}); + +test("a new version under the subject is not disputed by the old one's rating", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const a = agent(db, "a"); + await a("briefing"); + await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); + await accepted(db, "wrangler port", "wrangler dev listens on 8788"); + const brief = await agent(db, "x")("briefing"); + assert.match(brief, /wrangler port — wrangler dev listens on 8788/); + assert.doesNotMatch(brief, /rated this wrong/); +}); + +test("a later rating from the same session without write access doesn't lift its dispute", async () => { + const db = await setup(); + const id = await accepted(db, "wrangler port", "wrangler dev listens on 8787"); + const a = agent(db, "a"), aRo = agent(db, "a", { ...OWNER, readOnly: true }); + await a("briefing"); + await a("feedback", { ratings: [{ handle: id, ...WRONG }] }); + await aRo("feedback", { ratings: [{ handle: id, ...USED }] }); + assert.doesNotMatch(await agent(db, "x")("briefing"), /wrangler port/); +}); + +test("disputed claims don't use up a section's limit: the briefing fills past them", async () => { + const db = await setup(); + const w = agent(db, "w"), a = agent(db, "a"); + await w("claim", { subject: "older lesson", content: "an older lesson about ports" }); + const newer: string[] = []; + for (const s of ["one", "two", "three"]) newer.push((await w("claim", { subject: `lesson ${s}`, content: `a lesson about ports ${s}` })).split("\n")[0]); + await a("recall", { query: "ports", include_candidates: true }); + await a("feedback", { ratings: newer.map((handle) => ({ handle, ...WRONG })) }); + const recent = (await agent(db, "x")("briefing")).split("# recent from other agents (unconfirmed)\n")[1] ?? ""; + assert.match(recent, /older lesson/, `the older undisputed candidate fills the section: ${recent}`); + assert.doesNotMatch(recent, /lesson (one|two|three)/); +}); + +test("the briefing finds undisputed memory past more disputed claims than one query returns", async () => { + const db = await setup(); + const disputed: string[] = []; + for (let i = 0; i < 55; i++) disputed.push(await accepted(db, `trap ${i}`, `a trap about ports, number ${i}`)); + const keep = await accepted(db, "a lesson that held", "this one was never rated wrong"); + // The undisputed claim is the oldest, so it ranks after every disputed one. + await db.prepare("UPDATE memory_claims SET updated_at = '2000-01-01T00:00:00Z' WHERE id = ?").bind(keep).run(); + // Session "a" was shown every claim, so it may rate them. + await db.prepare("INSERT INTO claim_usages(id,consumer,claim_id,project,scope,surface,created_at)" + + " SELECT 'shown-' || id, 'a', id, 'kypp', 'project', 'recall', '2026-11-07T00:00:00Z' FROM memory_claims").run(); + const a = agent(db, "a"); + await a("feedback", { ratings: disputed.slice(0, 50).map((handle) => ({ handle, ...WRONG })) }); + await a("feedback", { ratings: disputed.slice(50).map((handle) => ({ handle, ...WRONG })) }); + const brief = await agent(db, "x")("briefing", { limit: 30 }); + assert.match(brief, /a lesson that held/); + assert.doesNotMatch(brief, /trap \d/); +});