diff --git a/docs/features/hosted-worker.md b/docs/features/hosted-worker.md index 5d4d64d..d7ba536 100644 --- a/docs/features/hosted-worker.md +++ b/docs/features/hosted-worker.md @@ -17,6 +17,7 @@ tests: - "worker/test/feedback.test.ts" - "worker/test/confirm.test.ts" - "worker/test/dispute.test.ts" + - "worker/test/evidence.test.ts" - "tests/test_usefulness.py" --- # Hosted Worker @@ -222,6 +223,11 @@ TypeSafe API's shape, for GitHub Actions OIDC tokens (`worker/src/oidc.ts`) and returns `rated_wrong: {at}` (`store.disputes`). The note is never shown to agents: it skips the write gate. Keyed on the claim id, so a new version under the subject starts clean; no status changes. The local store has no feedback, so local briefings don't drop them. +- A claim's `evidence` (how the writer knew it, at most `MAX_EVIDENCE` chars) lives in + `metadata.evidence`; `claimColumns` and `unjudged` lift it out as `Claim.evidence`, so a read that + builds its own column list has to add it too. Agents read it, so the write gate and `judgeLate` judge + `gatedText(content, evidence)`, never the content alone. Purge resets `metadata`, which erases it. + Local claims have no evidence yet. - `initialize` instructions are built from `STEPS` in `mcp.ts`, keeping only the steps whose tool the caller can see (`visibleTools`), so a read-only or `KYPP_CORRECT=off` caller is never told to call a tool it lacks. diff --git a/worker/src/defrag.ts b/worker/src/defrag.ts index ae1f57d..7370871 100644 --- a/worker/src/defrag.ts +++ b/worker/src/defrag.ts @@ -1,7 +1,7 @@ // defrag.ts — the hourly pass: judge claims the write gate missed, file unsorted claims into a known // project, then consolidate (supersede duplicates, promote subjects that recur). import { consolidate } from "./consolidate.ts"; -import { type Claim, type Policy, resolveOrigin } from "./memory.ts"; +import { type Claim, type Policy, gatedText, resolveOrigin } from "./memory.ts"; import { CONTROL_ABOVE, type ProjectInfo, type S1Client, fileUnder, judge } from "./s1.ts"; import type { D1Store } from "./store.ts"; @@ -26,7 +26,7 @@ export async function judgeLate(store: D1Store, s1: S1Client | null): Promise CONTROL_ABOVE) { if (await store.reject(c.id, `control text (p_control=${v.control.toFixed(2)})`)) rejected++; } else { diff --git a/worker/src/mcp.ts b/worker/src/mcp.ts index 4f0ec14..a51d959 100644 --- a/worker/src/mcp.ts +++ b/worker/src/mcp.ts @@ -1,7 +1,7 @@ // mcp.ts — kypp's agent tools as a stateless streamable-HTTP MCP endpoint (JSON responses, no SSE, // no Durable Object). The same contract as kypp/mcp_server.py's briefing / recall / claim / expand / // correct; consolidation is the scheduled pass in index.ts, not an agent tool. -import { type Claim, DEFAULT_POLICY, HUMAN_CORRECTION_CONFIDENCE, type Policy, TYPES, briefingOrder, renderBriefing, renderClaims, resolveOrigin } from "./memory.ts"; +import { type Claim, DEFAULT_POLICY, HUMAN_CORRECTION_CONFIDENCE, type Policy, TYPES, briefingOrder, gatedText, renderBriefing, renderClaims, resolveOrigin } from "./memory.ts"; import { consolidate } from "./consolidate.ts"; import { fileClaim } from "./defrag.ts"; import { CONTROL_ABOVE, DROP_BELOW, type ProjectInfo, RERANK_POOL, type S1Client, judge, rankForQuery } from "./s1.ts"; @@ -77,6 +77,7 @@ const TOOLS = [ scope: { type: "string", enum: ["project", "user"], default: "project", description: "Leave as project for anything about a repo; the server decides how widely it is shared. user = how this person works, in every repo." }, project: projectProp, code_refs: { type: "array", items: { type: "object" }, description: "[{symbol, path, query}] anchors" }, + evidence: { type: "string", description: "How you know it: the command you ran and what it showed, the file you read, the PR or doc. Leave it out for an untested guess." }, }, }, }, @@ -157,6 +158,7 @@ const ROOM_TOOLS = [ type: { type: "string", enum: TYPES, description: "Omit to let the server label it (defaults to fact)." }, confidence: { type: "number", default: 0.7 }, share_to: { type: "string", description: "A repo this room is linked to. Only when someone in the room asked to keep this lesson there." }, + evidence: { type: "string", description: "How you know it: the command you ran and what it showed, the file you read, the PR or doc. Leave it out for an untested guess." }, }, }, }, @@ -169,7 +171,7 @@ type Rpc = { jsonrpc: "2.0"; id?: string | number | null; method: string; params function claimDict(c: Claim, dispute?: Dispute) { const { id, type, subject, content, scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at } = c; - return { id, type, subject, content, scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at, + return { id, type, subject, content, ...(c.evidence ? { evidence: c.evidence } : {}), scope, project, status, authority, confidence, source_ids, code_refs, agent, user, created_at, ...(dispute ? { rated_wrong: dispute } : {}) }; } @@ -189,6 +191,14 @@ async function expanded(store: D1Store, c: Claim): Promise { // With a model, recall searches a wider pool than it returns, so the reranker can lift a relevant // claim the keyword order put past `limit`; without one, the pool is the answer. The model scores at // most RERANK_POOL claims, so a reranked recall returns at most that many. +// How the writer knew a claim, kept short: one line of what was run or read, never a log. +const MAX_EVIDENCE = 500; +function evidenceArg(v: unknown): string | null { + if (v === undefined || v === null) return null; + const t = String(v).trim(); + if (t.length > MAX_EVIDENCE) throw new Error(`evidence is ${t.length} chars; keep it to ${MAX_EVIDENCE}: what you ran or read and what it showed`); + return t || null; +} const recallLimit = (v: unknown) => Math.min(Math.max(Number(v) || 10, 1), 50); const poolSize = (s1: S1Client | null, query: string, limit: number) => (s1 && s1.rerank !== false && query.trim() ? RERANK_POOL : limit); @@ -296,6 +306,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key } case "claim": { const subject = String(a.subject ?? ""), content = String(a.content ?? ""); + const evidence = evidenceArg(a.evidence); const share = a.share_to === undefined || a.share_to === null || a.share_to === "" ? null : String(a.share_to); // A lesson leaves the room only by an explicit share into a linked origin. if (share !== null && !links.includes(share)) { @@ -303,7 +314,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key } const at: Lane = share !== null ? { project: share } : { room: identity }; // The model's verdict and the provenance reads don't depend on each other. - const [v, { prior, echoOf, stamp }] = await Promise.all([judge(ctx.s1, { subject, content, project: share }), + const [v, { prior, echoOf, stamp }] = await Promise.all([judge(ctx.s1, { subject, content: gatedText(content, evidence), project: share }), provenance(store, at, subject, content, ctx.session)]); if (v.keep !== null && v.keep < (ctx.s1?.dropBelow ?? DROP_BELOW)) { return `Not stored: this reads as status or conversation detail rather than a durable lesson (p_keep=${v.keep.toFixed(2)}).`; @@ -312,7 +323,7 @@ async function roomTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, key return `Not stored: this reads as instructions aimed at other agents rather than a lesson (p_control=${v.control.toFixed(2)}).`; } const s1 = v.keep === null && v.control === null ? undefined : { keep: v.keep, general: v.general, type: v.type, control: v.control }; - const metadata: Record = { ...(s1 ? { s1 } : {}), ...(echoOf.length ? { echo_of: echoOf } : {}), ...(share !== null ? { shared_from: identity } : {}) }; + const metadata: Record = { ...(s1 ? { s1 } : {}), ...(evidence ? { evidence } : {}), ...(echoOf.length ? { echo_of: echoOf } : {}), ...(share !== null ? { shared_from: identity } : {}) }; const id = await store.claim({ type: a.type ?? v.type ?? "fact", subject, content, scope: share !== null ? "project" : "room", project: share, origin: share ?? identity, session: ctx.session, @@ -454,6 +465,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, } case "claim": { const subject = String(a.subject ?? ""), content = String(a.content ?? ""); + const evidence = evidenceArg(a.evidence); // Why: the agent picks who a lesson is about (user), never how widely it applies, so global is refused. if (a.scope !== undefined && a.scope !== "project" && a.scope !== "user") { throw new Error(`bad scope ${JSON.stringify(a.scope)}: use "user" for how this person works, or leave it out; the server decides how widely a repo lesson is shared`); @@ -463,7 +475,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, const at = scope === "user" ? (user ? { user } : null) : project ? { project } : null; // A cheap model gates writes; it filters, labels and files, never accepts or widens a claim. // Its verdict and the provenance reads don't depend on each other. - const [v, { prior, echoOf, stamp }] = await Promise.all([judge(ctx.s1, { subject, content, project }), + const [v, { prior, echoOf, stamp }] = await Promise.all([judge(ctx.s1, { subject, content: gatedText(content, evidence), project }), at ? provenance(store, at, subject, content, ctx.session) : { prior: [] as Renamed[], echoOf: [] as string[], stamp: ctx.session ? [`session:${ctx.session}`] : [] }]); if (v.keep !== null && v.keep < (ctx.s1?.dropBelow ?? DROP_BELOW)) { @@ -474,7 +486,7 @@ async function repoTool(store: D1Store, name: string, a: Args, ctx: ToolCtx, return `Not stored: this reads as instructions aimed at other agents rather than a lesson about the code or the person (p_control=${v.control.toFixed(2)}).`; } const s1 = v.keep === null && v.control === null ? undefined : { keep: v.keep, general: v.general, type: v.type, control: v.control }; - const metadata: Record = { ...(s1 ? { s1 } : {}), ...(echoOf.length ? { echo_of: echoOf } : {}) }; + const metadata: Record = { ...(s1 ? { s1 } : {}), ...(evidence ? { evidence } : {}), ...(echoOf.length ? { echo_of: echoOf } : {}) }; // Why: the session stamp is what lets the scheduled pass's two-session gate count agreement; an echo gets none. const id = await store.claim({ type: a.type ?? v.type ?? "fact", subject, content, diff --git a/worker/src/memory.ts b/worker/src/memory.ts index ab36bb6..999941a 100644 --- a/worker/src/memory.ts +++ b/worker/src/memory.ts @@ -60,6 +60,7 @@ export interface Claim { created_at: string; updated_at: string; verify?: string | null; // the operator's freshness command; set only through POST /v1/operator + evidence?: string | null; // how the writer knew it: what they ran or read (metadata.evidence) pointer?: boolean; // set by recall when a path, anchor or doc:// ref in the query matched the claim, not its words confirmations?: Confirmation[]; // set by consolidate: other sessions whose newest feedback on this claim is `used` disputed?: boolean; // set by reads: a session rated it `wrong` and no later `used` cleared it @@ -202,9 +203,13 @@ export function planConsolidation(live: Claim[], policy: Policy = DEFAULT_POLICY return groupClaims(live).map((m) => planGroup(m, policy)).filter(hasWork); } +// What the write gate reads: the lesson and, when given, how it is known, since agents read both. +export const gatedText = (content: string, evidence?: string | null) => (evidence ? `${content}\n\nHow it is known: ${evidence}` : content); + // view.compact_line, minus code grounding (the Worker has no checkout to resolve anchors against; // it names the first anchored path instead). const CLIP = 240; +const EVIDENCE_CLIP = 100; const WHITESPACE = /\s+/; export function compactLine(c: Claim): string { const mark = c.status === "accepted" ? "✓" : "?"; @@ -214,6 +219,9 @@ export function compactLine(c: Claim): string { if (body.length > CLIP) line += `… (expand ${c.id.slice(0, 8)} for full)`; const path = c.code_refs.find((r) => r && typeof r === "object" && typeof r.path === "string")?.path; if (path) line += ` → ${path}`; + const how = (c.evidence ?? "").split(WHITESPACE).filter(Boolean).join(" "); + // A claim that says how it is known is marked apart from a bare assertion; expand gives the whole of it. + line += how ? ` [checked: ${how.slice(0, EVIDENCE_CLIP)}${how.length > EVIDENCE_CLIP ? "…" : ""}]` : ""; if (c.disputed) line += " ⚠ a session rated this wrong; check it before relying on it"; return line; } diff --git a/worker/src/store.ts b/worker/src/store.ts index a4dafbb..a02ae1a 100644 --- a/worker/src/store.ts +++ b/worker/src/store.ts @@ -39,7 +39,9 @@ const HANDLE = /^[0-9a-f]{8,32}$/; // The columns a Claim carries; metadata and file_tried_at stay in the table for the passes that need them. const CLAIM_COLUMNS = ["id", "type", "subject", "content", "scope", "project", "origin", "session", "agent", "user", "status", "authority", "confidence", "source_ids", "code_refs", "verify", "created_at", "updated_at"]; -const claimColumns = (t?: string) => CLAIM_COLUMNS.map((c) => (t ? `${t}.${c}` : c)).join(", "); +// How the writer knew the claim lives in metadata; reads lift it out so a hydrated Claim carries it. +const claimColumns = (t?: string) => [...CLAIM_COLUMNS.map((c) => (t ? `${t}.${c}` : c)), + `json_extract(${t ? `${t}.` : ""}metadata, '$.evidence') AS evidence`].join(", "); // A claim's key is a feature ref; a file path is a hint, and line numbers are not stored. function pointerRow(claimId: string, entryRef: string, path: string, symbol: string, kind: string, @@ -742,7 +744,7 @@ export class D1Store { // never-retried first, so defrag can judge them late. async unjudged(limit: number): Promise { const { results } = await this.db.prepare( - "SELECT * FROM memory_claims WHERE status IN ('candidate','accepted') AND authority = 'agent'" + "SELECT *, json_extract(metadata, '$.evidence') AS evidence FROM memory_claims WHERE status IN ('candidate','accepted') AND authority = 'agent'" + " AND json_extract(metadata, '$.s1') IS NULL" + " ORDER BY json_extract(metadata, '$.s1_tried') IS NOT NULL, json_extract(metadata, '$.s1_tried'), created_at LIMIT ?", ).bind(limit).all(); diff --git a/worker/test/evidence.test.ts b/worker/test/evidence.test.ts new file mode 100644 index 0000000..dd800c6 --- /dev/null +++ b/worker/test/evidence.test.ts @@ -0,0 +1,64 @@ +// A claim can say how it is known (`evidence`): reads mark it apart from a bare assertion, expand +// returns all of it, the write gate reads it with the lesson, and purge erases it. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { type Caller, handleMcp } from "../src/mcp.ts"; +import { gatedText } from "../src/memory.ts"; +import { D1Store } from "../src/store.ts"; +import { memoryD1 } from "./d1.ts"; + +const OWNER: Caller = { user: "owner", agent: "claude" }; + +async function setup() { + const db = memoryD1(); + await db.prepare("INSERT INTO projects(name, description, created_at) VALUES('kypp', '', '2026-11-06T00:00:00Z')").run(); + return db; +} + +function agent(db: D1Database, session: string) { + return async (name: string, args: Record = {}) => { + const req = new Request("https://k/mcp", { method: "POST", headers: { "X-Kypp-Project": "kypp", "Mcp-Session-Id": session }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }) }); + const { result } = await (await handleMcp(req, db, OWNER, { s1: null })).json() as any; + return { text: result.content[0].text as string, error: !!result.isError }; + }; +} + +const HOW = "ran `wrangler dev`; it answered on 8787"; + +test("a claim with evidence is marked in recall and expands to all of it", async () => { + const db = await setup(); + const w = agent(db, "w"); + const id = (await w("claim", { subject: "wrangler port", content: "wrangler dev listens on 8787", evidence: ` ${HOW} ` })).text.split("\n")[0]; + await w("claim", { subject: "deploy target", content: "the worker deploys on merge to main" }); + + const lines = (await agent(db, "r")("recall", { query: "wrangler deploys worker port", include_candidates: true })).text; + assert.match(lines, new RegExp(`wrangler port — wrangler dev listens on 8787 \\[checked: ${HOW.replace(/[`;]/g, ".")}\\]`)); + assert.match(lines, /deploy target — the worker deploys on merge to main$/m, "a bare claim carries no mark"); + const full = JSON.parse((await agent(db, "r")("expand", { handle: id })).text); + assert.equal(full.evidence, HOW); +}); + +test("long evidence is clipped on the line and refused past its limit", async () => { + const db = await setup(); + const w = agent(db, "w"); + const long = "x".repeat(150); + await w("claim", { subject: "s", content: "a lesson about ports", evidence: long }); + assert.match((await w("recall", { query: "ports", include_candidates: true })).text, /\[checked: x{100}…\]/); + const out = await w("claim", { subject: "t", content: "another lesson about ports", evidence: "y".repeat(501) }); + assert.equal(out.error, true); + assert.match(out.text, /keep it to 500/); +}); + +test("the write gate reads the evidence with the lesson", () => { + assert.equal(gatedText("lesson", null), "lesson"); + assert.equal(gatedText("lesson", "ran it"), "lesson\n\nHow it is known: ran it"); +}); + +test("purge erases the evidence", async () => { + const db = await setup(); + const id = (await agent(db, "w")("claim", { subject: "s", content: "token works", evidence: "curl with sk-123 returned 200" })).text.split("\n")[0]; + await new D1Store(db).operatorPurge(id, "leaked key"); + const row = await db.prepare("SELECT metadata FROM memory_claims WHERE id = ?").bind(id).first<{ metadata: string }>(); + assert.doesNotMatch(row!.metadata, /sk-123/); +}); diff --git a/worker/test/rooms.test.ts b/worker/test/rooms.test.ts index 3c6ce2e..3f09fed 100644 --- a/worker/test/rooms.test.ts +++ b/worker/test/rooms.test.ts @@ -239,7 +239,7 @@ test("a room is offered only its own tools", async () => { const tools = result.tools.map((t: any) => t.name); assert.deepEqual(tools, ["briefing", "recall", "claim", "expand", "feedback"]); const claim = result.tools.find((t: any) => t.name === "claim"); - assert.deepEqual(Object.keys(claim.inputSchema.properties).sort(), ["confidence", "content", "share_to", "subject", "type"]); + assert.deepEqual(Object.keys(claim.inputSchema.properties).sort(), ["confidence", "content", "evidence", "share_to", "subject", "type"]); assert.equal((await call(db, caller, "s1")("correct", { subject: "x", content: "y" })).error, true); });