diff --git a/src/coding-agent.ts b/src/coding-agent.ts index 527ab04..1a88531 100644 --- a/src/coding-agent.ts +++ b/src/coding-agent.ts @@ -55,6 +55,7 @@ import { type WatchdogConfig, } from "./watchdog"; import { shouldCompact, pickCutoff } from "./compaction-policy"; +import { detectSameToolRepetition } from "./loop-detection"; import { pruneOversizedToolResults } from "./own-loop-prune"; import { nextRetry } from "./overflow-retry"; import { assembleSystemPrompt } from "./prompt-composer"; @@ -283,9 +284,14 @@ const SYSTEM_PROMPT = [ "4. **Read only what you need.** After `explore` tells you which files and lines matter, use `read` with `offset`/`limit` to fetch only the sections you need to edit.", "5. **Plan, then edit.** State your plan in one short paragraph (or a todo list, preferred), then execute. Don't narrate each step.", "6. **Stay focused.** Only make changes that are directly requested or clearly necessary.", - "7. **Commit completed work only.** When you finish a coherent, working chunk in a git repo, stage and commit it before you reply unless the user explicitly says not to commit. Don't commit half-finished scaffolding or partial changes that would break the build. If your context runs out mid-task, commit what's complete and describe what remains.", - "8. **Delete unused code.** No commented-out code, no `_unused` renames.", - "9. **Be security-conscious.** Never commit secrets or credentials.", + "7. **Know when to stop.** When you have enough information to answer the user (or have completed the requested change), write your conclusion as a plain-text response and stop calling tools. Do NOT keep making speculative tool calls 'to be thorough' — every extra call costs context and the user is waiting. Specifically:", + " - If three consecutive tool calls have produced no new useful information, write what you have and stop.", + " - If you have answered the user's question, do not run more tools to 'verify'.", + " - If a tool keeps returning errors, write what failed and stop — don't retry with minor variations.", + " - Your final reply should always be plain text, never a tool call.", + "8. **Commit completed work only.** When you finish a coherent, working chunk in a git repo, stage and commit it before you reply unless the user explicitly says not to commit. Don't commit half-finished scaffolding or partial changes that would break the build. If your context runs out mid-task, commit what's complete and describe what remains.", + "9. **Delete unused code.** No commented-out code, no `_unused` renames.", + "10. **Be security-conscious.** Never commit secrets or credentials.", "", "## Workspace tools", "", @@ -862,6 +868,16 @@ export class CodingAgent extends Think { // OpenAI/Google/DeepSeek models work silently (tool calls without text narration). // The no-text detector is only useful for Anthropic models where silence means stuck. const noTextDetectionEnabled = modelId.startsWith("anthropic/"); + // ─── Same-tool repetition (softer than doom-loop) ─── + // The doom-loop detector requires *identical* tool+args calls. This + // catches the looser pattern of "same tool name, different args, N + // times in a row" which is how a model gets stuck speculatively + // calling codemode / explore / grep without ever wrapping up. + // Provider-agnostic — applies to every orchestrator. Nudge first, + // then hard-break if the model ignores the nudge. + const SAME_TOOL_NUDGE_THRESHOLD = 6; + const SAME_TOOL_HARD_BREAK_THRESHOLD = 10; + let sameToolNudgeInjected = false; // ─── Mid-loop compaction threshold ─── const MID_LOOP_COMPACTION_THRESHOLD = 0.50; // Compact when >50% of budget used @@ -1003,6 +1019,57 @@ export class CodingAgent extends Think { } } + // ─── Same-tool repetition detection (looser than doom-loop) ─── + // Catches the pattern where a model speculatively calls the same + // tool over and over with *different* args, never producing a + // text answer. Doom-loop misses this because each call's args + // differ. The harness intervenes in two stages: + // 1. Nudge at SAME_TOOL_NUDGE_THRESHOLD (default 6) — inject + // a system message asking the model whether it's done. + // 2. Hard-break at SAME_TOOL_HARD_BREAK_THRESHOLD (default 10) — + // if the model ignored the nudge and kept calling the same + // tool, we exit cleanly with a wrap-up message. + // Provider-agnostic — applies to every orchestrator regardless + // of whether silent tool-calling is normal for the provider. + // First observed on Gemma 4 26B running 18 consecutive codemode + // calls without ever writing a conclusion. + const hardBreakTool = detectSameToolRepetition( + recentToolCalls, + SAME_TOOL_HARD_BREAK_THRESHOLD, + ); + if (hardBreakTool) { + log("warn", "same-tool repetition hard break", { + sessionId, + toolName: hardBreakTool, + step, + repeats: SAME_TOOL_HARD_BREAK_THRESHOLD, + }); + yield { + type: "text-delta", + id: crypto.randomUUID(), + delta: `\n\n[Stopped: ${hardBreakTool} called ${SAME_TOOL_HARD_BREAK_THRESHOLD} times in a row without producing a text answer — write your conclusion from what you have so far.]\n\n`, + }; + exitReason = "doom-loop"; + break; + } + const nudgeTool = detectSameToolRepetition( + recentToolCalls, + SAME_TOOL_NUDGE_THRESHOLD, + ); + if (nudgeTool && !sameToolNudgeInjected) { + sameToolNudgeInjected = true; + injections.push({ + role: "system" as const, + content: `[STOP-CHECK] You've called \`${nudgeTool}\` ${SAME_TOOL_NUDGE_THRESHOLD} times in a row. Do you have enough information to answer the user? If yes: write your conclusion as plain text and do NOT call any more tools. If no: switch to a different tool or explain what's missing. Do not call \`${nudgeTool}\` again unless you have a concrete new question that requires it.`, + }); + log("info", "same-tool repetition nudge injected", { + sessionId, + toolName: nudgeTool, + step, + repeats: SAME_TOOL_NUDGE_THRESHOLD, + }); + } + // ─── Cost runaway backstop ─── // Per-call context pressure (below) is the primary stop signal, // but a model stuck in a non-doom-loop loop (tools succeeding, @@ -1386,9 +1453,17 @@ export class CodingAgent extends Think { for (const tc of lastStep.toolCalls) { recentToolCalls.push(`${tc.toolName}:${JSON.stringify(tc.input)}`); } - // Keep only the last DOOM_LOOP_THRESHOLD * 2 entries to bound memory - if (recentToolCalls.length > DOOM_LOOP_THRESHOLD * 2) { - recentToolCalls.splice(0, recentToolCalls.length - DOOM_LOOP_THRESHOLD * 2); + // Keep enough history to feed all the loop detectors that + // read from this buffer: the doom-loop check needs the last + // DOOM_LOOP_THRESHOLD * 2 entries; the same-tool hard-break + // check needs the last SAME_TOOL_HARD_BREAK_THRESHOLD. Pick + // the larger so neither detector starves. + const recentToolCallsRetention = Math.max( + DOOM_LOOP_THRESHOLD * 2, + SAME_TOOL_HARD_BREAK_THRESHOLD, + ); + if (recentToolCalls.length > recentToolCallsRetention) { + recentToolCalls.splice(0, recentToolCalls.length - recentToolCallsRetention); } } diff --git a/src/loop-detection.ts b/src/loop-detection.ts new file mode 100644 index 0000000..3ad2433 --- /dev/null +++ b/src/loop-detection.ts @@ -0,0 +1,46 @@ +/** + * Loop detection — pure decision functions over the own-loop's + * `recentToolCalls` history. + * + * The own-loop in `coding-agent.ts` tracks tool calls as + * `"toolName:argsJSON"` strings. These pure functions read that buffer + * and decide whether the model is in a loop pattern. + * + * Three signals, in increasing severity: + * + * 1. **Doom-loop** — identical tool+args repeated. Strong signal of + * stuck retry behaviour. Lives in `coding-agent.ts` already; not + * duplicated here. + * 2. **Same-tool repetition** — same tool name with *different* args + * repeated. Catches speculative-exploration loops (e.g. 10 codemode + * calls in a row with different code, no text answer). + * + * Each function is total (no side effects, no logging) so callers can + * compose them and react in their own way. + */ + +/** + * Did the last `threshold` tool calls all use the same tool *name* + * (regardless of args)? Pass entries as `"toolName:argsJSON"` strings, + * as the own-loop already stores them. + * + * Returns the repeated tool name, or null if the pattern isn't there. + * + * @param recentToolCalls Most recent first OR oldest first — only the + * tail is inspected, so ordering of older entries doesn't matter. + * @param threshold How many in a row must match before we say "yes". + * Must be at least 2. + */ +export function detectSameToolRepetition( + recentToolCalls: ReadonlyArray, + threshold: number, +): string | null { + if (threshold < 2) return null; + if (recentToolCalls.length < threshold) return null; + const lastN = recentToolCalls.slice(-threshold); + const firstName = lastN[0].split(":")[0]; + for (let i = 1; i < lastN.length; i++) { + if (lastN[i].split(":")[0] !== firstName) return null; + } + return firstName; +} diff --git a/test/loop-detection-unit.test.ts b/test/loop-detection-unit.test.ts new file mode 100644 index 0000000..d25ad93 --- /dev/null +++ b/test/loop-detection-unit.test.ts @@ -0,0 +1,111 @@ +/** + * Unit tests for `detectSameToolRepetition` — the looser-than-doom-loop + * detector that catches "same tool name, different args" patterns. + * + * Behaviour grid: + * + * - threshold < 2: always null (degenerate) + * - buffer shorter than threshold: null + * - last N entries all have the same tool name (any args): returns the name + * - mixed names in the tail window: null, even if earlier entries match + * - args don't matter — only the substring before the first `:` + */ +import { describe, expect, it } from "vitest"; +import { detectSameToolRepetition } from "../src/loop-detection"; + +const call = (name: string, args: unknown = {}) => `${name}:${JSON.stringify(args)}`; + +describe("detectSameToolRepetition", () => { + it("returns null for an empty buffer", () => { + expect(detectSameToolRepetition([], 6)).toBeNull(); + }); + + it("returns null when buffer is shorter than threshold", () => { + const buf = [call("codemode", { code: "1" }), call("codemode", { code: "2" })]; + expect(detectSameToolRepetition(buf, 6)).toBeNull(); + }); + + it("returns null for a degenerate threshold (<2)", () => { + const buf = [call("codemode"), call("codemode"), call("codemode")]; + expect(detectSameToolRepetition(buf, 1)).toBeNull(); + expect(detectSameToolRepetition(buf, 0)).toBeNull(); + }); + + it("returns the tool name when last N calls share the same name with different args", () => { + const buf = [ + call("explore", { q: "auth" }), + call("codemode", { code: "step1" }), + call("codemode", { code: "step2" }), + call("codemode", { code: "step3" }), + call("codemode", { code: "step4" }), + call("codemode", { code: "step5" }), + call("codemode", { code: "step6" }), + ]; + expect(detectSameToolRepetition(buf, 6)).toBe("codemode"); + }); + + it("returns null when the most recent call breaks the streak", () => { + const buf = [ + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + call("read", { path: "foo" }), + ]; + expect(detectSameToolRepetition(buf, 6)).toBeNull(); + }); + + it("ignores earlier entries outside the tail window", () => { + // The first 4 entries are mixed, but the last 6 are all `codemode`. + const buf = [ + call("read"), + call("explore"), + call("grep"), + call("read"), + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + ]; + expect(detectSameToolRepetition(buf, 6)).toBe("codemode"); + }); + + it("treats args as opaque — different args still count as same tool", () => { + const buf = [ + call("codemode", { code: 'return 1+1' }), + call("codemode", { code: 'return await env.WORKSPACE_BUCKET.list({ prefix: "x" })' }), + call("codemode", { code: 'return { a: 1, b: 2, nested: { c: [1, 2, 3] } }' }), + ]; + expect(detectSameToolRepetition(buf, 3)).toBe("codemode"); + }); + + it("handles tool names containing colons in args without confusion", () => { + // The split is on the FIRST `:` — args may contain JSON with colons. + const buf = [ + `codemode:{"code":"return 1"}`, + `codemode:{"code":"return 2"}`, + `codemode:{"code":"return 3"}`, + ]; + expect(detectSameToolRepetition(buf, 3)).toBe("codemode"); + }); + + it("works at exactly threshold size", () => { + const buf = [call("explore"), call("explore"), call("explore")]; + expect(detectSameToolRepetition(buf, 3)).toBe("explore"); + }); + + it("returns null when the streak is partial", () => { + // 5 codemode calls but threshold is 6 — should NOT trigger. + const buf = [ + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + call("codemode"), + ]; + expect(detectSameToolRepetition(buf, 6)).toBeNull(); + }); +});