From 8e32703d540fae95c0933af65e3c8daf585aa23a Mon Sep 17 00:00:00 2001 From: jonnyparris <6400000+jonnyparris@users.noreply.github.com> Date: Mon, 25 May 2026 10:46:00 +0100 Subject: [PATCH] fix(own-loop): budget thresholds use projected context, not cumulative cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrap-up / hard-stop / warn thresholds were gating on `cumulativeInputTokens / tokenBudget` — the sum of input tokens billed across every iteration of the turn. That's a **cost** signal, not a **context window pressure** signal. The two coincide when the model has prompt caching (Anthropic) but diverge sharply for providers without it (Workers AI: Gemma, Kimi). For a Gemma session today the per-step `inputTokens` was ~20k regardless of how much real history existed (system prompt + tool defs dominate). By step 8 cumulative was 158k (77% of the 205k budget) and the wrap-up injection fired — telling the model to stop because it was 'nearly out of context'. But the actual next-call message array was ~20k, leaving 180k+ of context window free. ## Fix `budgetUsage` now measures `estimateMessagesTokens(messages) / tokenBudget` — the size of the message array we'd send NEXT, which is what the system prompt actually talks about ('context budget nearly exhausted'). `cumulativeInputTokens` is retained as telemetry (logged on every step-complete and threshold log) but no longer gates injections. ## Cost runaway backstop (new) Without cumulative-based gating, a model stuck in a non-doom-loop loop (tools succeeding, results changing, but no real progress) could burn unbounded tokens. Added an absolute backstop: when `cumulativeInputTokens >= tokenBudget * 5` (1M tokens for a 200k-budget model) the loop exits with a 'cost runaway' message. Generous for real multi-step work, tight enough to catch obvious billing bombs. ## What this unblocks Gemma sessions that previously stopped at step 8 with 'context exhausted' should now keep iterating. Projected context stays low because tool results are small; the model has plenty of room. ## Trade-off If projected (next-call) tokens stay below 70% but cumulative climbs forever, the cost runaway backstop catches it at 5× tokenBudget. Strong models with prompt caching won't notice the change (cumulative ≈ projected for them). Weak models with no caching get the fix's full benefit. ## Tests All 28 existing compaction + prune tests pass. Typecheck clean. The budget threshold logic isn't directly unit-tested (it lives inside onChatMessage's own-loop generator). Behavioural validation via post-deploy Gemma re-run on the failing prompts. --- src/coding-agent.ts | 55 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/src/coding-agent.ts b/src/coding-agent.ts index 3b3a45c..527ab04 100644 --- a/src/coding-agent.ts +++ b/src/coding-agent.ts @@ -1003,14 +1003,59 @@ export class CodingAgent extends Think { } } - // Budget-aware injections - const budgetUsage = cumulativeInputTokens / tokenBudget; + // ─── Cost runaway backstop ─── + // Per-call context pressure (below) is the primary stop signal, + // but a model stuck in a non-doom-loop loop (tools succeeding, + // results changing, but no real progress) could still burn + // unbounded tokens. Cap absolute cumulative input at 5× the + // context budget. For a 200k-context model that's 1M tokens — + // generous for real work, low enough to catch obvious runaway. + const COST_RUNAWAY_FACTOR = 5; + if (cumulativeInputTokens >= tokenBudget * COST_RUNAWAY_FACTOR) { + log("warn", "own-loop: cost runaway backstop", { + sessionId, + step, + cumulativeInputTokens, + tokenBudget, + factor: COST_RUNAWAY_FACTOR, + }); + yield { + type: "text-delta", + id: crypto.randomUUID(), + delta: `\n\n[Stopped: cumulative input tokens exceeded ${COST_RUNAWAY_FACTOR}× the context budget — likely loop]\n\n`, + }; + exitReason = "budget-limit"; + break; + } + + // ─── Budget-aware injections (context pressure, not cumulative cost) ─── + // + // The thresholds below tell the model when it's running out of + // **context window space**, not when it's spent a lot of tokens. + // These are different: cumulative input grows by ~system-prompt + // size every step regardless of how much real history the model + // is carrying, so gating on cumulative tells the model to bail + // when its actual context window has plenty of room. + // + // We estimate the size of the message array we'd send NEXT to + // streamText (system prompt + tools are added by the SDK on top; + // this only measures the messages-array portion). That's the + // signal that matches what the wrap-up text actually says + // ("nearly exhausted... do not read new files"). + // + // `cumulativeInputTokens` is kept for telemetry — logged on every + // step-complete line — but no longer gates injections. We + // discovered the previous coupling by deploying #75 + #76 and + // watching Gemma get told to stop at step 8 of a session where + // projected context was at 10%. See PR #77. + const projectedNextCallTokens = estimateMessagesTokens(messages); + const budgetUsage = projectedNextCallTokens / tokenBudget; if (budgetUsage >= HARD_STOP_THRESHOLD) { - // Hard stop — yield a wrap-up message and break - log("warn", "own-loop: hard stop — budget exhausted", { + log("warn", "own-loop: hard stop — projected context budget exhausted", { sessionId, step, + projectedNextCallTokens, cumulativeInputTokens, tokenBudget, usage: `${Math.round(budgetUsage * 100)}%`, @@ -1028,6 +1073,7 @@ export class CodingAgent extends Think { log("info", "own-loop: wrap-up injection", { sessionId, step, + projectedNextCallTokens, cumulativeInputTokens, tokenBudget, usage: `${Math.round(budgetUsage * 100)}%`, @@ -1041,6 +1087,7 @@ export class CodingAgent extends Think { log("info", "own-loop: budget warning injection", { sessionId, step, + projectedNextCallTokens, cumulativeInputTokens, tokenBudget, usage: `${Math.round(budgetUsage * 100)}%`,