From 68da195ca6ebaaf1353a650142e2a820837ef339 Mon Sep 17 00:00:00 2001 From: kbkb628 <278338969+kbkb628@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:08:11 +0800 Subject: [PATCH] feat(shared): add run budget contract and runaway detectors Step 1 of #17: the runtime contract the agent loop can enforce, without wiring it into RunManager yet. - run-budget.ts: RunBudgetLimits / RunBudgetUsage over wall-clock, model calls, tool calls, search iterations, tokens and cost. resolveBudget applies defaults, then per-run overrides, always clamped by the system ceiling and reporting which keys it clamped; checkBudget reports the first exhausted key in a fixed order; budgetStop turns exhaustion into a machine-readable stop_reason with its detail. Invalid limits and negative usage deltas fail loudly instead of being silently ignored. - runaway-detector.ts: deterministic observers for repeated identical tool calls (argument order independent, with tool include/exclude scope), near-identical search queries (token overlap, so widening a query with more words still counts), iterations reporting the same evidence set, and retry storms inside a rolling window. State is plain data and every detection carries its evidence; runawayStop maps a detection to a stop reason. - 32 unit tests cover every budget key, the ceiling/override rules and each detector boundary. Wiring into RunManager, partial-result preservation and the real-provider E2E (a deliberately tiny budget plus an induced repeat-search case) follow in step 2. Refs #17 --- packages/shared/src/index.ts | 26 ++ packages/shared/src/kernel/index.ts | 30 ++ packages/shared/src/kernel/run-budget.test.ts | 119 ++++++ packages/shared/src/kernel/run-budget.ts | 200 ++++++++++ .../src/kernel/runaway-detector.test.ts | 260 +++++++++++++ .../shared/src/kernel/runaway-detector.ts | 366 ++++++++++++++++++ 6 files changed, 1001 insertions(+) create mode 100644 packages/shared/src/kernel/run-budget.test.ts create mode 100644 packages/shared/src/kernel/run-budget.ts create mode 100644 packages/shared/src/kernel/runaway-detector.test.ts create mode 100644 packages/shared/src/kernel/runaway-detector.ts diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 5d07a6a..cd924ad 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -87,8 +87,34 @@ export { AgentKernel, SessionManager, RunManager, + BUDGET_KEYS, + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, type AgentKernelOptions, type AgentProvider, + type BudgetExhaustion, + type BudgetKey, + type ResolveBudgetInput, + type ResolvedBudget, + type RunBudgetLimits, + type RunBudgetUsage, + type RunStop, + type RunawayDetection, + type RunawayPolicy, + type RunawaySignal, + type RunawayState, + type RunawayStep, + type StopReason, } from './kernel/index.ts'; export { LocalFinanceAgentBackend, diff --git a/packages/shared/src/kernel/index.ts b/packages/shared/src/kernel/index.ts index 1afdcb5..ac6f223 100644 --- a/packages/shared/src/kernel/index.ts +++ b/packages/shared/src/kernel/index.ts @@ -1,3 +1,33 @@ export { AgentKernel, type AgentKernelOptions, type AgentProvider } from './agent-kernel.ts'; export { SessionManager, type SessionManagerOptions } from './session-manager.ts'; export { RunManager, type RunManagerOptions } from './run-manager.ts'; +export { + BUDGET_KEYS, + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + type BudgetExhaustion, + type BudgetKey, + type ResolveBudgetInput, + type ResolvedBudget, + type RunBudgetLimits, + type RunBudgetUsage, + type RunStop, + type StopReason, +} from './run-budget.ts'; +export { + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, + type RunawayDetection, + type RunawayPolicy, + type RunawaySignal, + type RunawayState, + type RunawayStep, +} from './runaway-detector.ts'; diff --git a/packages/shared/src/kernel/run-budget.test.ts b/packages/shared/src/kernel/run-budget.test.ts new file mode 100644 index 0000000..99926b5 --- /dev/null +++ b/packages/shared/src/kernel/run-budget.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'bun:test'; +import { + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + type RunBudgetUsage, +} from './run-budget.ts'; + +describe('createUsage', () => { + it('starts every budget key at zero', () => { + expect(createUsage()).toEqual({ + wallClockMs: 0, + modelCalls: 0, + toolCalls: 0, + searchIterations: 0, + inputTokens: 0, + outputTokens: 0, + costUsd: 0, + }); + }); +}); + +describe('resolveBudget', () => { + it('applies overrides on top of defaults', () => { + const { limits, clamped } = resolveBudget({ + defaults: { modelCalls: 20, toolCalls: 30 }, + overrides: { modelCalls: 50 }, + }); + + expect(limits).toEqual({ modelCalls: 50, toolCalls: 30 }); + expect(clamped).toEqual([]); + }); + + it('lets a run tighten a limit below the default', () => { + const { limits } = resolveBudget({ defaults: { toolCalls: 30 }, overrides: { toolCalls: 5 } }); + + expect(limits.toolCalls).toBe(5); + }); + + it('clamps an override to the system ceiling and reports the clamped key', () => { + const { limits, clamped } = resolveBudget({ + defaults: { modelCalls: 20 }, + overrides: { modelCalls: 1000, costUsd: 999 }, + ceiling: { modelCalls: 100, costUsd: 5 }, + }); + + expect(limits).toEqual({ modelCalls: 100, costUsd: 5 }); + expect(clamped).toEqual(['modelCalls', 'costUsd']); + }); + + it('leaves a limit unlimited when no default, override or ceiling sets it', () => { + const { limits } = resolveBudget({ defaults: { modelCalls: 20 } }); + + expect(limits.toolCalls).toBeUndefined(); + expect(limits.costUsd).toBeUndefined(); + }); + + it('clamps a default that already exceeds the ceiling', () => { + const { limits, clamped } = resolveBudget({ defaults: { modelCalls: 100 }, ceiling: { modelCalls: 10 } }); + + expect(limits.modelCalls).toBe(10); + expect(clamped).toEqual(['modelCalls']); + }); + + it('fails loudly on a non-positive or non-finite limit', () => { + expect(() => resolveBudget({ overrides: { modelCalls: 0 } })).toThrow(/modelCalls/); + expect(() => resolveBudget({ overrides: { toolCalls: -1 } })).toThrow(/toolCalls/); + expect(() => resolveBudget({ overrides: { costUsd: Number.POSITIVE_INFINITY } })).toThrow(/costUsd/); + expect(() => resolveBudget({ overrides: { wallClockMs: Number.NaN } })).toThrow(/wallClockMs/); + }); +}); + +describe('addUsage', () => { + it('accumulates deltas without mutating the input usage', () => { + const before = createUsage(); + const after = addUsage(before, { modelCalls: 2, costUsd: 0.5 }); + + expect(after.modelCalls).toBe(2); + expect(after.costUsd).toBe(0.5); + expect(before.modelCalls).toBe(0); + expect(before.costUsd).toBe(0); + }); + + it('fails loudly on a negative delta', () => { + expect(() => addUsage(createUsage(), { toolCalls: -1 })).toThrow(/toolCalls/); + }); +}); + +describe('checkBudget', () => { + const usage: RunBudgetUsage = { ...createUsage(), modelCalls: 5, toolCalls: 3 }; + + it('reports the first exhausted key in a deterministic order', () => { + const exhaustion = checkBudget({ toolCalls: 3, modelCalls: 5 }, usage); + + // wall-clock and the call counters are checked before token/cost keys, so the + // model-call limit wins even though the tool-call limit is listed first above. + expect(exhaustion).toEqual({ key: 'modelCalls', limit: 5, used: 5 }); + }); + + it('treats a limit as exhausted once usage reaches it', () => { + expect(checkBudget({ modelCalls: 6 }, usage)).toBeUndefined(); + expect(checkBudget({ modelCalls: 5 }, usage)).toEqual({ key: 'modelCalls', limit: 5, used: 5 }); + }); + + it('ignores keys with no limit and usage that stays under budget', () => { + expect(checkBudget({ costUsd: 10, toolCalls: 10 }, usage)).toBeUndefined(); + }); +}); + +describe('budgetStop', () => { + it('turns an exhaustion into a machine-readable stop reason with its detail', () => { + const stop = budgetStop({ key: 'searchIterations', limit: 8, used: 8 }); + + expect(stop.stopReason).toBe('budget_exhausted'); + expect(stop.detail).toEqual({ key: 'searchIterations', limit: 8, used: 8 }); + }); +}); diff --git a/packages/shared/src/kernel/run-budget.ts b/packages/shared/src/kernel/run-budget.ts new file mode 100644 index 0000000..6e5faf3 --- /dev/null +++ b/packages/shared/src/kernel/run-budget.ts @@ -0,0 +1,200 @@ +/** + * Run budget contract for Agent / Deep Research runs. + * + * A run's ceiling is a contract, not a workflow-local constant: defaults come + * from the application, a run may override them, and a system-level ceiling + * always wins. Exhaustion surfaces as a machine-readable `StopReason` plus the + * key that ran out, so traces, UI and evaluation can explain why a run stopped + * instead of reporting it as an ordinary success. + */ + +/** Every dimension a run budget can constrain. */ +export type BudgetKey = + | 'wallClockMs' + | 'modelCalls' + | 'toolCalls' + | 'searchIterations' + | 'inputTokens' + | 'outputTokens' + | 'costUsd'; + +/** + * Budget keys in check order: wall-clock first, then the counters the runtime + * observes directly, then provider-reported usage. `checkBudget` reports the + * first exhausted key in this order, so the same state always yields the same + * reason. + */ +export const BUDGET_KEYS: readonly BudgetKey[] = [ + 'wallClockMs', + 'modelCalls', + 'toolCalls', + 'searchIterations', + 'inputTokens', + 'outputTokens', + 'costUsd', +]; + +/** Upper bounds for a run; an absent key is unlimited. */ +export type RunBudgetLimits = Partial>; + +/** + * Consumption accumulated so far; every key is always present. + * + * `wallClockMs` is absolute elapsed time since the run started, not a delta, so + * a caller recomputes it from the run's start timestamp at each check; every + * other key accumulates through {@link addUsage}. + */ +export type RunBudgetUsage = Record; + +/** The key that ran out, with the numbers needed to explain it. */ +export interface BudgetExhaustion { + key: BudgetKey; + limit: number; + used: number; +} + +/** + * Why a run stopped. Machine-readable so traces, run summaries and evaluation + * can branch on it; `completed` is the only success value. + */ +export type StopReason = + | 'completed' + | 'budget_exhausted' + | 'loop_detected' + | 'retry_storm' + | 'cancelled' + | 'error'; + +/** A run outcome paired with the detail behind a non-success reason. */ +export interface RunStop { + stopReason: StopReason; + detail?: Record; +} + +/** Resolution result: the limits a run must obey, and any override the ceiling cut down. */ +export interface ResolvedBudget { + limits: RunBudgetLimits; + clamped: BudgetKey[]; +} + +/** How a caller asks for effective limits. */ +export interface ResolveBudgetInput { + defaults?: RunBudgetLimits; + overrides?: RunBudgetLimits; + ceiling?: RunBudgetLimits; +} + +/** Zeroed usage; the starting point of every run. */ +export function createUsage(): RunBudgetUsage { + return { + wallClockMs: 0, + modelCalls: 0, + toolCalls: 0, + searchIterations: 0, + inputTokens: 0, + outputTokens: 0, + costUsd: 0, + }; +} + +/** + * Reject a limit the runtime cannot enforce. Fail-loud by design: a silently + * ignored budget is worse than a refused run, because it looks enforced. + * @param key - the budget dimension being validated. + * @param value - the requested limit. + */ +function assertLimit(key: BudgetKey, value: number): void { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + throw new Error( + `run-budget: ${key} must be a positive finite number, received ${String(value)}`, + ); + } +} + +/** + * Resolve the limits a run must obey: defaults, then per-run overrides, always + * clamped by the system ceiling. + * @param input - defaults, overrides and the ceiling. + * @returns the effective limits and the keys the ceiling cut down. + */ +export function resolveBudget(input: ResolveBudgetInput): ResolvedBudget { + const limits: RunBudgetLimits = {}; + const clamped: BudgetKey[] = []; + + for (const key of BUDGET_KEYS) { + const ceiling = input.ceiling?.[key]; + const override = input.overrides?.[key]; + const fallback = input.defaults?.[key]; + if (ceiling !== undefined) assertLimit(key, ceiling); + if (override !== undefined) assertLimit(key, override); + if (fallback !== undefined) assertLimit(key, fallback); + + const requested = override ?? fallback; + if (requested === undefined) { + if (ceiling !== undefined) limits[key] = ceiling; + continue; + } + + const effective = ceiling === undefined ? requested : Math.min(requested, ceiling); + if (effective !== requested) clamped.push(key); + limits[key] = effective; + } + + return { limits, clamped }; +} + +/** + * Accumulate one step's consumption without mutating the previous usage, so a + * run can report the usage of an abandoned branch of work. + * @param usage - usage accumulated so far. + * @param delta - what this step consumed. + * @returns a new usage record. + */ +export function addUsage(usage: RunBudgetUsage, delta: Partial): RunBudgetUsage { + const next: RunBudgetUsage = { ...usage }; + + for (const key of BUDGET_KEYS) { + const value = delta[key]; + if (value === undefined) continue; + if (!Number.isFinite(value) || value < 0) { + throw new Error( + `run-budget: ${key} delta must be a non-negative finite number, received ${String(value)}`, + ); + } + next[key] = usage[key] + value; + } + + return next; +} + +/** + * Whether a run has spent its budget. + * @param limits - effective limits; absent keys are unlimited. + * @param usage - consumption so far. + * @returns the first exhausted key in {@link BUDGET_KEYS} order, else undefined. + */ +export function checkBudget( + limits: RunBudgetLimits, + usage: RunBudgetUsage, +): BudgetExhaustion | undefined { + for (const key of BUDGET_KEYS) { + const limit = limits[key]; + if (limit === undefined) continue; + if (usage[key] >= limit) return { key, limit, used: usage[key] }; + } + + return undefined; +} + +/** + * Turn an exhaustion into the run's stop reason, keeping the numbers in the + * detail so a summary can say which budget ran out and by how much. + * @param exhaustion - the exhausted budget key. + * @returns a `budget_exhausted` stop with its detail. + */ +export function budgetStop(exhaustion: BudgetExhaustion): RunStop { + return { + stopReason: 'budget_exhausted', + detail: { key: exhaustion.key, limit: exhaustion.limit, used: exhaustion.used }, + }; +} diff --git a/packages/shared/src/kernel/runaway-detector.test.ts b/packages/shared/src/kernel/runaway-detector.test.ts new file mode 100644 index 0000000..8be9b4f --- /dev/null +++ b/packages/shared/src/kernel/runaway-detector.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from 'bun:test'; +import { + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, +} from './runaway-detector.ts'; + +describe('defaultRunawayPolicy', () => { + it('ships thresholds that need several consecutive signals before firing', () => { + expect(defaultRunawayPolicy()).toEqual({ + repeatedToolCallThreshold: 5, + repeatedSearchQueryThreshold: 5, + querySimilarity: 0.9, + noProgressIterations: 3, + retryWindowMs: 60_000, + retryThreshold: 4, + toolCallInclude: [], + toolCallExclude: [], + }); + }); +}); + +describe('policy validation', () => { + it('refuses a threshold that would fire on the first signal', () => { + expect(() => + observeToolCall(createRunawayState(), { tool: 'bash', args: {} }, { repeatedToolCallThreshold: 1 }), + ).toThrow(/repeatedToolCallThreshold/); + expect(() => observeEvidence(createRunawayState(), [], { noProgressIterations: 0 })).toThrow( + /noProgressIterations/, + ); + expect(() => observeRetry(createRunawayState(), 0, { retryThreshold: -1 })).toThrow(/retryThreshold/); + }); + + it('refuses an out-of-range similarity or window', () => { + expect(() => observeSearchQuery(createRunawayState(), 'x', { querySimilarity: 0 })).toThrow( + /querySimilarity/, + ); + expect(() => observeSearchQuery(createRunawayState(), 'x', { querySimilarity: 1.5 })).toThrow( + /querySimilarity/, + ); + expect(() => observeRetry(createRunawayState(), 0, { retryWindowMs: 0 })).toThrow(/retryWindowMs/); + }); +}); + +describe('observeToolCall', () => { + const call = (command: string) => ({ tool: 'bash', args: { command } }); + + it('fires once the same tool call repeats up to the threshold', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 5 }; + let state = createRunawayState(); + const detections = []; + + for (let i = 0; i < 5; i += 1) { + const step = observeToolCall(state, call('ls'), policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 4).every((detection) => !detection.detected)).toBe(true); + expect(detections[4].detected).toBe(true); + expect(detections[4].signal).toBe('repeated_tool_call'); + expect(detections[4].evidence).toMatchObject({ tool: 'bash', count: 5 }); + }); + + it('resets the run when the arguments change', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 3 }; + let state = createRunawayState(); + + for (const command of ['ls', 'ls', 'pwd']) { + state = observeToolCall(state, call(command), policy).state; + } + const afterReset = observeToolCall(state, call('pwd'), policy); + + expect(afterReset.detection.detected).toBe(false); + }); + + it('treats arguments that differ only in key order as the same call', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 2 }; + const first = observeToolCall(createRunawayState(), { tool: 'read', args: { a: 1, b: 2 } }, policy); + const second = observeToolCall(first.state, { tool: 'read', args: { b: 2, a: 1 } }, policy); + + expect(second.detection.detected).toBe(true); + }); + + it('does not mutate the state it was given', () => { + const state = createRunawayState(); + observeToolCall(state, call('ls'), defaultRunawayPolicy()); + + expect(state.toolCall).toBeUndefined(); + }); +}); + +describe('observeSearchQuery', () => { + it('fires on near-duplicate queries even when the wording changes', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 3 }; + let state = createRunawayState(); + const queries = ['NVIDIA earnings 2026 Q2', 'NVIDIA Q2 2026 earnings!', 'nvidia earnings, q2 2026']; + const detections = []; + + for (const query of queries) { + const step = observeSearchQuery(state, query, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[2].detected).toBe(true); + expect(detections[2].signal).toBe('repeated_search_query'); + expect(detections[2].evidence).toMatchObject({ count: 3 }); + }); + + it('resets the run on a genuinely different query', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 2 }; + const first = observeSearchQuery(createRunawayState(), 'NVIDIA earnings 2026 Q2', policy); + const second = observeSearchQuery(first.state, 'apple dividend history 2019', policy); + + expect(second.detection.detected).toBe(false); + }); +}); + +describe('observeEvidence', () => { + it('fires after consecutive iterations that add no new evidence', () => { + const policy = { ...defaultRunawayPolicy(), noProgressIterations: 3 }; + let state = createRunawayState(); + const iterations = [['a'], ['a'], ['a', 'b'], ['a', 'b'], ['a', 'b']]; + const detections = []; + + for (const ids of iterations) { + const step = observeEvidence(state, ids, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[3].detected).toBe(false); + expect(detections[4].detected).toBe(true); + expect(detections[4].signal).toBe('no_new_evidence'); + expect(detections[4].evidence).toMatchObject({ iterations: 3 }); + }); +}); + +describe('observeRetry', () => { + it('fires when retries bunch up inside the window', () => { + const policy = { ...defaultRunawayPolicy(), retryThreshold: 4, retryWindowMs: 60_000 }; + let state = createRunawayState(); + const detections = []; + + for (const at of [0, 1_000, 2_000, 3_000]) { + const step = observeRetry(state, at, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 3).every((detection) => !detection.detected)).toBe(true); + expect(detections[3].detected).toBe(true); + expect(detections[3].signal).toBe('retry_storm'); + expect(detections[3].evidence).toMatchObject({ retriesInWindow: 4, windowMs: 60_000 }); + }); + + it('forgets retries that fell out of the window', () => { + const policy = { ...defaultRunawayPolicy(), retryThreshold: 3, retryWindowMs: 10_000 }; + let state = createRunawayState(); + + for (const at of [0, 60_000, 120_000]) { + state = observeRetry(state, at, policy).state; + } + const step = observeRetry(state, 180_000, policy); + + expect(step.detection.detected).toBe(false); + }); +}); + +describe('tool call scope', () => { + it('never fires for an excluded tool, so polling a job is not a loop', () => { + const policy = { + ...defaultRunawayPolicy(), + repeatedToolCallThreshold: 2, + toolCallExclude: ['job_output'], + }; + let state = createRunawayState(); + + for (let i = 0; i < 4; i += 1) { + const step = observeToolCall(state, { tool: 'job_output', args: { id: 'j1' } }, policy); + state = step.state; + expect(step.detection.detected).toBe(false); + } + }); + + it('tracks only the listed tools when an include list is given', () => { + const policy = { + ...defaultRunawayPolicy(), + repeatedToolCallThreshold: 2, + toolCallInclude: ['bash*'], + }; + const read1 = observeToolCall(createRunawayState(), { tool: 'read', args: {} }, policy); + const read2 = observeToolCall(read1.state, { tool: 'read', args: {} }, policy); + expect(read2.detection.detected).toBe(false); + + const bash1 = observeToolCall(read2.state, { tool: 'bash', args: {} }, policy); + const bash2 = observeToolCall(bash1.state, { tool: 'bash', args: {} }, policy); + expect(bash2.detection.detected).toBe(true); + }); +}); + +describe('observeSearchQuery overlap', () => { + it('keeps counting a search that only widens with more words', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 3 }; + let state = createRunawayState(); + const queries = [ + 'nvidia earnings 2026', + 'nvidia earnings 2026 q2', + 'nvidia earnings 2026 q2 guidance', + ]; + const detections = []; + + for (const query of queries) { + const step = observeSearchQuery(state, query, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[2].detected).toBe(true); + }); + + it('treats an empty query as a reset rather than a false loop', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 2 }; + const first = observeSearchQuery(createRunawayState(), 'nvidia earnings 2026', policy); + const empty = observeSearchQuery(first.state, ' ', policy); + + expect(empty.detection.detected).toBe(false); + }); +}); + +describe('runawayStop', () => { + it('maps a loop detection to loop_detected and keeps the evidence', () => { + const stop = runawayStop({ + detected: true, + signal: 'repeated_tool_call', + evidence: { tool: 'bash', count: 5 }, + }); + + expect(stop.stopReason).toBe('loop_detected'); + expect(stop.detail).toEqual({ signal: 'repeated_tool_call', tool: 'bash', count: 5 }); + }); + + it('gives a retry storm its own stop reason', () => { + const stop = runawayStop({ detected: true, signal: 'retry_storm', evidence: { retriesInWindow: 4 } }); + + expect(stop.stopReason).toBe('retry_storm'); + }); + + it('refuses to build a stop from a non-detection', () => { + expect(() => runawayStop({ detected: false })).toThrow(/detection/); + }); +}); diff --git a/packages/shared/src/kernel/runaway-detector.ts b/packages/shared/src/kernel/runaway-detector.ts new file mode 100644 index 0000000..1e0e9a6 --- /dev/null +++ b/packages/shared/src/kernel/runaway-detector.ts @@ -0,0 +1,366 @@ +/** + * Deterministic runaway detectors for Agent / Deep Research runs. + * + * Each detector observes one signal and returns both the next state and a + * decision, so a caller can accumulate state per run and still replay it: the + * state is plain data, and every decision carries the numbers behind it. The + * detectors never call a model — classification stays explainable, and a + * provider is free to be missing or flaky without breaking the guard. + */ + +import type { RunStop, StopReason } from './run-budget.ts'; + +/** The failure modes a run can get stuck in. */ +export type RunawaySignal = + | 'repeated_tool_call' + | 'repeated_search_query' + | 'no_new_evidence' + | 'retry_storm'; + +/** A detector's decision; `evidence` is what a trace or run summary should record. */ +export interface RunawayDetection { + detected: boolean; + signal?: RunawaySignal; + evidence?: Record; +} + +/** Thresholds and scope shared by every detector of one run. */ +export interface RunawayPolicy { + /** Consecutive identical tool calls (same tool, same arguments) that count as a loop. */ + repeatedToolCallThreshold: number; + /** Consecutive near-identical search queries that count as a loop. */ + repeatedSearchQueryThreshold: number; + /** + * Token overlap at or above which two queries are the same search. Overlap is + * `|shared| / min(|a|, |b|)`, so a query that only widens with extra words + * still counts as the same search while an unrelated query does not. + */ + querySimilarity: number; + /** Consecutive iterations reporting the same evidence set before it counts as no progress. */ + noProgressIterations: number; + /** Rolling window over which retries are counted. */ + retryWindowMs: number; + /** Retries inside the window that count as a storm. */ + retryThreshold: number; + /** Tool name patterns to track; empty means every tool. `*` is a wildcard. */ + toolCallInclude: string[]; + /** Tool name patterns never tracked, whatever the include list says. */ + toolCallExclude: string[]; +} + +/** Accumulated per-run detector state: plain data, safe to persist and replay. */ +export interface RunawayState { + /** The last tool call's canonical key and how many times it repeated in a row. */ + toolCall?: { key: string; count: number }; + /** The last search's token set and how many times it repeated in a row. */ + searchQuery?: { tokens: string[]; count: number }; + /** The last evidence set (sorted ids) and how many times it repeated in a row. */ + evidence?: { key: string; iterations: number }; + /** Retry timestamps still inside the window. */ + retries: number[]; +} + +/** One detector's outcome plus the state to carry into the next observation. */ +export interface RunawayStep { + state: RunawayState; + detection: RunawayDetection; +} + +/** + * Thresholds that need several consecutive signals before firing, so a single + * retry or a repeated query that is genuinely new evidence does not stop a run. + * + * `toolCallExclude` is empty by default, but polling tools such as a job-status + * reader are legitimate repeats and belong there for a workflow that uses them. + * @returns a fresh policy with the default thresholds. + */ +export function defaultRunawayPolicy(): RunawayPolicy { + return { + repeatedToolCallThreshold: 5, + repeatedSearchQueryThreshold: 5, + querySimilarity: 0.9, + noProgressIterations: 3, + retryWindowMs: 60_000, + retryThreshold: 4, + toolCallInclude: [], + toolCallExclude: [], + }; +} + +/** Fresh detector state for a new run. */ +export function createRunawayState(): RunawayState { + return { retries: [] }; +} + +/** + * Reject a counting threshold the runtime cannot honour. Fail-loud by design: a + * threshold of 1 looks like a guard but fires on the first signal. + * @param key - policy field name, for the error message. + * @param value - requested value. + */ +function assertThreshold(key: string, value: number): void { + if (!Number.isInteger(value) || value < 2) { + throw new Error( + `runaway-detector: ${key} must be an integer >= 2, received ${String(value)}`, + ); + } +} + +/** + * Validate a resolved policy before any state is touched. + * @param policy - the merged policy. + */ +function assertPolicy(policy: RunawayPolicy): void { + assertThreshold('repeatedToolCallThreshold', policy.repeatedToolCallThreshold); + assertThreshold('repeatedSearchQueryThreshold', policy.repeatedSearchQueryThreshold); + assertThreshold('noProgressIterations', policy.noProgressIterations); + assertThreshold('retryThreshold', policy.retryThreshold); + if (!Number.isFinite(policy.querySimilarity) || policy.querySimilarity <= 0 || policy.querySimilarity > 1) { + throw new Error( + `runaway-detector: querySimilarity must be in (0, 1], received ${String(policy.querySimilarity)}`, + ); + } + if (!Number.isFinite(policy.retryWindowMs) || policy.retryWindowMs <= 0) { + throw new Error( + `runaway-detector: retryWindowMs must be a positive finite number, received ${String(policy.retryWindowMs)}`, + ); + } +} + +/** Merge a partial policy over the defaults and validate the result. */ +function withDefaults(policy: Partial): RunawayPolicy { + const resolved = { ...defaultRunawayPolicy(), ...policy }; + assertPolicy(resolved); + return resolved; +} + +/** + * Canonical string form of a tool call's arguments. Property order must not + * matter, so keys are sorted deeply before stringifying; a raw-string fallback + * (malformed argument JSON) is parsed first so both paths share one key. + * @param args - tool arguments, parsed or raw. + * @returns a stable key. + */ +function canonicalArguments(args: unknown): string { + if (typeof args === 'string') { + try { + return JSON.stringify(sortJson(JSON.parse(args))); + } catch { + return JSON.stringify(args); + } + } + return JSON.stringify(sortJson(args)) ?? String(args); +} + +/** Deep key-sort so two argument objects differing only in property order match. */ +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (value !== null && typeof value === 'object') { + const sorted: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortJson((value as Record)[key]); + } + return sorted; + } + return value; +} + +/** Compile one tool-name pattern; every regex metacharacter except `*` is literal. */ +function wildcardToRegExp(pattern: string): RegExp { + const escaped = pattern.replace(/[|\\{}()[\]^$+?.]/g, String.raw`\$&`); + return new RegExp(`^${escaped.replaceAll('*', '.*')}$`); +} + +/** + * Whether a tool participates in loop detection. Untracked tools are + * transparent: they neither count nor reset the chain, so polling through an + * excluded tool cannot hide a loop on a tracked one. + * @param policy - the resolved policy. + * @param tool - the tool being called. + * @returns true when this tool is tracked. + */ +function trackedTool(policy: RunawayPolicy, tool: string): boolean { + if (policy.toolCallInclude.length > 0 && !policy.toolCallInclude.some((pattern) => wildcardToRegExp(pattern).test(tool))) { + return false; + } + return !policy.toolCallExclude.some((pattern) => wildcardToRegExp(pattern).test(tool)); +} + +/** + * Normalize a search query into a sorted, unique token set. Unicode letters and + * numbers survive, so CJK queries tokenize the same way Latin ones do. + * @param query - the raw query text. + * @returns sorted unique tokens. + */ +function tokenize(query: string): string[] { + const tokens = query + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter((token) => token !== ''); + return [...new Set(tokens)].sort(); +} + +/** + * Token overlap of two queries: shared tokens over the smaller set. Unlike + * Jaccard this stays at 1 when one query merely adds words, which is the common + * shape of a search loop, while an unrelated query still scores near zero. + * @param a - first token set. + * @param b - second token set. + * @returns overlap in [0, 1]; 0 when either set is empty. + */ +function overlap(a: readonly string[], b: readonly string[]): number { + const left = new Set(a); + const right = new Set(b); + if (left.size === 0 || right.size === 0) return 0; + let shared = 0; + for (const token of left) if (right.has(token)) shared += 1; + return shared / Math.min(left.size, right.size); +} + +/** + * Observe one tool call and advance the identical-call chain. + * @param state - detector state so far. + * @param call - the tool name and its arguments. + * @param policy - threshold and scope overrides. + * @returns next state and whether the run is looping on this call. + */ +export function observeToolCall( + state: RunawayState, + call: { tool: string; args: unknown }, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + if (!trackedTool(resolved, call.tool)) return { state, detection: { detected: false } }; + + const canonical = canonicalArguments(call.args); + const key = `${call.tool}\u0000${canonical}`; + const count = state.toolCall?.key === key ? state.toolCall.count + 1 : 1; + const next: RunawayState = { ...state, toolCall: { key, count } }; + + if (count < resolved.repeatedToolCallThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'repeated_tool_call', + evidence: { tool: call.tool, count, canonicalArguments: canonical }, + }, + }; +} + +/** + * Observe a search query and advance the near-duplicate query chain. + * @param state - detector state so far. + * @param query - the raw query text. + * @param policy - threshold overrides. + * @returns next state and whether the run is re-running the same search. + */ +export function observeSearchQuery( + state: RunawayState, + query: string, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const tokens = tokenize(query); + const previous = state.searchQuery; + const score = previous === undefined ? 0 : overlap(previous.tokens, tokens); + const repeated = previous !== undefined && score >= resolved.querySimilarity; + const count = repeated ? previous.count + 1 : 1; + // A repeat keeps the cluster's original token set, so overlap is measured + // against the search the run started looping on rather than drifting wording. + const cluster = repeated ? previous.tokens : tokens; + const next: RunawayState = { ...state, searchQuery: { tokens: cluster, count } }; + + if (count < resolved.repeatedSearchQueryThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'repeated_search_query', + evidence: { query, count, similarity: score }, + }, + }; +} + +/** + * Observe the evidence retrieved by one iteration. + * @param state - detector state so far. + * @param evidenceIds - ids retrieved by this iteration. + * @param policy - threshold overrides. + * @returns next state and whether the run stopped making progress. + */ +export function observeEvidence( + state: RunawayState, + evidenceIds: readonly string[], + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const key = [...new Set(evidenceIds)].sort().join('\u0000'); + const iterations = state.evidence?.key === key ? state.evidence.iterations + 1 : 1; + const next: RunawayState = { ...state, evidence: { key, iterations } }; + + if (iterations < resolved.noProgressIterations) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'no_new_evidence', + evidence: { iterations, evidenceIds: [...evidenceIds] }, + }, + }; +} + +/** + * Observe one retry and count the retries still inside the rolling window. + * @param state - detector state so far. + * @param at - retry timestamp in milliseconds. + * @param policy - threshold overrides. + * @returns next state and whether the run is in a retry storm. + */ +export function observeRetry( + state: RunawayState, + at: number, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const retries = [...state.retries.filter((time) => at - time < resolved.retryWindowMs), at]; + const next: RunawayState = { ...state, retries }; + + if (retries.length < resolved.retryThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'retry_storm', + evidence: { retriesInWindow: retries.length, windowMs: resolved.retryWindowMs }, + }, + }; +} + +/** + * Turn a detection into the run's stop reason, mirroring `budgetStop` so a run + * has one shape of outcome whatever stopped it. + * @param detection - a detection reported by one of the observers. + * @returns a `loop_detected` or `retry_storm` stop carrying the evidence. + */ +export function runawayStop(detection: RunawayDetection): RunStop { + if (!detection.detected || detection.signal === undefined) { + throw new Error('runaway-detector: cannot build a run stop from a non-detection'); + } + + const stopReason: StopReason = detection.signal === 'retry_storm' ? 'retry_storm' : 'loop_detected'; + + return { stopReason, detail: { signal: detection.signal, ...detection.evidence } }; +}