diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0cc4058..beb62c5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -140,6 +140,19 @@ export interface Message { export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled'; +/** + * Why a run stopped. `completed` is the only success value: a run cut short by a + * budget or a runaway loop is not an ordinary answer, so telemetry, evaluation + * and the UI branch on this instead of treating every terminal run as success. + */ +export type StopReason = + | 'completed' + | 'budget_exhausted' + | 'loop_detected' + | 'retry_storm' + | 'cancelled' + | 'error'; + /** One agent execution inside a session. */ export interface Run { id: string; @@ -150,6 +163,10 @@ export interface Run { completedAt?: number; answer?: string; error?: ApiError; + /** Machine-readable reason the run stopped; absent on records written before #17. */ + stopReason?: StopReason; + /** The numbers behind a non-success stop (which budget ran out, which loop fired). */ + stopDetail?: Record; } /** Live tool call state, streamed through agent events. */ diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 5d07a6a..cd924ad 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -87,8 +87,34 @@ export { AgentKernel, SessionManager, RunManager, + BUDGET_KEYS, + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, type AgentKernelOptions, type AgentProvider, + type BudgetExhaustion, + type BudgetKey, + type ResolveBudgetInput, + type ResolvedBudget, + type RunBudgetLimits, + type RunBudgetUsage, + type RunStop, + type RunawayDetection, + type RunawayPolicy, + type RunawaySignal, + type RunawayState, + type RunawayStep, + type StopReason, } from './kernel/index.ts'; export { LocalFinanceAgentBackend, diff --git a/packages/shared/src/kernel/agent-kernel-budget.test.ts b/packages/shared/src/kernel/agent-kernel-budget.test.ts new file mode 100644 index 0000000..1f49974 --- /dev/null +++ b/packages/shared/src/kernel/agent-kernel-budget.test.ts @@ -0,0 +1,173 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import type { + AgentEvent, + AgentEventPayload, + AgentRunInput, + AgentRuntime, + ApiResult, + RuntimeSession, + ToolDefinition, +} from '@finagent/core'; +import { AgentKernel } from './agent-kernel.ts'; + +let dir = ''; +let clock = 1000; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'finagent-kernel-budget-')); + clock = 1000; +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +class ScriptedRuntime implements AgentRuntime { + cancelCalls: Array<{ sessionId: string; runId: string }> = []; + + constructor(private readonly script: (input: AgentRunInput) => AsyncIterable) {} + + async getTools(): Promise> { + return { ok: true, data: [] }; + } + + async ensureSession(session: { id: string }): Promise { + return { sessionId: session.id, status: 'active' }; + } + + async *run(input: AgentRunInput): AsyncIterable { + yield* this.script(input); + } + + async cancel(input: { sessionId: string; runId: string }): Promise { + this.cancelCalls.push(input); + } + + async dispose(): Promise {} +} + +function event( + sessionId: string, + runId: string, + type: AgentEvent['type'], + payload?: AgentEventPayload, + sequence = 1 +): AgentEvent { + return { + id: `evt-${type}-${sequence}`, + sessionId, + runId, + type, + timestamp: clock, + sequence, + ...(payload === undefined ? {} : { payload }), + } as unknown as AgentEvent; +} + +/** Two model calls, then a normal completion — over-budget only when a budget says so. */ +function twoStepScript(answer: string) { + return async function* (input: AgentRunInput) { + yield event(input.sessionId, input.runId, 'message_completed', { answer }, 1); + yield event(input.sessionId, input.runId, 'message_completed', { answer }, 2); + yield event(input.sessionId, input.runId, 'run_completed', { answer, toolCalls: [] }, 3); + }; +} + +describe('AgentKernel run budgets (#17)', () => { + it('forwards budget options to the run loop so a real run can be stopped', async () => { + const runtime = new ScriptedRuntime(twoStepScript('partial')); + const kernel = new AgentKernel({ + storageDir: dir, + piSessionDir: join(dir, 'pi-sessions'), + runtime, + now: () => clock, + budgets: { defaults: { modelCalls: 1 } }, + }); + const session = await kernel.sessions.createSession('Budget'); + + const run = await kernel.runs.startRun(session.id, 'q'); + await waitFor(async () => !kernel.runs.isRunning()); + + expect(runtime.cancelCalls).toEqual([{ sessionId: session.id, runId: run.id }]); + expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({ + status: 'cancelled', + answer: 'partial', + stopReason: 'budget_exhausted', + stopDetail: { key: 'modelCalls', limit: 1, used: 1 }, + }); + }); + + it('runs the same script to completion when no budget is configured', async () => { + const runtime = new ScriptedRuntime(twoStepScript('done')); + const kernel = new AgentKernel({ + storageDir: dir, + piSessionDir: join(dir, 'pi-sessions'), + runtime, + now: () => clock, + }); + const session = await kernel.sessions.createSession('Plain'); + + const run = await kernel.runs.startRun(session.id, 'q'); + await waitFor(async () => !kernel.runs.isRunning()); + + expect(runtime.cancelCalls).toEqual([]); + expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({ + status: 'completed', + answer: 'done', + }); + }); + + it('forwards runaway detector thresholds for repeated tool calls', async () => { + const runtime = new ScriptedRuntime(async function* (input) { + for (let i = 1; i <= 3; i += 1) { + yield event( + input.sessionId, + input.runId, + 'tool_completed', + { + toolCall: { + id: `t${i}`, + toolName: 'get_quote', + args: { symbol: 'AAPL.US' }, + startedAt: clock, + completedAt: clock, + status: 'success', + result: {}, + }, + }, + i + ); + } + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'done', toolCalls: [] }, 9); + }); + const kernel = new AgentKernel({ + storageDir: dir, + piSessionDir: join(dir, 'pi-sessions'), + runtime, + now: () => clock, + runaway: { repeatedToolCallThreshold: 2 }, + }); + const session = await kernel.sessions.createSession('Loop'); + + const run = await kernel.runs.startRun(session.id, 'q'); + await waitFor(async () => !kernel.runs.isRunning()); + + expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({ + status: 'cancelled', + stopReason: 'loop_detected', + }); + }); +}); + +async function waitFor(predicate: () => Promise, timeoutMs = 2000) { + const started = Date.now(); + while (!(await predicate())) { + if (Date.now() - started > timeoutMs) { + throw new Error('waitFor timed out'); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} diff --git a/packages/shared/src/kernel/agent-kernel.ts b/packages/shared/src/kernel/agent-kernel.ts index 8e6d4c2..a6e7eb8 100644 --- a/packages/shared/src/kernel/agent-kernel.ts +++ b/packages/shared/src/kernel/agent-kernel.ts @@ -12,6 +12,8 @@ import { createCodeError } from '../agent/errors.ts'; import type { PiRpcClientOptions } from '../agent/pi-rpc-client.ts'; import { SessionManager } from './session-manager.ts'; import { RunManager } from './run-manager.ts'; +import type { ResolveBudgetInput } from './run-budget.ts'; +import type { RunawayPolicy } from './runaway-detector.ts'; export type AgentProvider = 'local' | 'pi-runtime'; @@ -33,6 +35,15 @@ export interface AgentKernelOptions { /** Skill hub used for progressive skill loading in the runtime prompt. */ skillHub?: SkillHub; now?: () => number; + /** + * Budget defaults and the system ceiling every run obeys (#17). Without it + * runs are unbudgeted; a run may still tighten its own limits at startRun. + */ + budgets?: ResolveBudgetInput; + /** Tool-name patterns (`*` wildcard) whose `query` argument feeds the search-loop detector. */ + searchTools?: string[]; + /** Runaway detector thresholds; unset fields fall back to `defaultRunawayPolicy()`. */ + runaway?: Partial; } /** @@ -68,6 +79,9 @@ export class AgentKernel { runs: new RunRepository(store), runtime: this.runtime, now, + budgets: options.budgets, + searchTools: options.searchTools, + runaway: options.runaway, }); } diff --git a/packages/shared/src/kernel/index.ts b/packages/shared/src/kernel/index.ts index 1afdcb5..ac6f223 100644 --- a/packages/shared/src/kernel/index.ts +++ b/packages/shared/src/kernel/index.ts @@ -1,3 +1,33 @@ export { AgentKernel, type AgentKernelOptions, type AgentProvider } from './agent-kernel.ts'; export { SessionManager, type SessionManagerOptions } from './session-manager.ts'; export { RunManager, type RunManagerOptions } from './run-manager.ts'; +export { + BUDGET_KEYS, + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + type BudgetExhaustion, + type BudgetKey, + type ResolveBudgetInput, + type ResolvedBudget, + type RunBudgetLimits, + type RunBudgetUsage, + type RunStop, + type StopReason, +} from './run-budget.ts'; +export { + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, + type RunawayDetection, + type RunawayPolicy, + type RunawaySignal, + type RunawayState, + type RunawayStep, +} from './runaway-detector.ts'; diff --git a/packages/shared/src/kernel/run-budget.test.ts b/packages/shared/src/kernel/run-budget.test.ts new file mode 100644 index 0000000..99926b5 --- /dev/null +++ b/packages/shared/src/kernel/run-budget.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'bun:test'; +import { + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + type RunBudgetUsage, +} from './run-budget.ts'; + +describe('createUsage', () => { + it('starts every budget key at zero', () => { + expect(createUsage()).toEqual({ + wallClockMs: 0, + modelCalls: 0, + toolCalls: 0, + searchIterations: 0, + inputTokens: 0, + outputTokens: 0, + costUsd: 0, + }); + }); +}); + +describe('resolveBudget', () => { + it('applies overrides on top of defaults', () => { + const { limits, clamped } = resolveBudget({ + defaults: { modelCalls: 20, toolCalls: 30 }, + overrides: { modelCalls: 50 }, + }); + + expect(limits).toEqual({ modelCalls: 50, toolCalls: 30 }); + expect(clamped).toEqual([]); + }); + + it('lets a run tighten a limit below the default', () => { + const { limits } = resolveBudget({ defaults: { toolCalls: 30 }, overrides: { toolCalls: 5 } }); + + expect(limits.toolCalls).toBe(5); + }); + + it('clamps an override to the system ceiling and reports the clamped key', () => { + const { limits, clamped } = resolveBudget({ + defaults: { modelCalls: 20 }, + overrides: { modelCalls: 1000, costUsd: 999 }, + ceiling: { modelCalls: 100, costUsd: 5 }, + }); + + expect(limits).toEqual({ modelCalls: 100, costUsd: 5 }); + expect(clamped).toEqual(['modelCalls', 'costUsd']); + }); + + it('leaves a limit unlimited when no default, override or ceiling sets it', () => { + const { limits } = resolveBudget({ defaults: { modelCalls: 20 } }); + + expect(limits.toolCalls).toBeUndefined(); + expect(limits.costUsd).toBeUndefined(); + }); + + it('clamps a default that already exceeds the ceiling', () => { + const { limits, clamped } = resolveBudget({ defaults: { modelCalls: 100 }, ceiling: { modelCalls: 10 } }); + + expect(limits.modelCalls).toBe(10); + expect(clamped).toEqual(['modelCalls']); + }); + + it('fails loudly on a non-positive or non-finite limit', () => { + expect(() => resolveBudget({ overrides: { modelCalls: 0 } })).toThrow(/modelCalls/); + expect(() => resolveBudget({ overrides: { toolCalls: -1 } })).toThrow(/toolCalls/); + expect(() => resolveBudget({ overrides: { costUsd: Number.POSITIVE_INFINITY } })).toThrow(/costUsd/); + expect(() => resolveBudget({ overrides: { wallClockMs: Number.NaN } })).toThrow(/wallClockMs/); + }); +}); + +describe('addUsage', () => { + it('accumulates deltas without mutating the input usage', () => { + const before = createUsage(); + const after = addUsage(before, { modelCalls: 2, costUsd: 0.5 }); + + expect(after.modelCalls).toBe(2); + expect(after.costUsd).toBe(0.5); + expect(before.modelCalls).toBe(0); + expect(before.costUsd).toBe(0); + }); + + it('fails loudly on a negative delta', () => { + expect(() => addUsage(createUsage(), { toolCalls: -1 })).toThrow(/toolCalls/); + }); +}); + +describe('checkBudget', () => { + const usage: RunBudgetUsage = { ...createUsage(), modelCalls: 5, toolCalls: 3 }; + + it('reports the first exhausted key in a deterministic order', () => { + const exhaustion = checkBudget({ toolCalls: 3, modelCalls: 5 }, usage); + + // wall-clock and the call counters are checked before token/cost keys, so the + // model-call limit wins even though the tool-call limit is listed first above. + expect(exhaustion).toEqual({ key: 'modelCalls', limit: 5, used: 5 }); + }); + + it('treats a limit as exhausted once usage reaches it', () => { + expect(checkBudget({ modelCalls: 6 }, usage)).toBeUndefined(); + expect(checkBudget({ modelCalls: 5 }, usage)).toEqual({ key: 'modelCalls', limit: 5, used: 5 }); + }); + + it('ignores keys with no limit and usage that stays under budget', () => { + expect(checkBudget({ costUsd: 10, toolCalls: 10 }, usage)).toBeUndefined(); + }); +}); + +describe('budgetStop', () => { + it('turns an exhaustion into a machine-readable stop reason with its detail', () => { + const stop = budgetStop({ key: 'searchIterations', limit: 8, used: 8 }); + + expect(stop.stopReason).toBe('budget_exhausted'); + expect(stop.detail).toEqual({ key: 'searchIterations', limit: 8, used: 8 }); + }); +}); diff --git a/packages/shared/src/kernel/run-budget.ts b/packages/shared/src/kernel/run-budget.ts new file mode 100644 index 0000000..5af2b32 --- /dev/null +++ b/packages/shared/src/kernel/run-budget.ts @@ -0,0 +1,197 @@ +/** + * Run budget contract for Agent / Deep Research runs. + * + * A run's ceiling is a contract, not a workflow-local constant: defaults come + * from the application, a run may override them, and a system-level ceiling + * always wins. Exhaustion surfaces as a machine-readable `StopReason` plus the + * key that ran out, so traces, UI and evaluation can explain why a run stopped + * instead of reporting it as an ordinary success. + */ + +import type { StopReason } from '@finagent/core'; + +/** Every dimension a run budget can constrain. */ +export type BudgetKey = + | 'wallClockMs' + | 'modelCalls' + | 'toolCalls' + | 'searchIterations' + | 'inputTokens' + | 'outputTokens' + | 'costUsd'; + +/** + * Budget keys in check order: wall-clock first, then the counters the runtime + * observes directly, then provider-reported usage. `checkBudget` reports the + * first exhausted key in this order, so the same state always yields the same + * reason. + */ +export const BUDGET_KEYS: readonly BudgetKey[] = [ + 'wallClockMs', + 'modelCalls', + 'toolCalls', + 'searchIterations', + 'inputTokens', + 'outputTokens', + 'costUsd', +]; + +/** Upper bounds for a run; an absent key is unlimited. */ +export type RunBudgetLimits = Partial>; + +/** + * Consumption accumulated so far; every key is always present. + * + * `wallClockMs` is absolute elapsed time since the run started, not a delta, so + * a caller recomputes it from the run's start timestamp at each check; every + * other key accumulates through {@link addUsage}. + */ +export type RunBudgetUsage = Record; + +/** The key that ran out, with the numbers needed to explain it. */ +export interface BudgetExhaustion { + key: BudgetKey; + limit: number; + used: number; +} + +/** + * Why a run stopped. Owned by the core protocol because the UI, telemetry and + * evaluation read it off the persisted run record; re-exported here so callers + * that only work with budgets keep importing it from one place. + */ +export type { StopReason }; + +/** A run outcome paired with the detail behind a non-success reason. */ +export interface RunStop { + stopReason: StopReason; + detail?: Record; +} + +/** Resolution result: the limits a run must obey, and any override the ceiling cut down. */ +export interface ResolvedBudget { + limits: RunBudgetLimits; + clamped: BudgetKey[]; +} + +/** How a caller asks for effective limits. */ +export interface ResolveBudgetInput { + defaults?: RunBudgetLimits; + overrides?: RunBudgetLimits; + ceiling?: RunBudgetLimits; +} + +/** Zeroed usage; the starting point of every run. */ +export function createUsage(): RunBudgetUsage { + return { + wallClockMs: 0, + modelCalls: 0, + toolCalls: 0, + searchIterations: 0, + inputTokens: 0, + outputTokens: 0, + costUsd: 0, + }; +} + +/** + * Reject a limit the runtime cannot enforce. Fail-loud by design: a silently + * ignored budget is worse than a refused run, because it looks enforced. + * @param key - the budget dimension being validated. + * @param value - the requested limit. + */ +function assertLimit(key: BudgetKey, value: number): void { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + throw new Error( + `run-budget: ${key} must be a positive finite number, received ${String(value)}`, + ); + } +} + +/** + * Resolve the limits a run must obey: defaults, then per-run overrides, always + * clamped by the system ceiling. + * @param input - defaults, overrides and the ceiling. + * @returns the effective limits and the keys the ceiling cut down. + */ +export function resolveBudget(input: ResolveBudgetInput): ResolvedBudget { + const limits: RunBudgetLimits = {}; + const clamped: BudgetKey[] = []; + + for (const key of BUDGET_KEYS) { + const ceiling = input.ceiling?.[key]; + const override = input.overrides?.[key]; + const fallback = input.defaults?.[key]; + if (ceiling !== undefined) assertLimit(key, ceiling); + if (override !== undefined) assertLimit(key, override); + if (fallback !== undefined) assertLimit(key, fallback); + + const requested = override ?? fallback; + if (requested === undefined) { + if (ceiling !== undefined) limits[key] = ceiling; + continue; + } + + const effective = ceiling === undefined ? requested : Math.min(requested, ceiling); + if (effective !== requested) clamped.push(key); + limits[key] = effective; + } + + return { limits, clamped }; +} + +/** + * Accumulate one step's consumption without mutating the previous usage, so a + * run can report the usage of an abandoned branch of work. + * @param usage - usage accumulated so far. + * @param delta - what this step consumed. + * @returns a new usage record. + */ +export function addUsage(usage: RunBudgetUsage, delta: Partial): RunBudgetUsage { + const next: RunBudgetUsage = { ...usage }; + + for (const key of BUDGET_KEYS) { + const value = delta[key]; + if (value === undefined) continue; + if (!Number.isFinite(value) || value < 0) { + throw new Error( + `run-budget: ${key} delta must be a non-negative finite number, received ${String(value)}`, + ); + } + next[key] = usage[key] + value; + } + + return next; +} + +/** + * Whether a run has spent its budget. + * @param limits - effective limits; absent keys are unlimited. + * @param usage - consumption so far. + * @returns the first exhausted key in {@link BUDGET_KEYS} order, else undefined. + */ +export function checkBudget( + limits: RunBudgetLimits, + usage: RunBudgetUsage, +): BudgetExhaustion | undefined { + for (const key of BUDGET_KEYS) { + const limit = limits[key]; + if (limit === undefined) continue; + if (usage[key] >= limit) return { key, limit, used: usage[key] }; + } + + return undefined; +} + +/** + * Turn an exhaustion into the run's stop reason, keeping the numbers in the + * detail so a summary can say which budget ran out and by how much. + * @param exhaustion - the exhausted budget key. + * @returns a `budget_exhausted` stop with its detail. + */ +export function budgetStop(exhaustion: BudgetExhaustion): RunStop { + return { + stopReason: 'budget_exhausted', + detail: { key: exhaustion.key, limit: exhaustion.limit, used: exhaustion.used }, + }; +} diff --git a/packages/shared/src/kernel/run-manager.test.ts b/packages/shared/src/kernel/run-manager.test.ts index 6d13af6..e34d121 100644 --- a/packages/shared/src/kernel/run-manager.test.ts +++ b/packages/shared/src/kernel/run-manager.test.ts @@ -16,7 +16,7 @@ import { MessageRepository } from '../storage/message-repository.ts'; import { RunRepository } from '../storage/run-repository.ts'; import { SessionRepository } from '../storage/session-repository.ts'; import { SessionManager } from './session-manager.ts'; -import { RunManager } from './run-manager.ts'; +import { RunManager, type RunManagerOptions } from './run-manager.ts'; let dir = ''; let clock = 1000; @@ -30,7 +30,10 @@ afterEach(async () => { await rm(dir, { recursive: true, force: true }); }); -function makeKernel(script: (input: AgentRunInput) => AsyncIterable) { +function makeKernel( + script: (input: AgentRunInput) => AsyncIterable, + extra: Partial> = {} +) { const runtime = new ScriptedRuntime(script); const store = new JsonFileStore(dir); const sessions = new SessionManager({ @@ -45,6 +48,7 @@ function makeKernel(script: (input: AgentRunInput) => AsyncIterable) runs: new RunRepository(store), runtime, now: () => clock, + ...extra, }); return { runtime, sessions, runs, store }; } @@ -282,6 +286,159 @@ describe('RunManager', () => { }); }); +describe('RunManager budgets and runaway detection (#17)', () => { + const toolEvent = ( + input: AgentRunInput, + toolName: string, + args: Record, + sequence: number + ) => + event( + input.sessionId, + input.runId, + 'tool_completed', + { + toolCall: { + id: `t${sequence}`, + toolName, + args, + startedAt: clock, + completedAt: clock, + status: 'success', + result: {}, + }, + }, + sequence + ); + + it('stops a run that exceeds its model-call budget and keeps the partial answer', async () => { + const { sessions, runs, runtime } = makeKernel( + async function* (input) { + yield event(input.sessionId, input.runId, 'message_completed', { answer: 'partial one' }, 1); + yield event(input.sessionId, input.runId, 'message_completed', { answer: 'partial two' }, 2); + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'finished', toolCalls: [] }, 3); + }, + { budgets: { defaults: { modelCalls: 2 } } } + ); + const session = await sessions.createSession('Budget'); + + const run = await runs.startRun(session.id, 'long task'); + await waitFor(async () => !runs.isRunning()); + + expect(runtime.cancelCalls).toEqual([{ sessionId: session.id, runId: run.id }]); + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted).toMatchObject({ + status: 'cancelled', + answer: 'partial two', + stopReason: 'budget_exhausted', + stopDetail: { key: 'modelCalls', limit: 2, used: 2 }, + }); + const messages = await sessions.listMessages(session.id); + expect(messages[1]).toMatchObject({ role: 'assistant', content: 'partial two' }); + }); + + it('lets a run tighten its own budget but clamps it to the system ceiling', async () => { + const script = async function* (input: AgentRunInput) { + yield event(input.sessionId, input.runId, 'message_completed', { answer: 'one' }, 1); + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'one', toolCalls: [] }, 2); + }; + const { sessions, runs, runtime } = makeKernel(script, { + budgets: { defaults: { modelCalls: 10 }, ceiling: { modelCalls: 5 } }, + }); + const session = await sessions.createSession('Override'); + + const tightened = await runs.startRun(session.id, 'q', undefined, undefined, { modelCalls: 1 }); + await waitFor(async () => !runs.isRunning()); + expect(runtime.cancelCalls).toEqual([{ sessionId: session.id, runId: tightened.id }]); + expect(await sessions.getRun(session.id, tightened.id)).toMatchObject({ + stopReason: 'budget_exhausted', + stopDetail: { key: 'modelCalls', limit: 1, used: 1 }, + }); + + const clamped = await runs.startRun(session.id, 'q', undefined, undefined, { modelCalls: 100 }); + await waitFor(async () => !runs.isRunning()); + expect(runtime.cancelCalls).toHaveLength(1); + expect(await sessions.getRun(session.id, clamped.id)).toMatchObject({ status: 'completed' }); + }); + + it('stops on a repeated identical tool call and reports loop_detected', async () => { + const { sessions, runs, runtime } = makeKernel( + async function* (input) { + for (let i = 1; i <= 3; i += 1) { + yield toolEvent(input, 'get_quote', { symbol: 'AAPL.US' }, i); + } + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'done', toolCalls: [] }, 9); + }, + { runaway: { repeatedToolCallThreshold: 2 } } + ); + const session = await sessions.createSession('Loop'); + + const run = await runs.startRun(session.id, 'q'); + await waitFor(async () => !runs.isRunning()); + + expect(runtime.cancelCalls).toHaveLength(1); + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted).toMatchObject({ status: 'cancelled', stopReason: 'loop_detected' }); + expect(persisted?.stopDetail).toMatchObject({ + signal: 'repeated_tool_call', + tool: 'get_quote', + count: 2, + }); + }); + + it('stops on a repeated search query once search tools are configured', async () => { + const { sessions, runs } = makeKernel( + async function* (input) { + yield toolEvent(input, 'web_search', { query: 'nvidia earnings 2026' }, 1); + yield toolEvent(input, 'web_search', { query: 'nvidia earnings 2026 q2' }, 2); + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'done', toolCalls: [] }, 9); + }, + { searchTools: ['web_search*'], runaway: { repeatedSearchQueryThreshold: 2 } } + ); + const session = await sessions.createSession('Search'); + + const run = await runs.startRun(session.id, 'q'); + await waitFor(async () => !runs.isRunning()); + + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted).toMatchObject({ stopReason: 'loop_detected' }); + expect(persisted?.stopDetail).toMatchObject({ signal: 'repeated_search_query', count: 2 }); + }); + + it('stops on the wall-clock budget measured with the injected clock', async () => { + const { sessions, runs } = makeKernel( + async function* (input) { + clock += 5_000; + yield event(input.sessionId, input.runId, 'message_completed', { answer: 'slow step' }, 1); + yield event(input.sessionId, input.runId, 'run_completed', { answer: 'done', toolCalls: [] }, 2); + }, + { budgets: { defaults: { wallClockMs: 1_000 } } } + ); + const session = await sessions.createSession('Clock'); + + const run = await runs.startRun(session.id, 'q'); + await waitFor(async () => !runs.isRunning()); + + expect(await sessions.getRun(session.id, run.id)).toMatchObject({ + stopReason: 'budget_exhausted', + stopDetail: { key: 'wallClockMs', limit: 1_000, used: 5_000 }, + }); + }); + + it('leaves a run untouched when neither budgets nor detectors are configured', async () => { + const { sessions, runs, runtime } = makeKernel(completedScript('Answer')); + const session = await sessions.createSession('Plain'); + + const run = await runs.startRun(session.id, 'q'); + await waitFor(async () => !runs.isRunning()); + + expect(runtime.cancelCalls).toEqual([]); + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted).toMatchObject({ status: 'completed', answer: 'Answer' }); + expect(persisted?.stopReason).toBeUndefined(); + }); +}); + async function waitFor(predicate: () => Promise, timeoutMs = 2000) { const started = Date.now(); while (!(await predicate())) { diff --git a/packages/shared/src/kernel/run-manager.ts b/packages/shared/src/kernel/run-manager.ts index 91621eb..b552cde 100644 --- a/packages/shared/src/kernel/run-manager.ts +++ b/packages/shared/src/kernel/run-manager.ts @@ -15,18 +15,56 @@ import type { import type { RunRepository } from '../storage/index.ts'; import type { SessionManager } from './session-manager.ts'; import { createCodeError, isRuntimeInfraCode, toApiError } from '../agent/errors.ts'; +import { + addUsage, + budgetStop, + checkBudget, + createUsage, + resolveBudget, + type ResolveBudgetInput, + type RunBudgetLimits, + type RunBudgetUsage, + type RunStop, +} from './run-budget.ts'; +import { + createRunawayState, + observeSearchQuery, + observeToolCall, + runawayStop, + toolPatternMatches, + type RunawayPolicy, + type RunawayState, +} from './runaway-detector.ts'; export interface RunManagerOptions { sessions: SessionManager; runs: RunRepository; runtime: AgentRuntime; now?: () => number; + /** + * Budget defaults and the system ceiling applied to every run (#17). A run + * may override the defaults but never the ceiling; with no `budgets` option a + * run is unbudgeted and behaves exactly as before. + */ + budgets?: ResolveBudgetInput; + /** Tool-name patterns (`*` wildcard) whose `query` argument feeds the search-loop detector. */ + searchTools?: string[]; + /** Runaway detector thresholds; unset fields fall back to `defaultRunawayPolicy()`. */ + runaway?: Partial; } interface ActiveRun { sessionId: string; runId: string; cancelRequested: boolean; + /** When the run started, for the wall-clock budget. */ + startedAt: number; + /** Effective limits for this run, after defaults, overrides and the ceiling. */ + limits: RunBudgetLimits; + usage: RunBudgetUsage; + runaway: RunawayState; + /** Set when a budget or a runaway detector stopped the run. */ + stop?: RunStop; } /** @@ -43,6 +81,9 @@ export class RunManager { private readonly runs: RunRepository; private readonly runtime: AgentRuntime; private readonly now: () => number; + private readonly budgetInput: ResolveBudgetInput; + private readonly searchToolPatterns: readonly string[]; + private readonly runawayPolicy: Partial; private readonly listeners = new Set<(event: AgentEvent) => void>(); private activeRun: ActiveRun | null = null; @@ -51,6 +92,9 @@ export class RunManager { this.runs = options.runs; this.runtime = options.runtime; this.now = options.now ?? Date.now; + this.budgetInput = options.budgets ?? {}; + this.searchToolPatterns = options.searchTools ?? []; + this.runawayPolicy = options.runaway ?? {}; } subscribe(listener: (event: AgentEvent) => void): () => void { @@ -68,11 +112,21 @@ export class RunManager { return this.activeRun?.sessionId === sessionId; } + /** + * Start a run and drive it to a terminal state. + * @param sessionId - the session the run belongs to. + * @param content - the user message. + * @param workspaceContext - optional workspace context for the runtime. + * @param locale - optional UI locale. + * @param budgetOverrides - per-run budget overrides, clamped by the system ceiling. + * @returns the persisted running run. + */ async startRun( sessionId: string, content: string, workspaceContext?: WorkspaceContext, - locale?: SupportedLocale + locale?: SupportedLocale, + budgetOverrides?: RunBudgetLimits ): Promise { const text = content.trim(); if (!text) { @@ -109,7 +163,20 @@ export class RunManager { await this.sessions.appendMessage(sessionId, userMessage); await this.sessions.updateSession(sessionId, { status: 'running' }); - this.activeRun = { sessionId, runId: run.id, cancelRequested: false }; + const { limits } = resolveBudget({ + defaults: this.budgetInput.defaults, + ceiling: this.budgetInput.ceiling, + overrides: budgetOverrides, + }); + this.activeRun = { + sessionId, + runId: run.id, + cancelRequested: false, + startedAt: now, + limits, + usage: createUsage(), + runaway: createRunawayState(), + }; this.emit({ id: randomUUID(), sessionId, @@ -172,6 +239,10 @@ export class RunManager { answer = event.payload.answer; sawTerminal = true; } + + // Budgets and detectors are evaluated after the event is accounted for, + // so a run that stops keeps the evidence it had already produced. + if (await this.applyBudget(event)) break; } } catch (error) { failure = toApiError(error); @@ -179,6 +250,11 @@ export class RunManager { const active = this.activeRun; const cancelRequested = active?.cancelRequested ?? false; + const stop = active?.stop; + if (stop !== undefined) { + run.stopReason = stop.stopReason; + run.stopDetail = stop.detail; + } const now = this.now(); const cancelled = Boolean(cancelRequested || (failure && failure.code === 'RUN_CANCELLED')); @@ -225,7 +301,9 @@ export class RunManager { // stream failed before producing one (e.g. runtime spawn failure), so the // UI always observes a terminal event. if (!sawTerminal) { - if (cancelled) { + if (stop !== undefined) { + this.emitRunEvent(run, 'run_failed', { error: stopError(stop) }); + } else if (cancelled) { this.emitRunEvent(run, 'run_failed', { error: { code: 'RUN_CANCELLED', message: 'Run cancelled by user.' }, }); @@ -237,6 +315,71 @@ export class RunManager { } } + /** + * Account for one runtime event and decide whether the run must stop. A stop + * requests cancellation, so the caller stops consuming events and the run + * settles as `cancelled` — never as an ordinary success — carrying its partial + * answer, its tool calls and the machine-readable reason it stopped. + * @param event - the event that was just broadcast. + * @returns true when the run was stopped and cancellation was requested. + */ + private async applyBudget(event: AgentEvent): Promise { + const active = this.activeRun; + if (!active) return false; + + let usage = active.usage; + if (event.type === 'message_completed') usage = addUsage(usage, { modelCalls: 1 }); + if (event.type === 'tool_completed') usage = addUsage(usage, { toolCalls: 1 }); + + const searchQuery = + event.type === 'tool_completed' ? this.searchQueryOf(event.payload.toolCall) : undefined; + if (searchQuery !== undefined) usage = addUsage(usage, { searchIterations: 1 }); + + // Wall-clock is absolute: recomputed from the run's start on every event. + active.usage = { ...usage, wallClockMs: this.now() - active.startedAt }; + + let stop: RunStop | undefined; + const exhaustion = checkBudget(active.limits, active.usage); + if (exhaustion !== undefined) { + stop = budgetStop(exhaustion); + } else if (event.type === 'tool_completed') { + const call = observeToolCall( + active.runaway, + { tool: event.payload.toolCall.toolName, args: event.payload.toolCall.args }, + this.runawayPolicy + ); + active.runaway = call.state; + if (call.detection.detected) stop = runawayStop(call.detection); + } + + if (stop === undefined && searchQuery !== undefined) { + const search = observeSearchQuery(active.runaway, searchQuery, this.runawayPolicy); + active.runaway = search.state; + if (search.detection.detected) stop = runawayStop(search.detection); + } + + if (stop === undefined) return false; + + active.stop = stop; + active.cancelRequested = true; + await this.runtime.cancel({ sessionId: active.sessionId, runId: active.runId }); + return true; + } + + /** + * The query a search-tool call carries, when this run tracks search loops. + * @param toolCall - the completed tool call. + * @returns the query text, or undefined when the call is not a tracked search. + */ + private searchQueryOf(toolCall: ToolCall): string | undefined { + if (this.searchToolPatterns.length === 0) return undefined; + if (!this.searchToolPatterns.some((pattern) => toolPatternMatches(pattern, toolCall.toolName))) { + return undefined; + } + const query = toolCall.args.query ?? toolCall.args.q; + return typeof query === 'string' && query.trim() !== '' ? query : undefined; + } + private emitRunEvent(run: Run, type: AgentEvent['type'], payload?: AgentEventPayload): void { // Callers pair `type` with the matching payload shape. const event = { @@ -258,6 +401,24 @@ export class RunManager { } } +/** + * The error a budget or runaway stop reports to the UI. The code is stable so + * the renderer can tell a budget stop from a user cancel, and the detail keeps + * the numbers (which budget, which loop) attached to the message. + * @param stop - the stop recorded on the run. + * @returns an ApiError describing why the run stopped. + */ +function stopError(stop: RunStop): ApiError { + const code = + stop.stopReason === 'budget_exhausted' + ? 'BUDGET_EXHAUSTED' + : stop.stopReason === 'retry_storm' + ? 'RETRY_STORM' + : 'LOOP_DETECTED'; + const detail = stop.detail === undefined ? '' : ` ${JSON.stringify(stop.detail)}`; + return { code, message: `Run stopped: ${stop.stopReason}.${detail}` }; +} + function toRecord(toolCall: ToolCall): ToolCallRecord { return { id: toolCall.id, diff --git a/packages/shared/src/kernel/runaway-detector.test.ts b/packages/shared/src/kernel/runaway-detector.test.ts new file mode 100644 index 0000000..8be9b4f --- /dev/null +++ b/packages/shared/src/kernel/runaway-detector.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from 'bun:test'; +import { + createRunawayState, + defaultRunawayPolicy, + observeEvidence, + observeRetry, + observeSearchQuery, + observeToolCall, + runawayStop, +} from './runaway-detector.ts'; + +describe('defaultRunawayPolicy', () => { + it('ships thresholds that need several consecutive signals before firing', () => { + expect(defaultRunawayPolicy()).toEqual({ + repeatedToolCallThreshold: 5, + repeatedSearchQueryThreshold: 5, + querySimilarity: 0.9, + noProgressIterations: 3, + retryWindowMs: 60_000, + retryThreshold: 4, + toolCallInclude: [], + toolCallExclude: [], + }); + }); +}); + +describe('policy validation', () => { + it('refuses a threshold that would fire on the first signal', () => { + expect(() => + observeToolCall(createRunawayState(), { tool: 'bash', args: {} }, { repeatedToolCallThreshold: 1 }), + ).toThrow(/repeatedToolCallThreshold/); + expect(() => observeEvidence(createRunawayState(), [], { noProgressIterations: 0 })).toThrow( + /noProgressIterations/, + ); + expect(() => observeRetry(createRunawayState(), 0, { retryThreshold: -1 })).toThrow(/retryThreshold/); + }); + + it('refuses an out-of-range similarity or window', () => { + expect(() => observeSearchQuery(createRunawayState(), 'x', { querySimilarity: 0 })).toThrow( + /querySimilarity/, + ); + expect(() => observeSearchQuery(createRunawayState(), 'x', { querySimilarity: 1.5 })).toThrow( + /querySimilarity/, + ); + expect(() => observeRetry(createRunawayState(), 0, { retryWindowMs: 0 })).toThrow(/retryWindowMs/); + }); +}); + +describe('observeToolCall', () => { + const call = (command: string) => ({ tool: 'bash', args: { command } }); + + it('fires once the same tool call repeats up to the threshold', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 5 }; + let state = createRunawayState(); + const detections = []; + + for (let i = 0; i < 5; i += 1) { + const step = observeToolCall(state, call('ls'), policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 4).every((detection) => !detection.detected)).toBe(true); + expect(detections[4].detected).toBe(true); + expect(detections[4].signal).toBe('repeated_tool_call'); + expect(detections[4].evidence).toMatchObject({ tool: 'bash', count: 5 }); + }); + + it('resets the run when the arguments change', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 3 }; + let state = createRunawayState(); + + for (const command of ['ls', 'ls', 'pwd']) { + state = observeToolCall(state, call(command), policy).state; + } + const afterReset = observeToolCall(state, call('pwd'), policy); + + expect(afterReset.detection.detected).toBe(false); + }); + + it('treats arguments that differ only in key order as the same call', () => { + const policy = { ...defaultRunawayPolicy(), repeatedToolCallThreshold: 2 }; + const first = observeToolCall(createRunawayState(), { tool: 'read', args: { a: 1, b: 2 } }, policy); + const second = observeToolCall(first.state, { tool: 'read', args: { b: 2, a: 1 } }, policy); + + expect(second.detection.detected).toBe(true); + }); + + it('does not mutate the state it was given', () => { + const state = createRunawayState(); + observeToolCall(state, call('ls'), defaultRunawayPolicy()); + + expect(state.toolCall).toBeUndefined(); + }); +}); + +describe('observeSearchQuery', () => { + it('fires on near-duplicate queries even when the wording changes', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 3 }; + let state = createRunawayState(); + const queries = ['NVIDIA earnings 2026 Q2', 'NVIDIA Q2 2026 earnings!', 'nvidia earnings, q2 2026']; + const detections = []; + + for (const query of queries) { + const step = observeSearchQuery(state, query, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[2].detected).toBe(true); + expect(detections[2].signal).toBe('repeated_search_query'); + expect(detections[2].evidence).toMatchObject({ count: 3 }); + }); + + it('resets the run on a genuinely different query', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 2 }; + const first = observeSearchQuery(createRunawayState(), 'NVIDIA earnings 2026 Q2', policy); + const second = observeSearchQuery(first.state, 'apple dividend history 2019', policy); + + expect(second.detection.detected).toBe(false); + }); +}); + +describe('observeEvidence', () => { + it('fires after consecutive iterations that add no new evidence', () => { + const policy = { ...defaultRunawayPolicy(), noProgressIterations: 3 }; + let state = createRunawayState(); + const iterations = [['a'], ['a'], ['a', 'b'], ['a', 'b'], ['a', 'b']]; + const detections = []; + + for (const ids of iterations) { + const step = observeEvidence(state, ids, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[3].detected).toBe(false); + expect(detections[4].detected).toBe(true); + expect(detections[4].signal).toBe('no_new_evidence'); + expect(detections[4].evidence).toMatchObject({ iterations: 3 }); + }); +}); + +describe('observeRetry', () => { + it('fires when retries bunch up inside the window', () => { + const policy = { ...defaultRunawayPolicy(), retryThreshold: 4, retryWindowMs: 60_000 }; + let state = createRunawayState(); + const detections = []; + + for (const at of [0, 1_000, 2_000, 3_000]) { + const step = observeRetry(state, at, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 3).every((detection) => !detection.detected)).toBe(true); + expect(detections[3].detected).toBe(true); + expect(detections[3].signal).toBe('retry_storm'); + expect(detections[3].evidence).toMatchObject({ retriesInWindow: 4, windowMs: 60_000 }); + }); + + it('forgets retries that fell out of the window', () => { + const policy = { ...defaultRunawayPolicy(), retryThreshold: 3, retryWindowMs: 10_000 }; + let state = createRunawayState(); + + for (const at of [0, 60_000, 120_000]) { + state = observeRetry(state, at, policy).state; + } + const step = observeRetry(state, 180_000, policy); + + expect(step.detection.detected).toBe(false); + }); +}); + +describe('tool call scope', () => { + it('never fires for an excluded tool, so polling a job is not a loop', () => { + const policy = { + ...defaultRunawayPolicy(), + repeatedToolCallThreshold: 2, + toolCallExclude: ['job_output'], + }; + let state = createRunawayState(); + + for (let i = 0; i < 4; i += 1) { + const step = observeToolCall(state, { tool: 'job_output', args: { id: 'j1' } }, policy); + state = step.state; + expect(step.detection.detected).toBe(false); + } + }); + + it('tracks only the listed tools when an include list is given', () => { + const policy = { + ...defaultRunawayPolicy(), + repeatedToolCallThreshold: 2, + toolCallInclude: ['bash*'], + }; + const read1 = observeToolCall(createRunawayState(), { tool: 'read', args: {} }, policy); + const read2 = observeToolCall(read1.state, { tool: 'read', args: {} }, policy); + expect(read2.detection.detected).toBe(false); + + const bash1 = observeToolCall(read2.state, { tool: 'bash', args: {} }, policy); + const bash2 = observeToolCall(bash1.state, { tool: 'bash', args: {} }, policy); + expect(bash2.detection.detected).toBe(true); + }); +}); + +describe('observeSearchQuery overlap', () => { + it('keeps counting a search that only widens with more words', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 3 }; + let state = createRunawayState(); + const queries = [ + 'nvidia earnings 2026', + 'nvidia earnings 2026 q2', + 'nvidia earnings 2026 q2 guidance', + ]; + const detections = []; + + for (const query of queries) { + const step = observeSearchQuery(state, query, policy); + state = step.state; + detections.push(step.detection); + } + + expect(detections.slice(0, 2).every((detection) => !detection.detected)).toBe(true); + expect(detections[2].detected).toBe(true); + }); + + it('treats an empty query as a reset rather than a false loop', () => { + const policy = { ...defaultRunawayPolicy(), repeatedSearchQueryThreshold: 2 }; + const first = observeSearchQuery(createRunawayState(), 'nvidia earnings 2026', policy); + const empty = observeSearchQuery(first.state, ' ', policy); + + expect(empty.detection.detected).toBe(false); + }); +}); + +describe('runawayStop', () => { + it('maps a loop detection to loop_detected and keeps the evidence', () => { + const stop = runawayStop({ + detected: true, + signal: 'repeated_tool_call', + evidence: { tool: 'bash', count: 5 }, + }); + + expect(stop.stopReason).toBe('loop_detected'); + expect(stop.detail).toEqual({ signal: 'repeated_tool_call', tool: 'bash', count: 5 }); + }); + + it('gives a retry storm its own stop reason', () => { + const stop = runawayStop({ detected: true, signal: 'retry_storm', evidence: { retriesInWindow: 4 } }); + + expect(stop.stopReason).toBe('retry_storm'); + }); + + it('refuses to build a stop from a non-detection', () => { + expect(() => runawayStop({ detected: false })).toThrow(/detection/); + }); +}); diff --git a/packages/shared/src/kernel/runaway-detector.ts b/packages/shared/src/kernel/runaway-detector.ts new file mode 100644 index 0000000..fd48584 --- /dev/null +++ b/packages/shared/src/kernel/runaway-detector.ts @@ -0,0 +1,378 @@ +/** + * Deterministic runaway detectors for Agent / Deep Research runs. + * + * Each detector observes one signal and returns both the next state and a + * decision, so a caller can accumulate state per run and still replay it: the + * state is plain data, and every decision carries the numbers behind it. The + * detectors never call a model — classification stays explainable, and a + * provider is free to be missing or flaky without breaking the guard. + */ + +import type { RunStop, StopReason } from './run-budget.ts'; + +/** The failure modes a run can get stuck in. */ +export type RunawaySignal = + | 'repeated_tool_call' + | 'repeated_search_query' + | 'no_new_evidence' + | 'retry_storm'; + +/** A detector's decision; `evidence` is what a trace or run summary should record. */ +export interface RunawayDetection { + detected: boolean; + signal?: RunawaySignal; + evidence?: Record; +} + +/** Thresholds and scope shared by every detector of one run. */ +export interface RunawayPolicy { + /** Consecutive identical tool calls (same tool, same arguments) that count as a loop. */ + repeatedToolCallThreshold: number; + /** Consecutive near-identical search queries that count as a loop. */ + repeatedSearchQueryThreshold: number; + /** + * Token overlap at or above which two queries are the same search. Overlap is + * `|shared| / min(|a|, |b|)`, so a query that only widens with extra words + * still counts as the same search while an unrelated query does not. + */ + querySimilarity: number; + /** Consecutive iterations reporting the same evidence set before it counts as no progress. */ + noProgressIterations: number; + /** Rolling window over which retries are counted. */ + retryWindowMs: number; + /** Retries inside the window that count as a storm. */ + retryThreshold: number; + /** Tool name patterns to track; empty means every tool. `*` is a wildcard. */ + toolCallInclude: string[]; + /** Tool name patterns never tracked, whatever the include list says. */ + toolCallExclude: string[]; +} + +/** Accumulated per-run detector state: plain data, safe to persist and replay. */ +export interface RunawayState { + /** The last tool call's canonical key and how many times it repeated in a row. */ + toolCall?: { key: string; count: number }; + /** The last search's token set and how many times it repeated in a row. */ + searchQuery?: { tokens: string[]; count: number }; + /** The last evidence set (sorted ids) and how many times it repeated in a row. */ + evidence?: { key: string; iterations: number }; + /** Retry timestamps still inside the window. */ + retries: number[]; +} + +/** One detector's outcome plus the state to carry into the next observation. */ +export interface RunawayStep { + state: RunawayState; + detection: RunawayDetection; +} + +/** + * Thresholds that need several consecutive signals before firing, so a single + * retry or a repeated query that is genuinely new evidence does not stop a run. + * + * `toolCallExclude` is empty by default, but polling tools such as a job-status + * reader are legitimate repeats and belong there for a workflow that uses them. + * @returns a fresh policy with the default thresholds. + */ +export function defaultRunawayPolicy(): RunawayPolicy { + return { + repeatedToolCallThreshold: 5, + repeatedSearchQueryThreshold: 5, + querySimilarity: 0.9, + noProgressIterations: 3, + retryWindowMs: 60_000, + retryThreshold: 4, + toolCallInclude: [], + toolCallExclude: [], + }; +} + +/** Fresh detector state for a new run. */ +export function createRunawayState(): RunawayState { + return { retries: [] }; +} + +/** + * Reject a counting threshold the runtime cannot honour. Fail-loud by design: a + * threshold of 1 looks like a guard but fires on the first signal. + * @param key - policy field name, for the error message. + * @param value - requested value. + */ +function assertThreshold(key: string, value: number): void { + if (!Number.isInteger(value) || value < 2) { + throw new Error( + `runaway-detector: ${key} must be an integer >= 2, received ${String(value)}`, + ); + } +} + +/** + * Validate a resolved policy before any state is touched. + * @param policy - the merged policy. + */ +function assertPolicy(policy: RunawayPolicy): void { + assertThreshold('repeatedToolCallThreshold', policy.repeatedToolCallThreshold); + assertThreshold('repeatedSearchQueryThreshold', policy.repeatedSearchQueryThreshold); + assertThreshold('noProgressIterations', policy.noProgressIterations); + assertThreshold('retryThreshold', policy.retryThreshold); + if (!Number.isFinite(policy.querySimilarity) || policy.querySimilarity <= 0 || policy.querySimilarity > 1) { + throw new Error( + `runaway-detector: querySimilarity must be in (0, 1], received ${String(policy.querySimilarity)}`, + ); + } + if (!Number.isFinite(policy.retryWindowMs) || policy.retryWindowMs <= 0) { + throw new Error( + `runaway-detector: retryWindowMs must be a positive finite number, received ${String(policy.retryWindowMs)}`, + ); + } +} + +/** Merge a partial policy over the defaults and validate the result. */ +function withDefaults(policy: Partial): RunawayPolicy { + const resolved = { ...defaultRunawayPolicy(), ...policy }; + assertPolicy(resolved); + return resolved; +} + +/** + * Canonical string form of a tool call's arguments. Property order must not + * matter, so keys are sorted deeply before stringifying; a raw-string fallback + * (malformed argument JSON) is parsed first so both paths share one key. + * @param args - tool arguments, parsed or raw. + * @returns a stable key. + */ +function canonicalArguments(args: unknown): string { + if (typeof args === 'string') { + try { + return JSON.stringify(sortJson(JSON.parse(args))); + } catch { + return JSON.stringify(args); + } + } + return JSON.stringify(sortJson(args)) ?? String(args); +} + +/** Deep key-sort so two argument objects differing only in property order match. */ +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (value !== null && typeof value === 'object') { + const sorted: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortJson((value as Record)[key]); + } + return sorted; + } + return value; +} + +/** Compile one tool-name pattern; every regex metacharacter except `*` is literal. */ +function wildcardToRegExp(pattern: string): RegExp { + const escaped = pattern.replace(/[|\\{}()[\]^$+?.]/g, String.raw`\$&`); + return new RegExp(`^${escaped.replaceAll('*', '.*')}$`); +} + +/** + * Whether a tool name matches one `*`-wildcard pattern. Exported because the + * runtime uses the same matching to decide which tool calls carry a search + * query, so both sides agree on what "the same tool scope" means. + * @param pattern - a tool-name pattern. + * @param tool - the tool name to test. + * @returns true when the name matches. + */ +export function toolPatternMatches(pattern: string, tool: string): boolean { + return wildcardToRegExp(pattern).test(tool); +} + +/** + * Whether a tool participates in loop detection. Untracked tools are + * transparent: they neither count nor reset the chain, so polling through an + * excluded tool cannot hide a loop on a tracked one. + * @param policy - the resolved policy. + * @param tool - the tool being called. + * @returns true when this tool is tracked. + */ +function trackedTool(policy: RunawayPolicy, tool: string): boolean { + if (policy.toolCallInclude.length > 0 && !policy.toolCallInclude.some((pattern) => wildcardToRegExp(pattern).test(tool))) { + return false; + } + return !policy.toolCallExclude.some((pattern) => wildcardToRegExp(pattern).test(tool)); +} + +/** + * Normalize a search query into a sorted, unique token set. Unicode letters and + * numbers survive, so CJK queries tokenize the same way Latin ones do. + * @param query - the raw query text. + * @returns sorted unique tokens. + */ +function tokenize(query: string): string[] { + const tokens = query + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter((token) => token !== ''); + return [...new Set(tokens)].sort(); +} + +/** + * Token overlap of two queries: shared tokens over the smaller set. Unlike + * Jaccard this stays at 1 when one query merely adds words, which is the common + * shape of a search loop, while an unrelated query still scores near zero. + * @param a - first token set. + * @param b - second token set. + * @returns overlap in [0, 1]; 0 when either set is empty. + */ +function overlap(a: readonly string[], b: readonly string[]): number { + const left = new Set(a); + const right = new Set(b); + if (left.size === 0 || right.size === 0) return 0; + let shared = 0; + for (const token of left) if (right.has(token)) shared += 1; + return shared / Math.min(left.size, right.size); +} + +/** + * Observe one tool call and advance the identical-call chain. + * @param state - detector state so far. + * @param call - the tool name and its arguments. + * @param policy - threshold and scope overrides. + * @returns next state and whether the run is looping on this call. + */ +export function observeToolCall( + state: RunawayState, + call: { tool: string; args: unknown }, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + if (!trackedTool(resolved, call.tool)) return { state, detection: { detected: false } }; + + const canonical = canonicalArguments(call.args); + const key = `${call.tool}\u0000${canonical}`; + const count = state.toolCall?.key === key ? state.toolCall.count + 1 : 1; + const next: RunawayState = { ...state, toolCall: { key, count } }; + + if (count < resolved.repeatedToolCallThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'repeated_tool_call', + evidence: { tool: call.tool, count, canonicalArguments: canonical }, + }, + }; +} + +/** + * Observe a search query and advance the near-duplicate query chain. + * @param state - detector state so far. + * @param query - the raw query text. + * @param policy - threshold overrides. + * @returns next state and whether the run is re-running the same search. + */ +export function observeSearchQuery( + state: RunawayState, + query: string, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const tokens = tokenize(query); + const previous = state.searchQuery; + const score = previous === undefined ? 0 : overlap(previous.tokens, tokens); + const repeated = previous !== undefined && score >= resolved.querySimilarity; + const count = repeated ? previous.count + 1 : 1; + // A repeat keeps the cluster's original token set, so overlap is measured + // against the search the run started looping on rather than drifting wording. + const cluster = repeated ? previous.tokens : tokens; + const next: RunawayState = { ...state, searchQuery: { tokens: cluster, count } }; + + if (count < resolved.repeatedSearchQueryThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'repeated_search_query', + evidence: { query, count, similarity: score }, + }, + }; +} + +/** + * Observe the evidence retrieved by one iteration. + * @param state - detector state so far. + * @param evidenceIds - ids retrieved by this iteration. + * @param policy - threshold overrides. + * @returns next state and whether the run stopped making progress. + */ +export function observeEvidence( + state: RunawayState, + evidenceIds: readonly string[], + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const key = [...new Set(evidenceIds)].sort().join('\u0000'); + const iterations = state.evidence?.key === key ? state.evidence.iterations + 1 : 1; + const next: RunawayState = { ...state, evidence: { key, iterations } }; + + if (iterations < resolved.noProgressIterations) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'no_new_evidence', + evidence: { iterations, evidenceIds: [...evidenceIds] }, + }, + }; +} + +/** + * Observe one retry and count the retries still inside the rolling window. + * @param state - detector state so far. + * @param at - retry timestamp in milliseconds. + * @param policy - threshold overrides. + * @returns next state and whether the run is in a retry storm. + */ +export function observeRetry( + state: RunawayState, + at: number, + policy: Partial = {}, +): RunawayStep { + const resolved = withDefaults(policy); + const retries = [...state.retries.filter((time) => at - time < resolved.retryWindowMs), at]; + const next: RunawayState = { ...state, retries }; + + if (retries.length < resolved.retryThreshold) { + return { state: next, detection: { detected: false } }; + } + + return { + state: next, + detection: { + detected: true, + signal: 'retry_storm', + evidence: { retriesInWindow: retries.length, windowMs: resolved.retryWindowMs }, + }, + }; +} + +/** + * Turn a detection into the run's stop reason, mirroring `budgetStop` so a run + * has one shape of outcome whatever stopped it. + * @param detection - a detection reported by one of the observers. + * @returns a `loop_detected` or `retry_storm` stop carrying the evidence. + */ +export function runawayStop(detection: RunawayDetection): RunStop { + if (!detection.detected || detection.signal === undefined) { + throw new Error('runaway-detector: cannot build a run stop from a non-detection'); + } + + const stopReason: StopReason = detection.signal === 'retry_storm' ? 'retry_storm' : 'loop_detected'; + + return { stopReason, detail: { signal: detection.signal, ...detection.evidence } }; +} diff --git a/scripts/eval/budget-smoke.ts b/scripts/eval/budget-smoke.ts new file mode 100644 index 0000000..4123c82 --- /dev/null +++ b/scripts/eval/budget-smoke.ts @@ -0,0 +1,251 @@ +#!/usr/bin/env bun +// Run-budget smoke test (#17): a REAL provider run through the production agent +// path (AgentKernel -> RunManager -> Pi runtime), with budgets small enough that +// the run must be stopped by the guard rather than by the model finishing. +// +// source ~/.folio-e2e.env +// bun scripts/eval/budget-smoke.ts --prompt "..." --max-model-calls 2 +// bun scripts/eval/budget-smoke.ts --prompt "..." --loop-threshold 2 --max-tool-calls 20 +// +// Exit codes: 0 = the run stopped with the expected stop reason and kept a +// partial result; 1 = it did not (or the runtime failed), so CI/agents can gate +// on real evidence instead of reading prose. +// +// Credentials: ANTHROPIC_API_KEY (or the keys Pi resolves itself). Nothing is +// printed but counts, names and the partial answer. +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { AgentEvent } from '../../packages/core/src/index.ts'; +import { AgentKernel } from '../../packages/shared/src/kernel/agent-kernel.ts'; + +interface CliOptions { + prompt: string; + provider: string; + model?: string; + maxModelCalls?: number; + maxToolCalls?: number; + wallClockMs?: number; + loopThreshold?: number; + searchTools: string[]; + expect: string; + timeoutMs: number; + healthTimeoutMs: number; + waitMs: number; +} + +const USAGE = `Usage: bun scripts/eval/budget-smoke.ts [flags] + + --prompt prompt to run (required) + --provider provider for setModel (default: anthropic) + --model model id for setModel (default: runtime default) + --max-model-calls model-call budget for the run + --max-tool-calls tool-call budget for the run + --wall-clock-ms wall-clock budget for the run + --loop-threshold repeated identical tool calls that stop the run + --search-tools tool-name patterns whose query feeds the search detector + --expect expected stop reason (default: any non-completed stop) + --timeout-ms runtime request timeout (default: 120000) + --health-timeout-ms pi startup health check budget (default: 180000; the + first run may download the pi runtime) + --wait-ms how long to wait for the run to settle before + cancelling it (default: 600000)`; + +function parseFlags(argv: string[]): CliOptions { + const options: CliOptions = { + prompt: '', + provider: 'anthropic', + searchTools: [], + expect: '', + timeoutMs: 120_000, + healthTimeoutMs: 180_000, + waitMs: 600_000, + }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + const value = argv[i + 1]; + const takeNumber = () => { + i += 1; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${flag} needs a positive number`); + return parsed; + }; + switch (flag) { + case '--prompt': + i += 1; + options.prompt = value ?? ''; + break; + case '--provider': + i += 1; + options.provider = value ?? 'anthropic'; + break; + case '--model': + i += 1; + options.model = value; + break; + case '--max-model-calls': + options.maxModelCalls = takeNumber(); + break; + case '--max-tool-calls': + options.maxToolCalls = takeNumber(); + break; + case '--wall-clock-ms': + options.wallClockMs = takeNumber(); + break; + case '--loop-threshold': + options.loopThreshold = takeNumber(); + break; + case '--search-tools': + i += 1; + options.searchTools = (value ?? '').split(',').filter((entry) => entry !== ''); + break; + case '--expect': + i += 1; + options.expect = value ?? ''; + break; + case '--timeout-ms': + options.timeoutMs = takeNumber(); + break; + case '--health-timeout-ms': + options.healthTimeoutMs = takeNumber(); + break; + case '--wait-ms': + options.waitMs = takeNumber(); + break; + case '--help': + case '-h': + console.log(USAGE); + process.exit(0); + break; + default: + throw new Error(`unknown flag ${flag}\n\n${USAGE}`); + } + } + if (options.prompt.trim() === '') throw new Error(`--prompt is required\n\n${USAGE}`); + return options; +} + +async function main(): Promise { + const options = parseFlags(process.argv.slice(2)); + if (!process.env.ANTHROPIC_API_KEY && !process.env.FINAGENT_PROVIDER_OVERRIDES) { + console.error('no LLM credential: set ANTHROPIC_API_KEY or FINAGENT_PROVIDER_OVERRIDES'); + return 1; + } + + const runtimeDir = await mkdtemp(join(tmpdir(), 'folio-budget-smoke-')); + const kernel = new AgentKernel({ + provider: 'pi-runtime', + storageDir: join(runtimeDir, 'store'), + piSessionDir: join(runtimeDir, 'pi-sessions'), + // Budgets are what this smoke test exists to exercise; the ceiling keeps a + // mistyped flag from turning into a runaway real-provider bill. + budgets: { + defaults: { + modelCalls: options.maxModelCalls, + toolCalls: options.maxToolCalls, + wallClockMs: options.wallClockMs, + }, + ceiling: { modelCalls: 25, toolCalls: 25, wallClockMs: 15 * 60_000 }, + }, + runaway: options.loopThreshold === undefined ? {} : { repeatedToolCallThreshold: options.loopThreshold }, + searchTools: options.searchTools, + rpc: { + cwd: process.cwd(), + extensions: [], + env: () => process.env, + requestTimeoutMs: options.timeoutMs, + // The first run may have to fetch the pi runtime, which takes far longer + // than the 5s default health budget. + healthTimeoutMs: options.healthTimeoutMs, + }, + }); + + const llm = kernel.getLlmApi(); + if (llm && options.model !== undefined) { + const state = await llm.setModel(options.provider, options.model); + console.log(`model: ${state.model?.provider ?? '?'}/${state.model?.id ?? '?'}`); + } + + const toolNames: string[] = []; + let modelCalls = 0; + const unsubscribe = kernel.runs.subscribe((event: AgentEvent) => { + if (event.type === 'message_completed') modelCalls += 1; + if (event.type === 'tool_completed') { + toolNames.push(event.payload.toolCall.toolName); + console.log(` tool: ${event.payload.toolCall.toolName} ${JSON.stringify(event.payload.toolCall.args).slice(0, 120)}`); + } + }); + + const session = await kernel.sessions.createSession('budget-smoke'); + const startedAt = Date.now(); + const run = await kernel.runs.startRun(session.id, options.prompt); + console.log(`run ${run.id} started in ${runtimeDir}`); + + const deadline = Date.now() + options.waitMs; + while (kernel.runs.isRunning() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + const waitedOut = kernel.runs.isRunning(); + if (waitedOut) { + // Never leave a real-provider run burning: cancel before reporting. + console.error(`wait timed out after ${options.waitMs}ms; cancelling the run`); + await kernel.runs.cancelRun(session.id, run.id); + const cancelDeadline = Date.now() + 30_000; + while (kernel.runs.isRunning() && Date.now() < cancelDeadline) { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + const elapsedMs = Date.now() - startedAt; + unsubscribe(); + + const persisted = await kernel.sessions.getRun(session.id, run.id); + const messages = await kernel.sessions.listMessages(session.id); + await kernel.dispose(); + + const summary = { + status: persisted?.status, + stopReason: persisted?.stopReason, + stopDetail: persisted?.stopDetail, + modelCalls, + toolCalls: toolNames, + elapsedMs, + partialAnswer: (persisted?.answer ?? '').slice(0, 400), + persistedMessages: messages.map((message) => ({ + role: message.role, + chars: message.content.length, + toolCalls: message.toolCalls?.length ?? 0, + })), + runtimeDir, + }; + console.log('\n=== summary ==='); + console.log(JSON.stringify(summary, null, 2)); + + const stopped = persisted?.stopReason !== undefined && persisted.stopReason !== 'completed'; + const expected = options.expect === '' ? stopped : persisted?.stopReason === options.expect; + if (persisted?.status === 'failed') { + console.error(`FAIL: run failed: ${persisted.error?.code} ${persisted.error?.message}`); + return 1; + } + if (!expected) { + console.error( + `FAIL: expected stop reason ${options.expect || '(any non-completed)'}, got ${String(persisted?.stopReason)}` + ); + return 1; + } + // The stop reason is the contract under test. Whether any partial evidence + // exists depends on how far the real model got before the guard fired, so it + // is reported rather than asserted; a run that produced nothing is possible. + const evidence = (persisted?.answer ?? '') !== '' || toolNames.length > 0; + console.log( + `PASS: stopped with ${String(persisted?.stopReason)} after ${modelCalls} model calls, ` + + `${toolNames.length} tool calls, partial evidence: ${evidence ? 'yes' : 'none produced before the stop'}` + ); + return 0; +} + +main() + .then((code) => process.exit(code)) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + });