Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,19 @@ export interface Message {

export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';

/**
* Why a run stopped. `completed` is the only success value: a run cut short by a
* budget or a runaway loop is not an ordinary answer, so telemetry, evaluation
* and the UI branch on this instead of treating every terminal run as success.
*/
export type StopReason =
| 'completed'
| 'budget_exhausted'
| 'loop_detected'
| 'retry_storm'
| 'cancelled'
| 'error';

/** One agent execution inside a session. */
export interface Run {
id: string;
Expand All @@ -150,6 +163,10 @@ export interface Run {
completedAt?: number;
answer?: string;
error?: ApiError;
/** Machine-readable reason the run stopped; absent on records written before #17. */
stopReason?: StopReason;
/** The numbers behind a non-success stop (which budget ran out, which loop fired). */
stopDetail?: Record<string, unknown>;
}

/** Live tool call state, streamed through agent events. */
Expand Down
173 changes: 173 additions & 0 deletions packages/shared/src/kernel/agent-kernel-budget.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import type {
AgentEvent,
AgentEventPayload,
AgentRunInput,
AgentRuntime,
ApiResult,
RuntimeSession,
ToolDefinition,
} from '@finagent/core';
import { AgentKernel } from './agent-kernel.ts';

let dir = '';
let clock = 1000;

beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'finagent-kernel-budget-'));
clock = 1000;
});

afterEach(async () => {
await rm(dir, { recursive: true, force: true });
});

class ScriptedRuntime implements AgentRuntime {
cancelCalls: Array<{ sessionId: string; runId: string }> = [];

constructor(private readonly script: (input: AgentRunInput) => AsyncIterable<AgentEvent>) {}

async getTools(): Promise<ApiResult<ToolDefinition[]>> {
return { ok: true, data: [] };
}

async ensureSession(session: { id: string }): Promise<RuntimeSession> {
return { sessionId: session.id, status: 'active' };
}

async *run(input: AgentRunInput): AsyncIterable<AgentEvent> {
yield* this.script(input);
}

async cancel(input: { sessionId: string; runId: string }): Promise<void> {
this.cancelCalls.push(input);
}

async dispose(): Promise<void> {}
}

function event(
sessionId: string,
runId: string,
type: AgentEvent['type'],
payload?: AgentEventPayload,
sequence = 1
): AgentEvent {
return {
id: `evt-${type}-${sequence}`,
sessionId,
runId,
type,
timestamp: clock,
sequence,
...(payload === undefined ? {} : { payload }),
} as unknown as AgentEvent;
}

/** Two model calls, then a normal completion — over-budget only when a budget says so. */
function twoStepScript(answer: string) {
return async function* (input: AgentRunInput) {
yield event(input.sessionId, input.runId, 'message_completed', { answer }, 1);
yield event(input.sessionId, input.runId, 'message_completed', { answer }, 2);
yield event(input.sessionId, input.runId, 'run_completed', { answer, toolCalls: [] }, 3);
};
}

describe('AgentKernel run budgets (#17)', () => {
it('forwards budget options to the run loop so a real run can be stopped', async () => {
const runtime = new ScriptedRuntime(twoStepScript('partial'));
const kernel = new AgentKernel({
storageDir: dir,
piSessionDir: join(dir, 'pi-sessions'),
runtime,
now: () => clock,
budgets: { defaults: { modelCalls: 1 } },
});
const session = await kernel.sessions.createSession('Budget');

const run = await kernel.runs.startRun(session.id, 'q');
await waitFor(async () => !kernel.runs.isRunning());

expect(runtime.cancelCalls).toEqual([{ sessionId: session.id, runId: run.id }]);
expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({
status: 'cancelled',
answer: 'partial',
stopReason: 'budget_exhausted',
stopDetail: { key: 'modelCalls', limit: 1, used: 1 },
});
});

it('runs the same script to completion when no budget is configured', async () => {
const runtime = new ScriptedRuntime(twoStepScript('done'));
const kernel = new AgentKernel({
storageDir: dir,
piSessionDir: join(dir, 'pi-sessions'),
runtime,
now: () => clock,
});
const session = await kernel.sessions.createSession('Plain');

const run = await kernel.runs.startRun(session.id, 'q');
await waitFor(async () => !kernel.runs.isRunning());

expect(runtime.cancelCalls).toEqual([]);
expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({
status: 'completed',
answer: 'done',
});
});

it('forwards runaway detector thresholds for repeated tool calls', async () => {
const runtime = new ScriptedRuntime(async function* (input) {
for (let i = 1; i <= 3; i += 1) {
yield event(
input.sessionId,
input.runId,
'tool_completed',
{
toolCall: {
id: `t${i}`,
toolName: 'get_quote',
args: { symbol: 'AAPL.US' },
startedAt: clock,
completedAt: clock,
status: 'success',
result: {},
},
},
i
);
}
yield event(input.sessionId, input.runId, 'run_completed', { answer: 'done', toolCalls: [] }, 9);
});
const kernel = new AgentKernel({
storageDir: dir,
piSessionDir: join(dir, 'pi-sessions'),
runtime,
now: () => clock,
runaway: { repeatedToolCallThreshold: 2 },
});
const session = await kernel.sessions.createSession('Loop');

const run = await kernel.runs.startRun(session.id, 'q');
await waitFor(async () => !kernel.runs.isRunning());

expect(await kernel.sessions.getRun(session.id, run.id)).toMatchObject({
status: 'cancelled',
stopReason: 'loop_detected',
});
});
});

async function waitFor(predicate: () => Promise<boolean>, timeoutMs = 2000) {
const started = Date.now();
while (!(await predicate())) {
if (Date.now() - started > timeoutMs) {
throw new Error('waitFor timed out');
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
14 changes: 14 additions & 0 deletions packages/shared/src/kernel/agent-kernel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { createCodeError } from '../agent/errors.ts';
import type { PiRpcClientOptions } from '../agent/pi-rpc-client.ts';
import { SessionManager } from './session-manager.ts';
import { RunManager } from './run-manager.ts';
import type { ResolveBudgetInput } from './run-budget.ts';
import type { RunawayPolicy } from './runaway-detector.ts';

export type AgentProvider = 'local' | 'pi-runtime';

Expand All @@ -33,6 +35,15 @@ export interface AgentKernelOptions {
/** Skill hub used for progressive skill loading in the runtime prompt. */
skillHub?: SkillHub;
now?: () => number;
/**
* Budget defaults and the system ceiling every run obeys (#17). Without it
* runs are unbudgeted; a run may still tighten its own limits at startRun.
*/
budgets?: ResolveBudgetInput;
/** Tool-name patterns (`*` wildcard) whose `query` argument feeds the search-loop detector. */
searchTools?: string[];
/** Runaway detector thresholds; unset fields fall back to `defaultRunawayPolicy()`. */
runaway?: Partial<RunawayPolicy>;
}

/**
Expand Down Expand Up @@ -68,6 +79,9 @@ export class AgentKernel {
runs: new RunRepository(store),
runtime: this.runtime,
now,
budgets: options.budgets,
searchTools: options.searchTools,
runaway: options.runaway,
});
}

Expand Down
15 changes: 6 additions & 9 deletions packages/shared/src/kernel/run-budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
* instead of reporting it as an ordinary success.
*/

import type { StopReason } from '@finagent/core';

/** Every dimension a run budget can constrain. */
export type BudgetKey =
| 'wallClockMs'
Expand Down Expand Up @@ -54,16 +56,11 @@ export interface BudgetExhaustion {
}

/**
* Why a run stopped. Machine-readable so traces, run summaries and evaluation
* can branch on it; `completed` is the only success value.
* Why a run stopped. Owned by the core protocol because the UI, telemetry and
* evaluation read it off the persisted run record; re-exported here so callers
* that only work with budgets keep importing it from one place.
*/
export type StopReason =
| 'completed'
| 'budget_exhausted'
| 'loop_detected'
| 'retry_storm'
| 'cancelled'
| 'error';
export type { StopReason };

/** A run outcome paired with the detail behind a non-success reason. */
export interface RunStop {
Expand Down
Loading
Loading