diff --git a/README.md b/README.md index 37e4433..d45610d 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,6 @@ pnpm build # Compile to dist/ | `memory_query` | Query across all memory backends with unified results | | `memory_stats` | Get memory system statistics dashboard | -## Live integration mode - -```bash -NEXUS_LIVE=true npx tsx src/run-live.ts -``` - ## License MIT diff --git a/package.json b/package.json index 89e3c2e..5c50d87 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,9 @@ "scripts": { "build": "tsc", "test": "vitest run", - "typecheck": "tsc --noEmit", - "live": "NEXUS_LIVE=true tsx src/run-live.ts" + "typecheck": "tsc --noEmit" }, "devDependencies": { - "tsx": "^4.19.0", "typescript": "^5.7.0", "vitest": "^3.2.6", "@types/node": "^22.0.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9d4a551..b656ee3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,9 +18,6 @@ importers: '@types/node': specifier: ^22.0.0 version: 22.19.21 - tsx: - specifier: ^4.19.0 - version: 4.21.0 typescript: specifier: ^5.7.0 version: 5.9.3 @@ -883,6 +880,7 @@ snapshots: get-tsconfig@4.13.6: dependencies: resolve-pkg-maps: 1.0.0 + optional: true js-tokens@9.0.1: {} @@ -910,7 +908,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - resolve-pkg-maps@1.0.0: {} + resolve-pkg-maps@1.0.0: + optional: true rollup@4.62.0: dependencies: @@ -976,6 +975,7 @@ snapshots: get-tsconfig: 4.13.6 optionalDependencies: fsevents: 2.3.3 + optional: true typescript@5.9.3: {} diff --git a/src/benchmark.test.ts b/src/benchmark.test.ts index ac934fa..8c3d042 100644 --- a/src/benchmark.test.ts +++ b/src/benchmark.test.ts @@ -10,6 +10,8 @@ import { countAvailableBackends, computeSummary, runBenchmark, + withTimeout, + ToolCallTimeoutError, } from './benchmark.js'; import type { BenchmarkConfig, QueryBenchmarkResult } from './types.js'; import { @@ -341,3 +343,64 @@ describe('runBenchmark', () => { expect(queryCall?.source).toBe('session'); }); }); + +// ============================================================================ +// withTimeout +// ============================================================================ + +describe('withTimeout', () => { + it('rejects with ToolCallTimeoutError when the underlying call hangs', async () => { + const caller: ToolCaller = { + call: vi.fn(() => new Promise(() => {})), + }; + + const bounded = withTimeout(caller, 10); + + await expect(bounded.call('memory_query', {})).rejects.toBeInstanceOf( + ToolCallTimeoutError + ); + }); + + it('passes through a fast result', async () => { + const caller: ToolCaller = { + call: vi.fn(async () => ({ ok: true })), + }; + + const bounded = withTimeout(caller, 1000); + + await expect(bounded.call('memory_stats', {})).resolves.toEqual({ + ok: true, + }); + }); + + it('propagates an underlying thrown error', async () => { + const caller: ToolCaller = { + call: vi.fn(async () => { + throw new Error('backend exploded'); + }), + }; + + const bounded = withTimeout(caller, 1000); + + await expect(bounded.call('memory_query', {})).rejects.toThrow( + 'backend exploded' + ); + }); + + it('aborts the signal on timeout', async () => { + let captured: AbortSignal | undefined; + const caller: ToolCaller = { + call: vi.fn((_toolName: string, args: Record) => { + captured = args['signal'] as AbortSignal; + return new Promise(() => {}); + }), + }; + + const bounded = withTimeout(caller, 10); + + await expect(bounded.call('memory_query', {})).rejects.toBeInstanceOf( + ToolCallTimeoutError + ); + expect(captured?.aborted).toBe(true); + }); +}); diff --git a/src/benchmark.ts b/src/benchmark.ts index 90d61db..afe7682 100644 --- a/src/benchmark.ts +++ b/src/benchmark.ts @@ -23,6 +23,51 @@ export interface ToolCaller { call(toolName: string, args: Record): Promise; } +/** Default per-call timeout for remote tool calls (ms). */ +export const DEFAULT_TIMEOUT_MS = 30_000; + +/** Error thrown when a tool call exceeds its configured timeout. */ +export class ToolCallTimeoutError extends Error { + constructor(toolName: string, timeoutMs: number) { + super(`Tool call '${toolName}' timed out after ${timeoutMs}ms`); + this.name = 'ToolCallTimeoutError'; + } +} + +/** + * Wrap a ToolCaller so every call is bounded by `timeoutMs`. + * + * Uses an AbortController combined with a timer, guaranteeing the returned + * promise settles even if the underlying call hangs forever. + */ +export function withTimeout( + caller: ToolCaller, + timeoutMs: number = DEFAULT_TIMEOUT_MS +): ToolCaller { + return { + call(toolName: string, args: Record): Promise { + const controller = new AbortController(); + const callArgs = { ...args, signal: controller.signal }; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + controller.abort(); + reject(new ToolCallTimeoutError(toolName, timeoutMs)); + }, timeoutMs); + caller.call(toolName, callArgs).then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (err) => { + clearTimeout(timer); + reject(err); + } + ); + }); + }, + }; +} + // ============================================================================ // Individual steps // ============================================================================ @@ -73,17 +118,20 @@ export async function runBenchmark( caller: ToolCaller, config: BenchmarkConfig ): Promise { + // Bound every remote call so a hung server can't make the benchmark hang. + const boundedCaller = withTimeout(caller, config.timeoutMs); + // Step 1: Collect stats let stats: MemoryStatsResponse | null = null; if (config.includeStats !== false) { - stats = await fetchStats(caller); + stats = await fetchStats(boundedCaller); } // Step 2: Run all queries const queryResults: QueryBenchmarkResult[] = []; for (const bench of config.queries) { const { result, durationMs } = await runQuery( - caller, + boundedCaller, bench.query, config.source, bench.limit diff --git a/src/live-caller.ts b/src/live-caller.ts deleted file mode 100644 index c3f8d47..0000000 --- a/src/live-caller.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Live MCP tool caller — bridges ToolCaller interface to a real nexus-agents MCP server. - */ - -import type { ToolCaller } from './benchmark.js'; - -export function createLiveCaller( - callFn: (tool: string, args: Record) => Promise, -): ToolCaller { - return { call: callFn }; -} - -export function isLiveMode(): boolean { - return process.env['NEXUS_LIVE'] === 'true'; -} diff --git a/src/run-live.ts b/src/run-live.ts deleted file mode 100644 index 57fba4a..0000000 --- a/src/run-live.ts +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env tsx -/** - * Run the memory benchmark against a live nexus-agents MCP server. - * - * Usage: NEXUS_LIVE=true npx tsx src/run-live.ts - */ - -import { runBenchmark } from './benchmark.js'; -import { isLiveMode } from './live-caller.js'; -import type { ToolCaller } from './benchmark.js'; - -async function main(): Promise { - if (!isLiveMode()) { - console.error('Set NEXUS_LIVE=true to run against a live MCP server.'); - process.exit(1); - } - - let caller: ToolCaller; - try { - const bridgePath = './live-bridge.js'; - const mod: Record = await import(bridgePath); - const factory = mod['createMcpCaller'] as (() => Promise) | undefined; - if (typeof factory !== 'function') throw new Error('live-bridge.ts must export createMcpCaller()'); - caller = await factory(); - } catch (e) { - const msg = e instanceof Error ? e.message : String(e); - console.error(`Failed to load live bridge: ${msg}`); - process.exit(1); - } - - console.log('Running memory benchmark against live MCP server...\n'); - const result = await runBenchmark(caller, { - queries: [ - { label: 'orchestration', query: 'multi-agent orchestration', minResults: 0 }, - { label: 'routing', query: 'model routing strategy', minResults: 0 }, - { label: 'consensus', query: 'consensus voting', minResults: 0 }, - ], - includeStats: true, - }); - console.log(JSON.stringify(result, null, 2)); -} - -void main(); diff --git a/src/types.ts b/src/types.ts index 8f9cea7..78dabad 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,6 +113,8 @@ export interface BenchmarkConfig { readonly includeStats?: boolean; /** Filter memory source. */ readonly source?: MemoryQueryInput['source']; + /** Per-call timeout (ms) for remote tool calls. Omit to use the default. */ + readonly timeoutMs?: number; } export interface QueryBenchmark {