From 86bd48cc21e26171cd6cd9b8061d26026d6205a9 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Thu, 8 Oct 2026 16:41:43 -0600 Subject: [PATCH 1/4] refactor(mcp): share execution timeout handling across tools --- .changeset/mcp-shared-timeout.md | 6 ++ .../api/src/mcp/__tests__/timeout.test.ts | 88 +++++++++++++++++++ .../src/mcp/tools/dashboards/queryTiles.ts | 21 ++--- packages/api/src/mcp/tools/query/denoise.ts | 13 ++- .../api/src/mcp/tools/query/eventDeltas.ts | 9 +- packages/api/src/mcp/tools/query/helpers.ts | 9 +- .../src/mcp/tools/query/runEventPatterns.ts | 9 +- .../src/mcp/tools/sources/describeMetric.ts | 53 ++++------- .../src/mcp/tools/sources/describeSource.ts | 70 ++++++--------- .../api/src/mcp/tools/sources/listMetrics.ts | 53 ++++------- .../api/src/mcp/tools/sources/listSources.ts | 63 ++++++------- packages/api/src/mcp/utils/timeout.ts | 51 +++++++++++ 12 files changed, 267 insertions(+), 178 deletions(-) create mode 100644 .changeset/mcp-shared-timeout.md create mode 100644 packages/api/src/mcp/__tests__/timeout.test.ts create mode 100644 packages/api/src/mcp/utils/timeout.ts diff --git a/.changeset/mcp-shared-timeout.md b/.changeset/mcp-shared-timeout.md new file mode 100644 index 0000000000..e412f42588 --- /dev/null +++ b/.changeset/mcp-shared-timeout.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +--- + +refactor(mcp): share wall-clock timeout handling across MCP tools via +`runWithTimeout`, and source the 30s ClickHouse query cap from one constant. diff --git a/packages/api/src/mcp/__tests__/timeout.test.ts b/packages/api/src/mcp/__tests__/timeout.test.ts new file mode 100644 index 0000000000..d40cff13a5 --- /dev/null +++ b/packages/api/src/mcp/__tests__/timeout.test.ts @@ -0,0 +1,88 @@ +import { runWithTimeout } from '@/mcp/utils/timeout'; + +const never = () => new Promise(() => {}); + +describe('runWithTimeout', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + it('returns the value when work finishes in time and aborts the signal on exit', async () => { + let captured: AbortSignal | undefined; + const outcome = await runWithTimeout( + async signal => { + captured = signal; + return 'done'; + }, + { timeoutMs: 1000 }, + ); + expect(outcome).toEqual({ timedOut: false, value: 'done' }); + expect(captured?.aborted).toBe(true); + expect(jest.getTimerCount()).toBe(0); + }); + + it('times out and aborts the signal at the deadline', async () => { + let captured: AbortSignal | undefined; + const pending = runWithTimeout( + signal => { + captured = signal; + return never(); + }, + { timeoutMs: 1000 }, + ); + await jest.advanceTimersByTimeAsync(999); + expect(captured?.aborted).toBe(false); + await jest.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual({ timedOut: true }); + expect(captured?.aborted).toBe(true); + }); + + it('keeps a result returned inside the grace window after the abort', async () => { + const pending = runWithTimeout( + signal => + new Promise(resolve => + signal.addEventListener('abort', () => resolve('partial')), + ), + { timeoutMs: 1000, graceMs: 500 }, + ); + await jest.advanceTimersByTimeAsync(1000); + await expect(pending).resolves.toEqual({ + timedOut: false, + value: 'partial', + }); + }); + + it('stops waiting at the end of the grace window when work ignores the signal', async () => { + let settled = false; + const pending = runWithTimeout(never, { + timeoutMs: 1000, + graceMs: 500, + }).finally(() => { + settled = true; + }); + await jest.advanceTimersByTimeAsync(1000); + expect(settled).toBe(false); + await jest.advanceTimersByTimeAsync(500); + await expect(pending).resolves.toEqual({ timedOut: true }); + }); + + it('treats a rejection after the abort as a timeout', async () => { + const pending = runWithTimeout( + signal => + new Promise((_, reject) => + signal.addEventListener('abort', () => reject(new Error('aborted'))), + ), + { timeoutMs: 1000, graceMs: 500 }, + ); + await jest.advanceTimersByTimeAsync(1000); + await expect(pending).resolves.toEqual({ timedOut: true }); + }); + + it('rethrows a rejection raised before the deadline', async () => { + await expect( + runWithTimeout(() => Promise.reject(new Error('boom')), { + timeoutMs: 1000, + }), + ).rejects.toThrow('boom'); + expect(jest.getTimerCount()).toBe(0); + }); +}); diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 1923a7c67a..6d87051ae5 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -4,6 +4,7 @@ import { z } from 'zod'; import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; +import { runWithTimeout } from '@/mcp/utils/timeout'; import Dashboard from '@/models/dashboard'; import { convertToExternalDashboard, @@ -103,23 +104,11 @@ export async function withDeadline( if (remaining <= 0) { throw new TileDeadlineError(); } - const controller = new AbortController(); - let timer: ReturnType; - const deadline = new Promise((_, reject) => { - timer = setTimeout(() => { - // Cancel the query in ClickHouse, then surface the timeout. - controller.abort(); - reject(new TileDeadlineError()); - }, remaining); - }); - try { - return await Promise.race([startWork(controller.signal), deadline]); - } finally { - clearTimeout(timer!); - // Belt-and-suspenders: abort on any exit so a query started by a thunk - // that then rejected for another reason is never left running. - controller.abort(); + const outcome = await runWithTimeout(startWork, { timeoutMs: remaining }); + if (outcome.timedOut) { + throw new TileDeadlineError(); } + return outcome.value; } /** diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts index a3eeb0dba4..182aa54487 100644 --- a/packages/api/src/mcp/tools/query/denoise.ts +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -14,6 +14,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; +import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout'; import { resolveBodyExpression } from './helpers'; @@ -151,13 +152,21 @@ export async function denoiseSearchResults( config: sampleConfig, metadata, querySettings: source.querySettings, - opts: { clickhouse_settings: { max_execution_time: 30 } }, + opts: { + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, + }, }), clickhouseClient.queryChartConfig({ config: countConfig, metadata, querySettings: source.querySettings, - opts: { clickhouse_settings: { max_execution_time: 30 } }, + opts: { + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, + }, }), ]); } catch { diff --git a/packages/api/src/mcp/tools/query/eventDeltas.ts b/packages/api/src/mcp/tools/query/eventDeltas.ts index 8331187354..61145cdc75 100644 --- a/packages/api/src/mcp/tools/query/eventDeltas.ts +++ b/packages/api/src/mcp/tools/query/eventDeltas.ts @@ -16,6 +16,7 @@ import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; +import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog'; @@ -358,7 +359,9 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) { query_params: targetSql.params, format: 'JSON', connectionId: source.connection.toString(), - clickhouse_settings: { max_execution_time: 30 }, + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, abort_signal: abortController.signal, }), clickhouseClient.query({ @@ -366,7 +369,9 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) { query_params: baselineSql.params, format: 'JSON', connectionId: source.connection.toString(), - clickhouse_settings: { max_execution_time: 30 }, + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, abort_signal: abortController.signal, }), ]); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index f1dcbe426a..708b96c0f7 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -30,6 +30,11 @@ import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import type { McpErrorResult } from '@/mcp/utils/errors'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; +import { + MCP_QUERY_MAX_EXECUTION_SEC, + MCP_TIMEOUT_GRACE_MS, + MCP_TOOL_TIMEOUT_MS, +} from '@/mcp/utils/timeout'; import { convertToInternalTileConfig, isConfigTile, @@ -77,7 +82,7 @@ export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; /** ClickHouse settings applied to all MCP query-tool executions. * readonly=2 so max_execution_time can be set (readonly=1 rejects it). */ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = { - max_execution_time: 30, + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, readonly: '2', }; @@ -86,7 +91,7 @@ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = { * max_execution_time so ClickHouse returns a clean timeout before the HTTP * connection is aborted. */ -export const MCP_REQUEST_TIMEOUT = 32_000; // 30s query limit + 2s buffer +export const MCP_REQUEST_TIMEOUT = MCP_TOOL_TIMEOUT_MS + MCP_TIMEOUT_GRACE_MS; // ─── Increase top-N cap hint ──────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index ae4cec39b8..b9a2464c4d 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -18,6 +18,7 @@ import { mcpServerError, mcpUserError, } from '@/mcp/utils/errors'; +import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { @@ -186,7 +187,9 @@ export async function mineWindowPatterns( metadata, querySettings: source.querySettings, opts: { - clickhouse_settings: { max_execution_time: 30 }, + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, abort_signal: options?.abortSignal, }, }), @@ -195,7 +198,9 @@ export async function mineWindowPatterns( metadata, querySettings: source.querySettings, opts: { - clickhouse_settings: { max_execution_time: 30 }, + clickhouse_settings: { + max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC, + }, abort_signal: options?.abortSignal, }, }), diff --git a/packages/api/src/mcp/tools/sources/describeMetric.ts b/packages/api/src/mcp/tools/sources/describeMetric.ts index 1a3312cb16..5547f40dde 100644 --- a/packages/api/src/mcp/tools/sources/describeMetric.ts +++ b/packages/api/src/mcp/tools/sources/describeMetric.ts @@ -23,6 +23,7 @@ import { mcpUserError, sanitizeFetchError, } from '@/mcp/utils/errors'; +import { MCP_TOOL_TIMEOUT_MS, runWithTimeout } from '@/mcp/utils/timeout'; import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; @@ -32,8 +33,7 @@ import { METRIC_DEFAULT_LOOKBACK_MS, } from './metricKinds'; -// Matches the 30s cap the MCP query tools use. -const DESCRIBE_TIMEOUT_MS = 30_000; +const DESCRIBE_TIMEOUT_MS = MCP_TOOL_TIMEOUT_MS; // Server-side safety nets for the attribute-keys discovery query. // Sample at most N rows that match (MetricName, time range), then @@ -739,40 +739,23 @@ export function registerDescribeMetric({ // optional-field types into `unknown`, but the parser produces // the typed shape we need for downstream calls. const input = describeMetricSchema.parse(rawInput); - const controller = new AbortController(); - // Hoist the timer handle so the finally block can cancel it on the - // success path — otherwise a stale controller.abort() fires - // DESCRIBE_TIMEOUT_MS after every successful call and the - // setTimeout closure stays pinned for the same duration. - let timeoutId: ReturnType | undefined; - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - controller.abort(); - reject(new Error('DESCRIBE_METRIC_TIMEOUT')); - }, DESCRIBE_TIMEOUT_MS); - }); - try { - return await Promise.race([ - describeMetricImpl(teamId.toString(), input, controller.signal), - timeoutPromise, - ]); - } catch (e) { - if (e instanceof Error && e.message === 'DESCRIBE_METRIC_TIMEOUT') { - logger.warn( - { teamId, sourceId: input.sourceId, metricName: input.metricName }, - 'clickstack_describe_metric timed out', - ); - return mcpServerError( - 'Discovery timed out. The metric table may be under load or the ' + - 'attribute set may be very high-cardinality. Try narrowing ' + - 'startTime/endTime or setting sampleValues:false to skip the ' + - 'value-sampling stage.', - ); - } - throw e; - } finally { - if (timeoutId !== undefined) clearTimeout(timeoutId); + const outcome = await runWithTimeout( + signal => describeMetricImpl(teamId.toString(), input, signal), + { timeoutMs: DESCRIBE_TIMEOUT_MS }, + ); + if (!outcome.timedOut) { + return outcome.value; } + logger.warn( + { teamId, sourceId: input.sourceId, metricName: input.metricName }, + 'clickstack_describe_metric timed out', + ); + return mcpServerError( + 'Discovery timed out. The metric table may be under load or the ' + + 'attribute set may be very high-cardinality. Try narrowing ' + + 'startTime/endTime or setting sampleValues:false to skip the ' + + 'value-sampling stage.', + ); }, ); } diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 5b22e00acc..6cce352b72 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -14,6 +14,12 @@ import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import type { ToolRegistrar, ToolResult } from '@/mcp/tools/types'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; +import { + MCP_TIMEOUT_GRACE_MS, + MCP_TOOL_TIMEOUT_MS, + runWithTimeout, + type TimeoutOutcome, +} from '@/mcp/utils/timeout'; import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; @@ -33,15 +39,11 @@ import { extractSourceConfig } from './schemas'; const VALUE_SAMPLE_LOOKBACK_MS = 24 * 60 * 60 * 1000; // 24 hours // When discovery's signal is aborted; it then returns what it has so far. -// Matches the 30s budget of the other MCP query tools. -export const DESCRIBE_TIMEOUT_MS = 30_000; +export const DESCRIBE_TIMEOUT_MS = MCP_TOOL_TIMEOUT_MS; // Extra time after the abort before the handler stops waiting, in case a -// ClickHouse call ignores the signal. 30s + 2s lines up with -// MCP_REQUEST_TIMEOUT in query/helpers.ts. -export const DESCRIBE_BACKSTOP_MS = 2_000; - -const BACKSTOP = Symbol('backstop'); +// ClickHouse call ignores the signal. +export const DESCRIBE_BACKSTOP_MS = MCP_TIMEOUT_GRACE_MS; // Max sampled values per low-cardinality column / map attribute key. const MAX_LC_VALUES = 20; @@ -578,25 +580,8 @@ export function registerDescribeSource({ }), }, async ({ sourceId }) => { - const controller = new AbortController(); const progress: DescribeProgress = {}; - // At the deadline, abort so discovery skips its remaining stages and - // returns what it has. The backstop covers ClickHouse calls that - // ignore the signal: it answers from the latest snapshot instead of - // waiting on them. - const abortTimer = setTimeout( - () => controller.abort(), - DESCRIBE_TIMEOUT_MS, - ); - let backstopTimer: ReturnType | undefined; - const backstop = new Promise(resolve => { - backstopTimer = setTimeout( - () => resolve(BACKSTOP), - DESCRIBE_TIMEOUT_MS + DESCRIBE_BACKSTOP_MS, - ); - }); - const span = trace.getActiveSpan(); const recordOutcome = ( outcome: 'complete' | 'partial' | 'timeout' | 'error', @@ -622,36 +607,31 @@ export function registerDescribeSource({ ); }; + // At the deadline, discovery skips its remaining stages and returns + // what it has. The grace window covers ClickHouse calls that ignore + // the signal: past it we answer from the latest snapshot instead. + let outcome: TimeoutOutcome; try { - const result = await Promise.race([ - describeSourceSchema( - teamId.toString(), - sourceId, - controller.signal, - progress, - ), - backstop, - ]); - if (result === BACKSTOP) { - return timedOutResult(); - } - recordOutcome( - result.isError ? 'error' : progress.partial ? 'partial' : 'complete', + outcome = await runWithTimeout( + signal => + describeSourceSchema(teamId.toString(), sourceId, signal, progress), + { timeoutMs: DESCRIBE_TIMEOUT_MS, graceMs: DESCRIBE_BACKSTOP_MS }, ); - return result; } catch (e) { - if (controller.signal.aborted) { - return timedOutResult(); - } logger.warn( { teamId, sourceId, error: e }, 'Failed to describe source schema', ); throw e; - } finally { - clearTimeout(abortTimer); - clearTimeout(backstopTimer); } + if (outcome.timedOut) { + return timedOutResult(); + } + const result = outcome.value; + recordOutcome( + result.isError ? 'error' : progress.partial ? 'partial' : 'complete', + ); + return result; }, ); } diff --git a/packages/api/src/mcp/tools/sources/listMetrics.ts b/packages/api/src/mcp/tools/sources/listMetrics.ts index c3de415be9..a3fe42b9d7 100644 --- a/packages/api/src/mcp/tools/sources/listMetrics.ts +++ b/packages/api/src/mcp/tools/sources/listMetrics.ts @@ -9,6 +9,7 @@ import { getSource } from '@/controllers/sources'; import { parseTimeRange } from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; +import { MCP_TOOL_TIMEOUT_MS, runWithTimeout } from '@/mcp/utils/timeout'; import logger from '@/utils/logger'; import { KIND_TIMED_OUT_ERROR, scanKindsForPage } from './listMetricsPage'; @@ -25,9 +26,8 @@ import { METRIC_DEFAULT_LOOKBACK_MS, } from './metricKinds'; -// Wall-clock budget for the whole call, matching the 30s cap the MCP query -// tools use. -const LIST_TIMEOUT_MS = 30_000; +// Wall-clock budget for the whole call. +const LIST_TIMEOUT_MS = MCP_TOOL_TIMEOUT_MS; // Time held back from the name scan for the unit/description lookup and // response assembly. Kinds still scanning when it starts are reported as @@ -75,40 +75,21 @@ export function registerListMetrics({ listMetricsSchema.parse(rawInput); const deadlineAt = Date.now() + LIST_TIMEOUT_MS; - const controller = new AbortController(); - let timeoutId: ReturnType | undefined; - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - controller.abort(); - reject(new Error('LIST_METRICS_TIMEOUT')); - }, LIST_TIMEOUT_MS); - }); - - try { - return await Promise.race([ - listMetricsImpl( - teamId.toString(), - input, - deadlineAt, - controller.signal, - ), - timeoutPromise, - ]); - } catch (e) { - if (e instanceof Error && e.message === 'LIST_METRICS_TIMEOUT') { - logger.warn( - { teamId, sourceId: input.sourceId }, - 'clickstack_list_metrics timed out', - ); - return mcpServerError( - 'clickstack_list_metrics timed out. Try narrowing the time window ' + - '(startTime/endTime), pinning a single `kind`, or adding a namePattern filter.', - ); - } - throw e; - } finally { - clearTimeout(timeoutId); + const outcome = await runWithTimeout( + signal => listMetricsImpl(teamId.toString(), input, deadlineAt, signal), + { timeoutMs: LIST_TIMEOUT_MS }, + ); + if (!outcome.timedOut) { + return outcome.value; } + logger.warn( + { teamId, sourceId: input.sourceId }, + 'clickstack_list_metrics timed out', + ); + return mcpServerError( + 'clickstack_list_metrics timed out. Try narrowing the time window ' + + '(startTime/endTime), pinning a single `kind`, or adding a namePattern filter.', + ); }, ); } diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index a85061d367..b7ce2beac9 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -17,6 +17,7 @@ import { } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import type { ToolRegistrar } from '@/mcp/tools/types'; +import { runWithTimeout } from '@/mcp/utils/timeout'; import logger from '@/utils/logger'; import { QUERYABLE_METRIC_KINDS, sanitizeMetricTables } from './metricKinds'; @@ -35,15 +36,15 @@ const MAX_PREVIEW_NAMES_PER_KIND = 10; const PREVIEW_CONCURRENCY = 6; // Server-side cap per preview query, slightly under the wall-clock budget -// so break-mode partial results make it back and get attached before the -// AbortController fires. +// so break-mode partial results make it back and get attached before it +// aborts. const PREVIEW_QUERY_MAX_EXECUTION_SEC = 2.5; /** * ClickhouseClient that caps every query with max_execution_time + * timeout_overflow_mode=break, so a slow sampling query returns whatever * rows ClickHouse processed within the budget instead of timing out with - * nothing. The AbortController in attachMetricNamePreviews remains the + * nothing. The timeout in attachMetricNamePreviews remains the * wall-clock backstop for stalls the server-side cap cannot cover (e.g. * network). */ @@ -111,12 +112,6 @@ async function attachMetricNamePreviews({ }), ); - const controller = new AbortController(); - const timeoutId = setTimeout( - () => controller.abort(), - METRIC_PREVIEW_TIMEOUT_MS, - ); - const now = new Date(); // Multiple sources can point at the same physical table (e.g. cloned // source configs) — dedupe the sampling per (connection, db, table, @@ -129,13 +124,13 @@ async function attachMetricNamePreviews({ // clickstack_timeseries / clickstack_table, and summary metrics are not. const queryableKinds = new Set(QUERYABLE_METRIC_KINDS); - const tasks: Array<() => Promise> = []; + const tasks: Array<(signal: AbortSignal) => Promise> = []; for (const entry of entries) { const client = clients.get(entry.connectionId); if (!client) continue; for (const [kind, tableName] of Object.entries(entry.metricTables)) { if (!queryableKinds.has(kind)) continue; - tasks.push(async () => { + tasks.push(async signal => { // JSON-encode the tuple so delimiter characters inside a // component cannot make two distinct tuples collide. const cacheKey = JSON.stringify([ @@ -154,7 +149,7 @@ async function attachMetricNamePreviews({ connectionId: entry.connectionId, now, timestampValueExpression: entry.timestampValueExpression, - signal: controller.signal, + signal, maxNames: MAX_PREVIEW_NAMES_PER_KIND, enrich: false, }).then(samples => samples.map(s => s.name)); @@ -172,35 +167,27 @@ async function attachMetricNamePreviews({ } } - const abortedPromise = new Promise(resolve => { - controller.signal.addEventListener('abort', () => resolve(), { - once: true, - }); - }); - const queue = new PQueue({ concurrency: PREVIEW_CONCURRENCY }); - const drained = Promise.all( - tasks.map(task => - queue.add(async () => { - // Don't start new sampling once the budget has expired. - if (controller.signal.aborted) return; - try { - await task(); - } catch { - // Best-effort: individual sampling failures never fail the call. - } - }), - ), + // Stops waiting at the budget even if a ClickHouse call ignores the + // signal, so it cannot hold list_sources past it. + await runWithTimeout( + signal => + Promise.all( + tasks.map(task => + queue.add(async () => { + // Don't start new sampling once the budget has expired. + if (signal.aborted) return; + try { + await task(signal); + } catch { + // Best-effort: individual sampling failures never fail the call. + } + }), + ), + ), + { timeoutMs: METRIC_PREVIEW_TIMEOUT_MS }, ); - try { - // Race the queue against the abort so a ClickHouse call that ignores - // the signal cannot hold list_sources past its budget. - await Promise.race([drained, abortedPromise]); - } finally { - clearTimeout(timeoutId); - } - for (const [summary, preview] of previews) { if (preview.size > 0) { summary.metricNamesPreview = Object.fromEntries(preview); diff --git a/packages/api/src/mcp/utils/timeout.ts b/packages/api/src/mcp/utils/timeout.ts new file mode 100644 index 0000000000..217e46b0e4 --- /dev/null +++ b/packages/api/src/mcp/utils/timeout.ts @@ -0,0 +1,51 @@ +/** Server-side ClickHouse max_execution_time for MCP tool queries. */ +export const MCP_QUERY_MAX_EXECUTION_SEC = 30; + +/** Default wall-clock budget for a whole MCP tool call. */ +export const MCP_TOOL_TIMEOUT_MS = MCP_QUERY_MAX_EXECUTION_SEC * 1000; + +/** + * Extra time past a deadline to wait for ClickHouse to return its own clean + * timeout, or for work to wind down after its signal aborts. + */ +export const MCP_TIMEOUT_GRACE_MS = 2_000; + +export type TimeoutOutcome = + | { timedOut: false; value: T } + | { timedOut: true }; + +const TIMED_OUT = Symbol('timedOut'); + +/** + * Run `work` under a wall-clock budget. At `timeoutMs` the signal handed to + * `work` aborts; `graceMs` later the call stops waiting and reports a timeout. + * Work that settles inside the grace window keeps its result, so it can + * return partial output once it sees the abort. A rejection after the abort + * counts as a timeout. The signal also aborts on every exit so no query + * outlives the call. + */ +export async function runWithTimeout( + work: (signal: AbortSignal) => Promise, + { timeoutMs, graceMs = 0 }: { timeoutMs: number; graceMs?: number }, +): Promise> { + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), timeoutMs); + let backstopTimer: ReturnType | undefined; + const backstop = new Promise(resolve => { + backstopTimer = setTimeout(() => resolve(TIMED_OUT), timeoutMs + graceMs); + }); + + try { + const result = await Promise.race([work(controller.signal), backstop]); + return result === TIMED_OUT + ? { timedOut: true } + : { timedOut: false, value: result }; + } catch (e) { + if (controller.signal.aborted) return { timedOut: true }; + throw e; + } finally { + clearTimeout(abortTimer); + clearTimeout(backstopTimer); + controller.abort(); + } +} From de0e1d55ae1dbdb367ebe21dc5d981fe50a74fc7 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 9 Oct 2026 08:58:41 -0600 Subject: [PATCH 2/4] fix(mcp): report a timeout when zero-grace work resolves on abort (greptile P2) --- .../api/src/mcp/__tests__/timeout.test.ts | 39 +++++++++++++++++-- .../src/mcp/tools/dashboards/queryTiles.ts | 8 +--- packages/api/src/mcp/utils/timeout.ts | 30 +++++++++----- 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/packages/api/src/mcp/__tests__/timeout.test.ts b/packages/api/src/mcp/__tests__/timeout.test.ts index d40cff13a5..a97d2258e9 100644 --- a/packages/api/src/mcp/__tests__/timeout.test.ts +++ b/packages/api/src/mcp/__tests__/timeout.test.ts @@ -36,6 +36,31 @@ describe('runWithTimeout', () => { expect(captured?.aborted).toBe(true); }); + it('reports a timeout without grace even when work resolves on abort', async () => { + const pending = runWithTimeout( + signal => + new Promise(resolve => + signal.addEventListener('abort', () => resolve('too late')), + ), + { timeoutMs: 1000 }, + ); + await jest.advanceTimersByTimeAsync(1000); + await expect(pending).resolves.toEqual({ timedOut: true }); + }); + + it('reports a timeout without grace when work rejects on abort', async () => { + const pending = runWithTimeout( + signal => + new Promise((_, reject) => + signal.addEventListener('abort', () => reject(new Error('aborted'))), + ), + { timeoutMs: 1000 }, + ); + await jest.advanceTimersByTimeAsync(1000); + await expect(pending).resolves.toEqual({ timedOut: true }); + expect(jest.getTimerCount()).toBe(0); + }); + it('keeps a result returned inside the grace window after the abort', async () => { const pending = runWithTimeout( signal => @@ -77,12 +102,18 @@ describe('runWithTimeout', () => { await expect(pending).resolves.toEqual({ timedOut: true }); }); - it('rethrows a rejection raised before the deadline', async () => { + it('rethrows a rejection raised before the deadline and aborts the signal', async () => { + let captured: AbortSignal | undefined; await expect( - runWithTimeout(() => Promise.reject(new Error('boom')), { - timeoutMs: 1000, - }), + runWithTimeout( + signal => { + captured = signal; + return Promise.reject(new Error('boom')); + }, + { timeoutMs: 1000 }, + ), ).rejects.toThrow('boom'); + expect(captured?.aborted).toBe(true); expect(jest.getTimerCount()).toBe(0); }); }); diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 6d87051ae5..d4b9e3dc48 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -86,12 +86,8 @@ export class TileDeadlineError extends Error { * checked BEFORE the query is issued: once the budget is spent, tiles PQueue * schedules during the drain fail fast without touching ClickHouse. * - * A tile that did start races the timer. When the deadline elapses we both - * reject AND abort the `AbortSignal` handed to `startWork`, so the in-flight - * ClickHouse query is cancelled server-side rather than left running headless - * until it finishes on its own. The signal is also aborted on any other exit - * (the work throwing, or resolving after we already lost the race is a no-op), - * so a query never outlives the call it belongs to. + * A tile that did start runs under `runWithTimeout`: at the deadline its + * query is cancelled and the tile rejects with `TileDeadlineError`. * * @internal Exported for testing only. */ diff --git a/packages/api/src/mcp/utils/timeout.ts b/packages/api/src/mcp/utils/timeout.ts index 217e46b0e4..10d4e9c7e8 100644 --- a/packages/api/src/mcp/utils/timeout.ts +++ b/packages/api/src/mcp/utils/timeout.ts @@ -1,8 +1,12 @@ /** Server-side ClickHouse max_execution_time for MCP tool queries. */ export const MCP_QUERY_MAX_EXECUTION_SEC = 30; -/** Default wall-clock budget for a whole MCP tool call. */ -export const MCP_TOOL_TIMEOUT_MS = MCP_QUERY_MAX_EXECUTION_SEC * 1000; +/** + * Default wall-clock budget for a whole MCP tool call. Kept equal to the + * per-query cap so a single query cannot outlive the call, but set + * separately so the two can be tuned independently. + */ +export const MCP_TOOL_TIMEOUT_MS = 30_000; /** * Extra time past a deadline to wait for ClickHouse to return its own clean @@ -29,14 +33,23 @@ export async function runWithTimeout( { timeoutMs, graceMs = 0 }: { timeoutMs: number; graceMs?: number }, ): Promise> { const controller = new AbortController(); - const abortTimer = setTimeout(() => controller.abort(), timeoutMs); - let backstopTimer: ReturnType | undefined; - const backstop = new Promise(resolve => { - backstopTimer = setTimeout(() => resolve(TIMED_OUT), timeoutMs + graceMs); + let timer: ReturnType | undefined; + const deadline = new Promise(resolve => { + timer = setTimeout(() => { + if (graceMs <= 0) { + // Settle before aborting so a result produced by an abort listener + // cannot win the race and be reported as success. + resolve(TIMED_OUT); + controller.abort(); + return; + } + controller.abort(); + timer = setTimeout(() => resolve(TIMED_OUT), graceMs); + }, timeoutMs); }); try { - const result = await Promise.race([work(controller.signal), backstop]); + const result = await Promise.race([work(controller.signal), deadline]); return result === TIMED_OUT ? { timedOut: true } : { timedOut: false, value: result }; @@ -44,8 +57,7 @@ export async function runWithTimeout( if (controller.signal.aborted) return { timedOut: true }; throw e; } finally { - clearTimeout(abortTimer); - clearTimeout(backstopTimer); + clearTimeout(timer); controller.abort(); } } From 9323934f812cb38f5604b16ff354aeabb684cba7 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 9 Oct 2026 09:14:47 -0600 Subject: [PATCH 3/4] fix(mcp): derive MCP_REQUEST_TIMEOUT from the query cap it buffers (review P2) --- packages/api/src/mcp/tools/query/helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 708b96c0f7..919749f296 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -33,7 +33,6 @@ import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; import { MCP_QUERY_MAX_EXECUTION_SEC, MCP_TIMEOUT_GRACE_MS, - MCP_TOOL_TIMEOUT_MS, } from '@/mcp/utils/timeout'; import { convertToInternalTileConfig, @@ -91,7 +90,8 @@ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = { * max_execution_time so ClickHouse returns a clean timeout before the HTTP * connection is aborted. */ -export const MCP_REQUEST_TIMEOUT = MCP_TOOL_TIMEOUT_MS + MCP_TIMEOUT_GRACE_MS; +export const MCP_REQUEST_TIMEOUT = + MCP_QUERY_MAX_EXECUTION_SEC * 1000 + MCP_TIMEOUT_GRACE_MS; // ─── Increase top-N cap hint ──────────────────────────────────────────────── From 4c55a4a665261d83e0726116cd15b5330eefb05f Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 9 Oct 2026 12:23:55 -0600 Subject: [PATCH 4/4] refactor(mcp): derive the tool timeout from the query cap in seconds --- packages/api/src/mcp/tools/query/helpers.ts | 4 ++-- packages/api/src/mcp/tools/sources/describeSource.ts | 3 ++- packages/api/src/mcp/utils/timeout.ts | 8 ++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 919749f296..708b96c0f7 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -33,6 +33,7 @@ import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; import { MCP_QUERY_MAX_EXECUTION_SEC, MCP_TIMEOUT_GRACE_MS, + MCP_TOOL_TIMEOUT_MS, } from '@/mcp/utils/timeout'; import { convertToInternalTileConfig, @@ -90,8 +91,7 @@ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = { * max_execution_time so ClickHouse returns a clean timeout before the HTTP * connection is aborted. */ -export const MCP_REQUEST_TIMEOUT = - MCP_QUERY_MAX_EXECUTION_SEC * 1000 + MCP_TIMEOUT_GRACE_MS; +export const MCP_REQUEST_TIMEOUT = MCP_TOOL_TIMEOUT_MS + MCP_TIMEOUT_GRACE_MS; // ─── Increase top-N cap hint ──────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 6cce352b72..193a6f0a6d 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -15,6 +15,7 @@ import { getSource } from '@/controllers/sources'; import type { ToolRegistrar, ToolResult } from '@/mcp/tools/types'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; import { + MCP_QUERY_MAX_EXECUTION_SEC, MCP_TIMEOUT_GRACE_MS, MCP_TOOL_TIMEOUT_MS, runWithTimeout, @@ -568,7 +569,7 @@ export function registerDescribeSource({ '(SeverityText, StatusCode, ServiceName, etc.) — use these in filters instead of guessing\n' + '- mapAttributeValues: sampled top values for the most common map attribute keys ' + "(e.g. ResourceAttributes['service.name'] top values) — requires rollup tables\n\n" + - `Value sampling stops after ${DESCRIBE_TIMEOUT_MS / 1000} seconds. If it is cut short, the result still includes ` + + `Value sampling stops after ${MCP_QUERY_MAX_EXECUTION_SEC} seconds. If it is cut short, the result still includes ` + 'the columns and sets partial: true with skippedStages listing what is missing.\n\n' + 'Cost: one describe call prevents 3–5 exploratory queries against non-existent columns.', inputSchema: z.object({ diff --git a/packages/api/src/mcp/utils/timeout.ts b/packages/api/src/mcp/utils/timeout.ts index 10d4e9c7e8..4152435bad 100644 --- a/packages/api/src/mcp/utils/timeout.ts +++ b/packages/api/src/mcp/utils/timeout.ts @@ -1,12 +1,8 @@ /** Server-side ClickHouse max_execution_time for MCP tool queries. */ export const MCP_QUERY_MAX_EXECUTION_SEC = 30; -/** - * Default wall-clock budget for a whole MCP tool call. Kept equal to the - * per-query cap so a single query cannot outlive the call, but set - * separately so the two can be tuned independently. - */ -export const MCP_TOOL_TIMEOUT_MS = 30_000; +/** Default wall-clock budget for a whole MCP tool call. */ +export const MCP_TOOL_TIMEOUT_MS = MCP_QUERY_MAX_EXECUTION_SEC * 1000; /** * Extra time past a deadline to wait for ClickHouse to return its own clean