Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/mcp-shared-timeout.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@hyperdx/api': patch
---

refactor(mcp): share wall-clock timeout handling across MCP tools via
`runWithTimeout`, and source the 30s ClickHouse query cap from one constant.
119 changes: 119 additions & 0 deletions packages/api/src/mcp/__tests__/timeout.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { runWithTimeout } from '@/mcp/utils/timeout';

const never = () => new Promise<never>(() => {});

describe('runWithTimeout', () => {
beforeEach(() => jest.useFakeTimers());
afterEach(() => jest.useRealTimers());

it('returns the value when work finishes in time and aborts the signal on exit', async () => {
let captured: AbortSignal | undefined;
const outcome = await runWithTimeout(
async signal => {
captured = signal;
return 'done';
},
{ timeoutMs: 1000 },
);
expect(outcome).toEqual({ timedOut: false, value: 'done' });
expect(captured?.aborted).toBe(true);
expect(jest.getTimerCount()).toBe(0);
});

it('times out and aborts the signal at the deadline', async () => {
let captured: AbortSignal | undefined;
const pending = runWithTimeout(
signal => {
captured = signal;
return never();
},
{ timeoutMs: 1000 },
);
await jest.advanceTimersByTimeAsync(999);
expect(captured?.aborted).toBe(false);
await jest.advanceTimersByTimeAsync(1);
await expect(pending).resolves.toEqual({ timedOut: true });
expect(captured?.aborted).toBe(true);
});

it('reports a timeout without grace even when work resolves on abort', async () => {
const pending = runWithTimeout(
signal =>
new Promise<string>(resolve =>
signal.addEventListener('abort', () => resolve('too late')),
),
{ timeoutMs: 1000 },
);
await jest.advanceTimersByTimeAsync(1000);
await expect(pending).resolves.toEqual({ timedOut: true });
});

it('reports a timeout without grace when work rejects on abort', async () => {
const pending = runWithTimeout(
signal =>
new Promise<never>((_, reject) =>
signal.addEventListener('abort', () => reject(new Error('aborted'))),
),
{ timeoutMs: 1000 },
);
await jest.advanceTimersByTimeAsync(1000);
await expect(pending).resolves.toEqual({ timedOut: true });
expect(jest.getTimerCount()).toBe(0);
});

it('keeps a result returned inside the grace window after the abort', async () => {
const pending = runWithTimeout(
signal =>
Comment thread
brandon-pereira marked this conversation as resolved.
Resolved
new Promise<string>(resolve =>
signal.addEventListener('abort', () => resolve('partial')),
),
{ timeoutMs: 1000, graceMs: 500 },
);
await jest.advanceTimersByTimeAsync(1000);
await expect(pending).resolves.toEqual({
timedOut: false,
value: 'partial',
});
});

it('stops waiting at the end of the grace window when work ignores the signal', async () => {
let settled = false;
const pending = runWithTimeout(never, {
timeoutMs: 1000,
graceMs: 500,
}).finally(() => {
settled = true;
});
await jest.advanceTimersByTimeAsync(1000);
expect(settled).toBe(false);
await jest.advanceTimersByTimeAsync(500);
await expect(pending).resolves.toEqual({ timedOut: true });
});

it('treats a rejection after the abort as a timeout', async () => {
const pending = runWithTimeout(
signal =>
new Promise<never>((_, reject) =>
signal.addEventListener('abort', () => reject(new Error('aborted'))),
),
{ timeoutMs: 1000, graceMs: 500 },
);
await jest.advanceTimersByTimeAsync(1000);
await expect(pending).resolves.toEqual({ timedOut: true });
});

it('rethrows a rejection raised before the deadline and aborts the signal', async () => {
let captured: AbortSignal | undefined;
await expect(
runWithTimeout(
signal => {
captured = signal;
return Promise.reject(new Error('boom'));
},
{ timeoutMs: 1000 },
),
).rejects.toThrow('boom');
expect(captured?.aborted).toBe(true);
expect(jest.getTimerCount()).toBe(0);
});
});
29 changes: 7 additions & 22 deletions packages/api/src/mcp/tools/dashboards/queryTiles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { z } from 'zod';
import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers';
import type { ToolRegistrar } from '@/mcp/tools/types';
import { mcpUserError } from '@/mcp/utils/errors';
import { runWithTimeout } from '@/mcp/utils/timeout';
import Dashboard from '@/models/dashboard';
import {
convertToExternalDashboard,
Expand Down Expand Up @@ -85,12 +86,8 @@ export class TileDeadlineError extends Error {
* checked BEFORE the query is issued: once the budget is spent, tiles PQueue
* schedules during the drain fail fast without touching ClickHouse.
*
* A tile that did start races the timer. When the deadline elapses we both
* reject AND abort the `AbortSignal` handed to `startWork`, so the in-flight
* ClickHouse query is cancelled server-side rather than left running headless
* until it finishes on its own. The signal is also aborted on any other exit
* (the work throwing, or resolving after we already lost the race is a no-op),
* so a query never outlives the call it belongs to.
* A tile that did start runs under `runWithTimeout`: at the deadline its
* query is cancelled and the tile rejects with `TileDeadlineError`.
*
* @internal Exported for testing only.
*/
Expand All @@ -103,23 +100,11 @@ export async function withDeadline<T>(
if (remaining <= 0) {
throw new TileDeadlineError();
}
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout>;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(() => {
// Cancel the query in ClickHouse, then surface the timeout.
controller.abort();
reject(new TileDeadlineError());
}, remaining);
});
try {
return await Promise.race([startWork(controller.signal), deadline]);
} finally {
clearTimeout(timer!);
// Belt-and-suspenders: abort on any exit so a query started by a thunk
// that then rejected for another reason is never left running.
controller.abort();
const outcome = await runWithTimeout(startWork, { timeoutMs: remaining });
if (outcome.timedOut) {
throw new TileDeadlineError();
}
return outcome.value;
}

/**
Expand Down
13 changes: 11 additions & 2 deletions packages/api/src/mcp/tools/query/denoise.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types';
import { ClickhouseClient } from '@/clickhouse';
import { getConnectionById } from '@/controllers/connection';
import { getSource } from '@/controllers/sources';
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';

import { resolveBodyExpression } from './helpers';

Expand Down Expand Up @@ -151,13 +152,21 @@ export async function denoiseSearchResults(
config: sampleConfig,
metadata,
querySettings: source.querySettings,
opts: { clickhouse_settings: { max_execution_time: 30 } },
opts: {
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
},
}),
clickhouseClient.queryChartConfig({
config: countConfig,
metadata,
querySettings: source.querySettings,
opts: { clickhouse_settings: { max_execution_time: 30 } },
opts: {
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
},
}),
]);
} catch {
Expand Down
9 changes: 7 additions & 2 deletions packages/api/src/mcp/tools/query/eventDeltas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { getConnectionById } from '@/controllers/connection';
import { getSource } from '@/controllers/sources';
import type { ToolRegistrar } from '@/mcp/tools/types';
import { mcpUserError } from '@/mcp/utils/errors';
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
import { trimToolResponse } from '@/utils/trimToolResponse';

import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
Expand Down Expand Up @@ -358,15 +359,19 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) {
query_params: targetSql.params,
format: 'JSON',
connectionId: source.connection.toString(),
clickhouse_settings: { max_execution_time: 30 },
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
abort_signal: abortController.signal,
}),
clickhouseClient.query({
query: baselineSql.sql,
query_params: baselineSql.params,
format: 'JSON',
connectionId: source.connection.toString(),
clickhouse_settings: { max_execution_time: 30 },
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
abort_signal: abortController.signal,
}),
]);
Expand Down
9 changes: 7 additions & 2 deletions packages/api/src/mcp/tools/query/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ import { getConnectionById } from '@/controllers/connection';
import { getSource } from '@/controllers/sources';
import type { McpErrorResult } from '@/mcp/utils/errors';
import { mcpServerError, mcpUserError } from '@/mcp/utils/errors';
import {
MCP_QUERY_MAX_EXECUTION_SEC,
MCP_TIMEOUT_GRACE_MS,
MCP_TOOL_TIMEOUT_MS,
} from '@/mcp/utils/timeout';
import {
convertToInternalTileConfig,
isConfigTile,
Expand Down Expand Up @@ -77,7 +82,7 @@ export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/;
/** ClickHouse settings applied to all MCP query-tool executions.
* readonly=2 so max_execution_time can be set (readonly=1 rejects it). */
export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = {
max_execution_time: 30,
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
readonly: '2',
};

Expand All @@ -86,7 +91,7 @@ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = {
* max_execution_time so ClickHouse returns a clean timeout before the HTTP
* connection is aborted.
*/
export const MCP_REQUEST_TIMEOUT = 32_000; // 30s query limit + 2s buffer
export const MCP_REQUEST_TIMEOUT = MCP_TOOL_TIMEOUT_MS + MCP_TIMEOUT_GRACE_MS;

// ─── Increase top-N cap hint ────────────────────────────────────────────────

Expand Down
9 changes: 7 additions & 2 deletions packages/api/src/mcp/tools/query/runEventPatterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
mcpServerError,
mcpUserError,
} from '@/mcp/utils/errors';
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
import { trimToolResponse } from '@/utils/trimToolResponse';

import {
Expand Down Expand Up @@ -186,7 +187,9 @@ export async function mineWindowPatterns(
metadata,
querySettings: source.querySettings,
opts: {
clickhouse_settings: { max_execution_time: 30 },
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
abort_signal: options?.abortSignal,
},
}),
Expand All @@ -195,7 +198,9 @@ export async function mineWindowPatterns(
metadata,
querySettings: source.querySettings,
opts: {
clickhouse_settings: { max_execution_time: 30 },
clickhouse_settings: {
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
},
abort_signal: options?.abortSignal,
},
}),
Expand Down
53 changes: 18 additions & 35 deletions packages/api/src/mcp/tools/sources/describeMetric.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
mcpUserError,
sanitizeFetchError,
} from '@/mcp/utils/errors';
import { MCP_TOOL_TIMEOUT_MS, runWithTimeout } from '@/mcp/utils/timeout';
import logger from '@/utils/logger';
import { trimToolResponse } from '@/utils/trimToolResponse';

Expand All @@ -32,8 +33,7 @@ import {
METRIC_DEFAULT_LOOKBACK_MS,
} from './metricKinds';

// Matches the 30s cap the MCP query tools use.
const DESCRIBE_TIMEOUT_MS = 30_000;
const DESCRIBE_TIMEOUT_MS = MCP_TOOL_TIMEOUT_MS;

// Server-side safety nets for the attribute-keys discovery query.
// Sample at most N rows that match (MetricName, time range), then
Expand Down Expand Up @@ -739,40 +739,23 @@ export function registerDescribeMetric({
// optional-field types into `unknown`, but the parser produces
// the typed shape we need for downstream calls.
const input = describeMetricSchema.parse(rawInput);
const controller = new AbortController();
// Hoist the timer handle so the finally block can cancel it on the
// success path — otherwise a stale controller.abort() fires
// DESCRIBE_TIMEOUT_MS after every successful call and the
// setTimeout closure stays pinned for the same duration.
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
controller.abort();
reject(new Error('DESCRIBE_METRIC_TIMEOUT'));
}, DESCRIBE_TIMEOUT_MS);
});
try {
return await Promise.race([
describeMetricImpl(teamId.toString(), input, controller.signal),
timeoutPromise,
]);
} catch (e) {
if (e instanceof Error && e.message === 'DESCRIBE_METRIC_TIMEOUT') {
logger.warn(
{ teamId, sourceId: input.sourceId, metricName: input.metricName },
'clickstack_describe_metric timed out',
);
return mcpServerError(
'Discovery timed out. The metric table may be under load or the ' +
'attribute set may be very high-cardinality. Try narrowing ' +
'startTime/endTime or setting sampleValues:false to skip the ' +
'value-sampling stage.',
);
}
throw e;
} finally {
if (timeoutId !== undefined) clearTimeout(timeoutId);
const outcome = await runWithTimeout(
signal => describeMetricImpl(teamId.toString(), input, signal),
{ timeoutMs: DESCRIBE_TIMEOUT_MS },
);
if (!outcome.timedOut) {
return outcome.value;
}
logger.warn(
{ teamId, sourceId: input.sourceId, metricName: input.metricName },
'clickstack_describe_metric timed out',
);
return mcpServerError(
'Discovery timed out. The metric table may be under load or the ' +
'attribute set may be very high-cardinality. Try narrowing ' +
'startTime/endTime or setting sampleValues:false to skip the ' +
'value-sampling stage.',
);
},
);
}
Loading
Loading