Skip to content

Commit ef34105

Browse files
Merge branch 'main' into claude/ci-int-unprefixed-output
2 parents c599dd6 + 7aeb621 commit ef34105

12 files changed

Lines changed: 310 additions & 185 deletions

File tree

‎.changeset/mcp-shared-timeout.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@hyperdx/api': patch
3+
---
4+
5+
refactor(mcp): share wall-clock timeout handling across MCP tools via
6+
`runWithTimeout`, and source the 30s ClickHouse query cap from one constant.
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { runWithTimeout } from '@/mcp/utils/timeout';
2+
3+
const never = () => new Promise<never>(() => {});
4+
5+
describe('runWithTimeout', () => {
6+
beforeEach(() => jest.useFakeTimers());
7+
afterEach(() => jest.useRealTimers());
8+
9+
it('returns the value when work finishes in time and aborts the signal on exit', async () => {
10+
let captured: AbortSignal | undefined;
11+
const outcome = await runWithTimeout(
12+
async signal => {
13+
captured = signal;
14+
return 'done';
15+
},
16+
{ timeoutMs: 1000 },
17+
);
18+
expect(outcome).toEqual({ timedOut: false, value: 'done' });
19+
expect(captured?.aborted).toBe(true);
20+
expect(jest.getTimerCount()).toBe(0);
21+
});
22+
23+
it('times out and aborts the signal at the deadline', async () => {
24+
let captured: AbortSignal | undefined;
25+
const pending = runWithTimeout(
26+
signal => {
27+
captured = signal;
28+
return never();
29+
},
30+
{ timeoutMs: 1000 },
31+
);
32+
await jest.advanceTimersByTimeAsync(999);
33+
expect(captured?.aborted).toBe(false);
34+
await jest.advanceTimersByTimeAsync(1);
35+
await expect(pending).resolves.toEqual({ timedOut: true });
36+
expect(captured?.aborted).toBe(true);
37+
});
38+
39+
it('reports a timeout without grace even when work resolves on abort', async () => {
40+
const pending = runWithTimeout(
41+
signal =>
42+
new Promise<string>(resolve =>
43+
signal.addEventListener('abort', () => resolve('too late')),
44+
),
45+
{ timeoutMs: 1000 },
46+
);
47+
await jest.advanceTimersByTimeAsync(1000);
48+
await expect(pending).resolves.toEqual({ timedOut: true });
49+
});
50+
51+
it('reports a timeout without grace when work rejects on abort', async () => {
52+
const pending = runWithTimeout(
53+
signal =>
54+
new Promise<never>((_, reject) =>
55+
signal.addEventListener('abort', () => reject(new Error('aborted'))),
56+
),
57+
{ timeoutMs: 1000 },
58+
);
59+
await jest.advanceTimersByTimeAsync(1000);
60+
await expect(pending).resolves.toEqual({ timedOut: true });
61+
expect(jest.getTimerCount()).toBe(0);
62+
});
63+
64+
it('keeps a result returned inside the grace window after the abort', async () => {
65+
const pending = runWithTimeout(
66+
signal =>
67+
new Promise<string>(resolve =>
68+
signal.addEventListener('abort', () => resolve('partial')),
69+
),
70+
{ timeoutMs: 1000, graceMs: 500 },
71+
);
72+
await jest.advanceTimersByTimeAsync(1000);
73+
await expect(pending).resolves.toEqual({
74+
timedOut: false,
75+
value: 'partial',
76+
});
77+
});
78+
79+
it('stops waiting at the end of the grace window when work ignores the signal', async () => {
80+
let settled = false;
81+
const pending = runWithTimeout(never, {
82+
timeoutMs: 1000,
83+
graceMs: 500,
84+
}).finally(() => {
85+
settled = true;
86+
});
87+
await jest.advanceTimersByTimeAsync(1000);
88+
expect(settled).toBe(false);
89+
await jest.advanceTimersByTimeAsync(500);
90+
await expect(pending).resolves.toEqual({ timedOut: true });
91+
});
92+
93+
it('treats a rejection after the abort as a timeout', async () => {
94+
const pending = runWithTimeout(
95+
signal =>
96+
new Promise<never>((_, reject) =>
97+
signal.addEventListener('abort', () => reject(new Error('aborted'))),
98+
),
99+
{ timeoutMs: 1000, graceMs: 500 },
100+
);
101+
await jest.advanceTimersByTimeAsync(1000);
102+
await expect(pending).resolves.toEqual({ timedOut: true });
103+
});
104+
105+
it('rethrows a rejection raised before the deadline and aborts the signal', async () => {
106+
let captured: AbortSignal | undefined;
107+
await expect(
108+
runWithTimeout(
109+
signal => {
110+
captured = signal;
111+
return Promise.reject(new Error('boom'));
112+
},
113+
{ timeoutMs: 1000 },
114+
),
115+
).rejects.toThrow('boom');
116+
expect(captured?.aborted).toBe(true);
117+
expect(jest.getTimerCount()).toBe(0);
118+
});
119+
});

‎packages/api/src/mcp/tools/dashboards/queryTiles.ts‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { z } from 'zod';
44
import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers';
55
import type { ToolRegistrar } from '@/mcp/tools/types';
66
import { mcpUserError } from '@/mcp/utils/errors';
7+
import { runWithTimeout } from '@/mcp/utils/timeout';
78
import Dashboard from '@/models/dashboard';
89
import {
910
convertToExternalDashboard,
@@ -85,12 +86,8 @@ export class TileDeadlineError extends Error {
8586
* checked BEFORE the query is issued: once the budget is spent, tiles PQueue
8687
* schedules during the drain fail fast without touching ClickHouse.
8788
*
88-
* A tile that did start races the timer. When the deadline elapses we both
89-
* reject AND abort the `AbortSignal` handed to `startWork`, so the in-flight
90-
* ClickHouse query is cancelled server-side rather than left running headless
91-
* until it finishes on its own. The signal is also aborted on any other exit
92-
* (the work throwing, or resolving after we already lost the race is a no-op),
93-
* so a query never outlives the call it belongs to.
89+
* A tile that did start runs under `runWithTimeout`: at the deadline its
90+
* query is cancelled and the tile rejects with `TileDeadlineError`.
9491
*
9592
* @internal Exported for testing only.
9693
*/
@@ -103,23 +100,11 @@ export async function withDeadline<T>(
103100
if (remaining <= 0) {
104101
throw new TileDeadlineError();
105102
}
106-
const controller = new AbortController();
107-
let timer: ReturnType<typeof setTimeout>;
108-
const deadline = new Promise<never>((_, reject) => {
109-
timer = setTimeout(() => {
110-
// Cancel the query in ClickHouse, then surface the timeout.
111-
controller.abort();
112-
reject(new TileDeadlineError());
113-
}, remaining);
114-
});
115-
try {
116-
return await Promise.race([startWork(controller.signal), deadline]);
117-
} finally {
118-
clearTimeout(timer!);
119-
// Belt-and-suspenders: abort on any exit so a query started by a thunk
120-
// that then rejected for another reason is never left running.
121-
controller.abort();
103+
const outcome = await runWithTimeout(startWork, { timeoutMs: remaining });
104+
if (outcome.timedOut) {
105+
throw new TileDeadlineError();
122106
}
107+
return outcome.value;
123108
}
124109

125110
/**

‎packages/api/src/mcp/tools/query/denoise.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types';
1414
import { ClickhouseClient } from '@/clickhouse';
1515
import { getConnectionById } from '@/controllers/connection';
1616
import { getSource } from '@/controllers/sources';
17+
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
1718

1819
import { resolveBodyExpression } from './helpers';
1920

@@ -151,13 +152,21 @@ export async function denoiseSearchResults(
151152
config: sampleConfig,
152153
metadata,
153154
querySettings: source.querySettings,
154-
opts: { clickhouse_settings: { max_execution_time: 30 } },
155+
opts: {
156+
clickhouse_settings: {
157+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
158+
},
159+
},
155160
}),
156161
clickhouseClient.queryChartConfig({
157162
config: countConfig,
158163
metadata,
159164
querySettings: source.querySettings,
160-
opts: { clickhouse_settings: { max_execution_time: 30 } },
165+
opts: {
166+
clickhouse_settings: {
167+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
168+
},
169+
},
161170
}),
162171
]);
163172
} catch {

‎packages/api/src/mcp/tools/query/eventDeltas.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { getConnectionById } from '@/controllers/connection';
1616
import { getSource } from '@/controllers/sources';
1717
import type { ToolRegistrar } from '@/mcp/tools/types';
1818
import { mcpUserError } from '@/mcp/utils/errors';
19+
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
1920
import { trimToolResponse } from '@/utils/trimToolResponse';
2021

2122
import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
@@ -358,15 +359,19 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) {
358359
query_params: targetSql.params,
359360
format: 'JSON',
360361
connectionId: source.connection.toString(),
361-
clickhouse_settings: { max_execution_time: 30 },
362+
clickhouse_settings: {
363+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
364+
},
362365
abort_signal: abortController.signal,
363366
}),
364367
clickhouseClient.query({
365368
query: baselineSql.sql,
366369
query_params: baselineSql.params,
367370
format: 'JSON',
368371
connectionId: source.connection.toString(),
369-
clickhouse_settings: { max_execution_time: 30 },
372+
clickhouse_settings: {
373+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
374+
},
370375
abort_signal: abortController.signal,
371376
}),
372377
]);

‎packages/api/src/mcp/tools/query/helpers.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ import { getConnectionById } from '@/controllers/connection';
3030
import { getSource } from '@/controllers/sources';
3131
import type { McpErrorResult } from '@/mcp/utils/errors';
3232
import { mcpServerError, mcpUserError } from '@/mcp/utils/errors';
33+
import {
34+
MCP_QUERY_MAX_EXECUTION_SEC,
35+
MCP_TIMEOUT_GRACE_MS,
36+
MCP_TOOL_TIMEOUT_MS,
37+
} from '@/mcp/utils/timeout';
3338
import {
3439
convertToInternalTileConfig,
3540
isConfigTile,
@@ -77,7 +82,7 @@ export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/;
7782
/** ClickHouse settings applied to all MCP query-tool executions.
7883
* readonly=2 so max_execution_time can be set (readonly=1 rejects it). */
7984
export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = {
80-
max_execution_time: 30,
85+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
8186
readonly: '2',
8287
};
8388

@@ -86,7 +91,7 @@ export const MCP_CLICKHOUSE_SETTINGS: ClickHouseSettings = {
8691
* max_execution_time so ClickHouse returns a clean timeout before the HTTP
8792
* connection is aborted.
8893
*/
89-
export const MCP_REQUEST_TIMEOUT = 32_000; // 30s query limit + 2s buffer
94+
export const MCP_REQUEST_TIMEOUT = MCP_TOOL_TIMEOUT_MS + MCP_TIMEOUT_GRACE_MS;
9095

9196
// ─── Increase top-N cap hint ────────────────────────────────────────────────
9297

‎packages/api/src/mcp/tools/query/runEventPatterns.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
mcpServerError,
1919
mcpUserError,
2020
} from '@/mcp/utils/errors';
21+
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
2122
import { trimToolResponse } from '@/utils/trimToolResponse';
2223

2324
import {
@@ -186,7 +187,9 @@ export async function mineWindowPatterns(
186187
metadata,
187188
querySettings: source.querySettings,
188189
opts: {
189-
clickhouse_settings: { max_execution_time: 30 },
190+
clickhouse_settings: {
191+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
192+
},
190193
abort_signal: options?.abortSignal,
191194
},
192195
}),
@@ -195,7 +198,9 @@ export async function mineWindowPatterns(
195198
metadata,
196199
querySettings: source.querySettings,
197200
opts: {
198-
clickhouse_settings: { max_execution_time: 30 },
201+
clickhouse_settings: {
202+
max_execution_time: MCP_QUERY_MAX_EXECUTION_SEC,
203+
},
199204
abort_signal: options?.abortSignal,
200205
},
201206
}),

‎packages/api/src/mcp/tools/sources/describeMetric.ts‎

Lines changed: 18 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
mcpUserError,
2424
sanitizeFetchError,
2525
} from '@/mcp/utils/errors';
26+
import { MCP_TOOL_TIMEOUT_MS, runWithTimeout } from '@/mcp/utils/timeout';
2627
import logger from '@/utils/logger';
2728
import { trimToolResponse } from '@/utils/trimToolResponse';
2829

@@ -32,8 +33,7 @@ import {
3233
METRIC_DEFAULT_LOOKBACK_MS,
3334
} from './metricKinds';
3435

35-
// Matches the 30s cap the MCP query tools use.
36-
const DESCRIBE_TIMEOUT_MS = 30_000;
36+
const DESCRIBE_TIMEOUT_MS = MCP_TOOL_TIMEOUT_MS;
3737

3838
// Server-side safety nets for the attribute-keys discovery query.
3939
// Sample at most N rows that match (MetricName, time range), then
@@ -739,40 +739,23 @@ export function registerDescribeMetric({
739739
// optional-field types into `unknown`, but the parser produces
740740
// the typed shape we need for downstream calls.
741741
const input = describeMetricSchema.parse(rawInput);
742-
const controller = new AbortController();
743-
// Hoist the timer handle so the finally block can cancel it on the
744-
// success path — otherwise a stale controller.abort() fires
745-
// DESCRIBE_TIMEOUT_MS after every successful call and the
746-
// setTimeout closure stays pinned for the same duration.
747-
let timeoutId: ReturnType<typeof setTimeout> | undefined;
748-
const timeoutPromise = new Promise<never>((_, reject) => {
749-
timeoutId = setTimeout(() => {
750-
controller.abort();
751-
reject(new Error('DESCRIBE_METRIC_TIMEOUT'));
752-
}, DESCRIBE_TIMEOUT_MS);
753-
});
754-
try {
755-
return await Promise.race([
756-
describeMetricImpl(teamId.toString(), input, controller.signal),
757-
timeoutPromise,
758-
]);
759-
} catch (e) {
760-
if (e instanceof Error && e.message === 'DESCRIBE_METRIC_TIMEOUT') {
761-
logger.warn(
762-
{ teamId, sourceId: input.sourceId, metricName: input.metricName },
763-
'clickstack_describe_metric timed out',
764-
);
765-
return mcpServerError(
766-
'Discovery timed out. The metric table may be under load or the ' +
767-
'attribute set may be very high-cardinality. Try narrowing ' +
768-
'startTime/endTime or setting sampleValues:false to skip the ' +
769-
'value-sampling stage.',
770-
);
771-
}
772-
throw e;
773-
} finally {
774-
if (timeoutId !== undefined) clearTimeout(timeoutId);
742+
const outcome = await runWithTimeout(
743+
signal => describeMetricImpl(teamId.toString(), input, signal),
744+
{ timeoutMs: DESCRIBE_TIMEOUT_MS },
745+
);
746+
if (!outcome.timedOut) {
747+
return outcome.value;
775748
}
749+
logger.warn(
750+
{ teamId, sourceId: input.sourceId, metricName: input.metricName },
751+
'clickstack_describe_metric timed out',
752+
);
753+
return mcpServerError(
754+
'Discovery timed out. The metric table may be under load or the ' +
755+
'attribute set may be very high-cardinality. Try narrowing ' +
756+
'startTime/endTime or setting sampleValues:false to skip the ' +
757+
'value-sampling stage.',
758+
);
776759
},
777760
);
778761
}

0 commit comments

Comments
 (0)