Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/mcp-sql-table-names.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@hyperdx/api': patch
---

fix(mcp): stop `clickstack_sql` callers from guessing table and timestamp column
names. `clickstack_list_sources` now returns each source's database-qualified
`table` (and `database` for metric sources), the `sql` examples use `<table>`
and `<timestampColumn>` placeholders instead of `otel_logs` and `TimestampTime`,
and unknown-table errors from `clickstack_sql` point at `clickstack_list_sources`.
32 changes: 32 additions & 0 deletions packages/api/src/mcp/__tests__/query.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,38 @@ describe('errorHint', () => {
expect(hint).toBeNull();
});

const unknownTableErrors = [
"Unknown table expression identifier 'otel_logs' in scope SELECT count() FROM otel_logs",
'Table default.otel_traces does not exist.',
'Database o11y does not exist.',
];

it.each(unknownTableErrors)(
'points unknown-table errors in raw SQL at clickstack_list_sources: %s',
msg => {
expect(errorHint(msg, undefined, { rawSql: true })).toContain(
'clickstack_list_sources',
);
},
);

// Source-based tools take a sourceId, so the failing table is the source's
// own and listing sources would only hand it back.
it.each(unknownTableErrors)(
'does not point source-based tools at clickstack_list_sources: %s',
msg => {
expect(errorHint(msg) ?? '').not.toContain('clickstack_list_sources');
},
);

it('keeps pointing unknown-column errors at clickstack_describe_source', () => {
const hint = errorHint(
'Unknown expression or function identifier `TimestampTime` in scope SELECT TimestampTime FROM t',
);
expect(hint).toContain('clickstack_describe_source');
expect(hint).not.toContain('clickstack_list_sources');
});

it('should NOT match lowercase "as" in cast expressions', () => {
const hint = errorHint(
'Syntax error: cannot cast as Float64 near token...',
Expand Down
106 changes: 106 additions & 0 deletions packages/api/src/mcp/__tests__/queryTool.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,69 @@ describe('MCP Query Tools', () => {
},
);

describe('sql parameter examples', () => {
const getDescription = async () => {
const { tools } = await client.listTools();
const sqlParam = tools.find(t => t.name === 'clickstack_sql')!
.inputSchema.properties?.sql;
return sqlParam &&
typeof sqlParam === 'object' &&
'description' in sqlParam
? String(sqlParam.description)
: '';
};
const getExamples = async () =>
[
...(await getDescription()).matchAll(/^Example \(.+\):\n {2}(.+)$/gm),
].map(match => match[1]);

// timestampColumn can list several columns ("TimestampTime, Timestamp"),
// which breaks $__timeFilter if pasted whole. Metric sources return a
// database instead of a table.
it('say how to fill placeholders for multi-column timestamps and metric sources', async () => {
const description = await getDescription();

expect(description).toMatch(
/first expression in its `timestampColumn`/,
);
expect(description).toMatch(/metricTables.*`database`/);
});

// Agents copy examples literally. A concrete table or column in one
// becomes the guess, and it only exists on some schemas.
it('leave the table and timestamp column as placeholders', async () => {
const examples = await getExamples();

expect(examples).toHaveLength(2);
for (const sql of examples) {
expect(sql).toContain('FROM <table>');
expect(sql).toContain('<timestampColumn>');
expect(sql).not.toMatch(/otel_|TimestampTime/);
}
});

it('run once the placeholders are filled from a source', async () => {
const examples = await getExamples();

expect(examples).toHaveLength(2);
for (const example of examples) {
const sql = example
.replaceAll('<table>', `${DEFAULT_DATABASE}.${DEFAULT_LOGS_TABLE}`)
.replaceAll('<timestampColumn>', 'Timestamp');
const result = await callTool(client, 'clickstack_sql', {
connectionId: connection._id.toString(),
sql,
startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
endTime: new Date().toISOString(),
});
expect({
sql,
error: result.isError ? getFirstText(result) : null,
}).toEqual({ sql, error: null });
}
});
});

it('returns plain query rows unchanged', async () => {
const result = await callTool(client, 'clickstack_sql', {
connectionId: connection._id.toString(),
Expand Down Expand Up @@ -865,6 +928,24 @@ describe('MCP Query Tools', () => {
/UNKNOWN_TABLE|doesn.t exist|unknown/i,
);
});

it.each([
'nonexistent_table_xyz',
`${DEFAULT_DATABASE}.nonexistent_table_xyz`,
])(
'hints that %s should come from clickstack_list_sources',
async table => {
const result = await callTool(client, 'clickstack_sql', {
connectionId: connection._id.toString(),
sql: `SELECT 1 FROM ${table} LIMIT 1`,
});

expect(result.isError).toBe(true);
expect(getFirstText(result)).toContain(
'HINT: Use the `table` of a source from clickstack_list_sources',
);
},
);
});

describe('clickstack_table errors', () => {
Expand All @@ -879,6 +960,31 @@ describe('MCP Query Tools', () => {
/UNKNOWN_IDENTIFIER|Missing columns|unknown/i,
);
});

it('does not send a source whose table is missing back to clickstack_list_sources', async () => {
const missing = await Source.create({
kind: SourceKind.Log,
team: team._id,
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'nonexistent_table_xyz',
},
timestampValueExpression: 'Timestamp',
connection: connection._id,
name: 'Missing logs',
});

const result = await callTool(client, 'clickstack_table', {
sourceId: missing._id.toString(),
select: [{ aggFn: 'count' }],
});

expect(result.isError).toBe(true);
expect(getFirstText(result)).toMatch(
/UNKNOWN_TABLE|does(n.t| not) exist/i,
);
expect(getFirstText(result)).not.toContain('clickstack_list_sources');
});
});

describe('clickstack_timeseries errors', () => {
Expand Down
75 changes: 75 additions & 0 deletions packages/api/src/mcp/__tests__/sources.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,40 @@ describe('MCP Source Tools', () => {
expect(output.nextStep).toContain('clickstack_describe_source');
});

// clickstack_sql callers need this for FROM. Without it they guess
// otel_logs, which only resolves when the tables sit in the
// connection's default database.
it('includes the database-qualified table each log and trace source reads', async () => {
const result = await callTool(client, 'clickstack_list_sources');
const output = JSON.parse(getFirstText(result));

const trace = output.sources.find(
(s: any) => s.kind === SourceKind.Trace,
);
const log = output.sources.find((s: any) => s.kind === SourceKind.Log);
expect(trace.table).toBe(`${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE}`);
expect(log.table).toBe(`${DEFAULT_DATABASE}.${DEFAULT_LOGS_TABLE}`);
});

it('quotes table parts that are not bare identifiers so FROM still parses', async () => {
const source = await Source.create({
kind: SourceKind.Log,
team: team._id,
from: { databaseName: 'observability-prod', tableName: 'otel.logs' },
timestampValueExpression: 'Timestamp',
connection: connection._id,
name: 'Quoted logs',
});

const result = await callTool(client, 'clickstack_list_sources');
const output = JSON.parse(getFirstText(result));

const log = output.sources.find(
(s: any) => s.id === source._id.toString(),
);
expect(log.table).toBe('`observability-prod`.`otel.logs`');
});

it('should include keyColumns from source config without ClickHouse queries', async () => {
const result = await callTool(client, 'clickstack_list_sources');
const output = JSON.parse(getFirstText(result));
Expand Down Expand Up @@ -248,6 +282,47 @@ describe('MCP Source Tools', () => {
expect(output.metricsUsage).toContain('metricType + metricName');
});

it('includes the database its metricTables live in', async () => {
const metricSource = await createMetricSource();

const result = await callTool(client, 'clickstack_list_sources');
const output = JSON.parse(getFirstText(result));

const metric = output.sources.find(
(s: any) => s.id === metricSource._id.toString(),
);
expect(metric.database).toBe(DEFAULT_DATABASE);
expect(metric.table).toBeUndefined();
});

// Metric sources may also set from.tableName, but their queryable
// tables are still the ones in metricTables.
it('includes the database for a metric source that also names a table', async () => {
const metricSource = await Source.create({
kind: SourceKind.Metric,
team: team._id,
from: {
databaseName: DEFAULT_DATABASE,
tableName: 'otel_metrics_gauge',
},
metricTables: {
[MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge',
},
timestampValueExpression: 'TimeUnix',
connection: connection._id,
name: 'Named metrics',
});

const result = await callTool(client, 'clickstack_list_sources');
const output = JSON.parse(getFirstText(result));

const metric = output.sources.find(
(s: any) => s.id === metricSource._id.toString(),
);
expect(metric.database).toBe(DEFAULT_DATABASE);
expect(metric.table).toBeUndefined();
});

it('falls back to a wider lookback when no metrics reported in the last 24h', async () => {
const metricSource = await createMetricSource('Sparse Metrics');
const threeDaysAgo = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000);
Expand Down
32 changes: 26 additions & 6 deletions packages/api/src/mcp/tools/query/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -684,7 +684,7 @@ export async function runConfigTile(
});
return formatQueryResult(result);
} catch (e) {
return clickHouseErrorResult(e);
return clickHouseErrorResult(e, { rawSql: true });

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — The rawSql hint reaches dashboard raw-SQL tiles too, not just clickstack_sql

runConfigTile's raw-SQL branch is also called by clickstack_query_tile and clickstack_query_tiles (tools/dashboards/queryTile.ts:85, queryTiles.ts:303). For a tile using $__sourceTable, the failing table is the source's own table, so pointing the agent at clickstack_list_sources just returns that same table. That is the case this PR deliberately excludes for builder tools. Fix: pass rawSql from sql.ts via a runConfigTile option, or set it only when savedConfig.source is unset.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

}
}

Expand Down Expand Up @@ -841,11 +841,11 @@ function hasNodeErrorCode(val: unknown, codes: ReadonlySet<string>): boolean {
*/
export function clickHouseErrorResult(
e: unknown,
context?: string | { prefix?: string; suffix?: string },
context?: string | { prefix?: string; suffix?: string; rawSql?: boolean },
): McpErrorResult {
const { prefix, suffix } =
const { prefix, suffix, rawSql } =
typeof context === 'string'
? { prefix: context, suffix: undefined }
? { prefix: context, suffix: undefined, rawSql: undefined }
: (context ?? {});

// Prefer .message, but fall back to .cause.message when the wrapper
Expand All @@ -857,7 +857,7 @@ export function clickHouseErrorResult(
(e.cause instanceof Error ? e.cause.message : '') ||
String(e)
: String(e);
const hint = errorHint(raw, e);
const hint = errorHint(raw, e, { rawSql });
const base = hint ? `${raw}\n\nHINT: ${hint}` : raw;
const text = `${prefix ? `${prefix}: ` : ''}${base}${suffix ? ` ${suffix}` : ''}`;

Expand Down Expand Up @@ -897,7 +897,11 @@ export function isQueryOutOfTime(e: unknown): boolean {
}

/** @internal Exported for testing only. */
export function errorHint(msg: string, error?: unknown): string | null {
export function errorHint(
msg: string,
error?: unknown,
options?: { rawSql?: boolean },
): string | null {
const unknownVariableError = findCause(
error,
(e): e is UnknownVariableError => e instanceof UnknownVariableError,
Expand Down Expand Up @@ -954,6 +958,22 @@ export function errorHint(msg: string, error?: unknown): string | null {
'Add a LIMIT, narrow the time range, or add filters to reduce the result set.'
);
}
// Checked before unknown identifiers: "Unknown table expression identifier"
// names a table, so describe_source's column list would not help. Only raw
// SQL names its own table; source-based tools would be sent back to the
// same source table that just failed.
if (

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — New unknown-table hint is SQL-specific but shown for every MCP tool's ClickHouse errors

errorHint runs for every tool that calls clickHouseErrorResult, not just clickstack_sql. That includes runQuery/runConfigQuery in helpers.ts, runEventPatterns.ts, eventDeltas.ts, trace/waterfall.ts, trace/breakdown.ts and describeMetric.ts. When a builder tool's source points at a dropped or renamed table, the agent is told to "use the table of a source from clickstack_list_sources", which is the same broken table and doesn't help, since builder tools take a sourceId. Fix: attach this hint only from sql.ts (for example via the existing suffix context, or a flag passed to errorHint). For source-backed tools, give a hint that says the source's configured table is missing.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in cc07b20: errorHint only adds the unknown-table hint when called from the raw SQL path (clickHouseErrorResult(e, { rawSql: true })). Source-based tools get no hint for a missing source table, as on main. New unit tests and a clickstack_table integration test cover it.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — The unknown-table hint also fires for source-based builder tools, where the agent never wrote a table name

errorHint is reached through clickHouseErrorResult, which is shared by every tool: the builder path at helpers.ts:617 (clickstack_timeseries/table/search), trace/waterfall.ts, trace/breakdown.ts, eventDeltas.ts, runEventPatterns.ts and describeMetric.ts. All of these get the table from the source's from. If that table has been dropped or renamed, the agent is told to "use the table of a source from clickstack_list_sources". That table is the one that just failed, and these tools take no table argument anyway. Fix: attach this hint only on the clickstack_sql path (for example, pass it as a suffix from sql.ts), or reword it for the builder tools so it says the source's configured table is missing.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in cc07b20: errorHint only adds the unknown-table hint when called from the raw SQL path (clickHouseErrorResult(e, { rawSql: true })). Source-based tools get no hint for a missing source table, as on main. New unit tests and a clickstack_table integration test cover it.

options?.rawSql &&
/Unknown table expression identifier|(Table|Database) \S+ does not exist|UNKNOWN_(TABLE|DATABASE)/.test(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — Unknown-table regex misses the "doesn't exist" wording

Older ClickHouse servers report Table db.t doesn't exist.. @clickhouse/client puts the UNKNOWN_TABLE code in error.type and leaves it out of .message, so neither branch of the regex matches on those servers. The existing int test (queryTool.int.test.ts ~line 928) already allows /doesn.t exist/ for this reason. Fix: use does(n't| not) exist, and/or check e.type for UNKNOWN_TABLE/UNKNOWN_DATABASE instead of the message.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

msg,
)
) {
return (
'Use the `table` of a source from clickstack_list_sources, which is ' +
'qualified with its database. Unqualified names resolve only in the ' +
"connection's default database."
);
}
if (
/Unknown (expression|identifier)|UNKNOWN_IDENTIFIER|Missing columns/i.test(
msg,
Expand Down
15 changes: 10 additions & 5 deletions packages/api/src/mcp/tools/query/sql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,18 @@ const sqlSchema = z.object({
' $__interval_s — raw interval in seconds\n' +
' $__filters — dashboard filter conditions (resolves to 1=1 when no filters)\n' +
' {intervalSeconds:Int64} / {intervalMilliseconds:Int64} — bucket size parameters\n\n' +
"In the examples, replace <table> with a source's `table` and <timestampColumn> " +
'with the first expression in its `timestampColumn`, both from ' +
'clickstack_list_sources. For a metric source, use a table from its ' +
'metricTables qualified with its `database`. Table and column names differ ' +
'between sources, so do not guess them.\n\n' +
'Example (time-series):\n' +
' SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' +
'FROM otel_logs WHERE $__timeFilter(TimestampTime) GROUP BY ServiceName, ts ORDER BY ts\n\n' +
' SELECT $__timeInterval(<timestampColumn>) AS ts, ServiceName, count() ' +
'FROM <table> WHERE $__timeFilter(<timestampColumn>) GROUP BY ServiceName, ts ORDER BY ts\n\n' +
'Example (table):\n' +
' SELECT ServiceName, count() AS n FROM otel_logs ' +
'WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) ' +
'AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) ' +
' SELECT ServiceName, count() AS n FROM <table> ' +
'WHERE <timestampColumn> >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) ' +
'AND <timestampColumn> < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) ' +
'GROUP BY ServiceName ORDER BY n DESC LIMIT 20',
),
startTime: startTimeSchema,
Expand Down
11 changes: 11 additions & 0 deletions packages/api/src/mcp/tools/sources/listSources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
import {
Metadata,
MetadataCache,
quoteIdentifierIfNeeded,
} from '@hyperdx/common-utils/dist/core/metadata';
import { SourceKind } from '@hyperdx/common-utils/dist/types';
import { z } from 'zod';
Expand Down Expand Up @@ -244,6 +245,16 @@ export function registerListSources({
timestampColumn: s.timestampValueExpression,
};

// For clickstack_sql's FROM, so each part is quoted when it needs to
// be. An unqualified name only resolves in the connection's default
// database. Metric sources query the tables in metricTables, even
// when from.tableName is set, so they get the database instead.
if (s.kind !== SourceKind.Metric && s.from.tableName) {
meta.table = `${quoteIdentifierIfNeeded(s.from.databaseName)}.${quoteIdentifierIfNeeded(s.from.tableName)}`;
} else {
meta.database = quoteIdentifierIfNeeded(s.from.databaseName);
}

if (s.section) {
meta.section = s.section;
}
Expand Down
Loading