Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/sql-query-dashboard-filter-api.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/api': patch
---

feat: accept SQL query dashboard filters in the API, external API, and MCP

Saving a dashboard now rejects a `SQL_QUERY` filter whose connection doesn't belong to the team. The external API and MCP read and write the filter with a `connectionId`, and MCP warns about unknown variables and source-only macros in its SQL.
92 changes: 91 additions & 1 deletion packages/api/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -3951,14 +3951,18 @@
},
{
"$ref": "#/components/schemas/PrometheusLabelFilterInput"
},
{
"$ref": "#/components/schemas/SqlQueryFilterInput"
}
],
"discriminator": {
"propertyName": "type",
"mapping": {
"QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput",
"STATIC_LIST": "#/components/schemas/StaticListFilterInput",
"PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput"
"PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput",
"SQL_QUERY": "#/components/schemas/SqlQueryFilterInput"
}
}
},
Expand Down Expand Up @@ -4262,6 +4266,92 @@
}
}
},
"SqlQueryFilterInput": {
"type": "object",
"description": "A filter whose dropdown lists the first column returned by a ClickHouse\nquery, cast to strings with nulls and duplicates removed. Supports\nvariable mode only.\n",
"required": [
"type",
"name",
"connectionId",
"sqlTemplate"
],
"properties": {
"type": {
"type": "string",
"enum": [
"SQL_QUERY"
],
"description": "Discriminator. Must be \"SQL_QUERY\".",
"example": "SQL_QUERY"
},
"name": {
"type": "string",
"minLength": 1,
"description": "Display name for the dashboard filter",
"example": "Environment"
},
"connectionId": {
"type": "string",
"description": "ID of the ClickHouse connection the query runs on.",
"example": "65f5e4a3b9e77c001a567890"
},
"sqlTemplate": {
"type": "string",
"minLength": 1,
"description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables, but not `$__sourceTable` or\n`$__filters`, since the filter has no source.\n",
"example": "SELECT DISTINCT env FROM default.envs ORDER BY env"
},
"isBroadcastEnabled": {
"type": "boolean",
"enum": [
false
],
"default": false,
"description": "Must be false or omitted.",
"example": false
},
"isVariableEnabled": {
"type": "boolean",
"enum": [
true
],
"default": true,
"description": "Must be true if provided, and is true when omitted. Tiles reference\nthe selection as `$variableName`.\n",
"example": true
},
"variableName": {
"type": "string",
"maxLength": 64,
"pattern": "^[a-zA-Z][a-zA-Z0-9_]*$",
"description": "Token that tiles reference this filter's selected value by, as\n`$variableName`. Defaults to the display name with whitespace\nreplaced by underscores and remaining illegal characters removed.\nMust be unique across a dashboard's variable-enabled filters.\n",
"example": "env"
},
"minSelections": {
"type": "integer",
"enum": [
0,
1
],
"default": 0,
"description": "Set to 1 to make the filter required.",
"example": 1
},
"isGlobalRequirement": {
"type": "boolean",
"default": false,
"description": "Widens a required filter's block to every tile on the dashboard.\nIgnored unless minSelections is 1.\n",
"example": true
},
"maxSelections": {
"type": "integer",
"enum": [
1
],
"description": "Set to 1 to make the filter single-select.",
"example": 1
}
}
},
"Filter": {
"allOf": [
{
Expand Down
62 changes: 61 additions & 1 deletion packages/api/src/mcp/__tests__/dashboards/validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -594,9 +594,22 @@ describe('mcpFiltersParam variable fields', () => {
options: ['prod', 'staging'],
variableName,
}),
SQL_QUERY: (variableName: string) => ({
type: 'SQL_QUERY',
name: 'Environment',
connectionId: sourceId,
sqlTemplate: 'SELECT DISTINCT env FROM envs',
variableName,
}),
};

describe.each(['QUERY_EXPRESSION', 'STATIC_LIST'] as const)('%s', type => {
const variableFilterTypes = [
'QUERY_EXPRESSION',
'STATIC_LIST',
'SQL_QUERY',
] as const;

describe.each(variableFilterTypes)('%s', type => {
it.each([
['a space', 'has space'],
['a leading digit', '1service'],
Expand Down Expand Up @@ -921,4 +934,51 @@ describe('getFilterVariableWarnings', () => {
expect(joined).toContain("unknown variable 'tenant'");
expect(joined).toContain('Available variables: service');
});

describe('SQL query filters', () => {
const sqlFilter = (sqlTemplate: string) => ({
name: 'Environment',
sqlTemplate,
isVariableEnabled: true,
variableName: 'env',
});

it('accepts a query guarded by $__filter', () => {
expect(
getFilterVariableWarnings([
serviceFilter,
sqlFilter(
'SELECT env FROM envs WHERE $__filter(service, $service) AND $__timeFilter(ts)',
),
]),
).toEqual([]);
});

it('checks the query as SQL', () => {
const warnings = getFilterVariableWarnings([
serviceFilter,
sqlFilter('SELECT env FROM envs WHERE $__filter(service, $tenant)'),
]);
expect(warnings).toHaveLength(1);
expect(warnings[0]).toContain("unknown variable 'tenant'");
});

it('warns about a macro that needs a source', () => {
const warnings = getFilterVariableWarnings([
sqlFilter('SELECT env FROM $__sourceTable'),
]);
expect(warnings).toEqual([
'Filter "Environment": SQL query filters have no source, so $__sourceTable cannot be used',
]);
});

it.each([
['a source macro', 'SELECT env FROM $__sourceTable(gauge'],
['a time macro', 'SELECT env FROM envs WHERE $__timeFilter(ts'],
])('warns about an unclosed %s instead of throwing', (_label, sql) => {
expect(getFilterVariableWarnings([sqlFilter(sql)])).toEqual([
'Filter "Environment": Failed to parse macro arguments',
]);
});
});
});
17 changes: 15 additions & 2 deletions packages/api/src/mcp/prompts/dashboards/content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie
- Using $__filter or $__conditionalAll in a Lucene where (they are matched as literal text; switch to whereLanguage: "sql", or write ServiceName:$var).
- Chaining a filter's where off a filter that is not variable-enabled. Only isVariableEnabled: true publishes the $variableName token; without it the reference resolves to nothing and the downstream dropdown is unscoped or empty.
- Referencing a filter's OWN variable in its where. That narrows its dropdown to the values already picked, so the rest of the options vanish from a multi-select. Reference a sibling filter's variable instead.
- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters.
- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate to SQL_QUERY filters.
- Expecting a STATIC_LIST filter to broadcast. It cannot; a tile must reference its $variableName (via $__filter with the column passed explicitly) or nothing changes when the user picks a value.`;
}

Expand Down Expand Up @@ -1168,7 +1168,7 @@ Rules:

Optional dashboard-level filter declarations. Each entry adds a dropdown to the dashboard header. Use this for focused per-dimension dashboards (per-service, per-tenant, per-endpoint) instead of hardcoding the dimension into every tile's where clause.

Two filter types: QUERY_EXPRESSION queries its dropdown values from a source column, while STATIC_LIST declares them inline as a hand-authored options list and is always variable-only (see the static filter shape below).
Filter types: QUERY_EXPRESSION queries its dropdown values from a source column. STATIC_LIST declares them inline as a hand-authored options list, and SQL_QUERY reads them from a ClickHouse query; both are always variable-only (see their shapes below).

A QUERY_EXPRESSION filter does one of two things with the value the user picks, or both:

Expand Down Expand Up @@ -1207,6 +1207,19 @@ Example (static list):
{ type: "STATIC_LIST", name: "Environment", options: ["prod", "staging", "dev"], variableName: "env" }
]

SQL query filter shape:
{ type: "SQL_QUERY", name, connectionId, sqlTemplate, variableName? }

connectionId ClickHouse connection the query runs on (from clickstack_list_sources).
sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables; cannot use $__sourceTable or $__filters, because the filter has no source.

Use it when the values live in a lookup table, dictionary, or join that no single source column covers; otherwise prefer QUERY_EXPRESSION. Like a static filter it is variable-only, so tiles pass the column explicitly: $__filter(<expression>, $<variableName>).

Example (SQL query, narrowed by another filter's variable):
filters: [
{ type: "SQL_QUERY", name: "Environment", connectionId: "<connection-id>", sqlTemplate: "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env", variableName: "env" }
]

Example (broadcast to every tile, the common case):
filters: [
{ type: "QUERY_EXPRESSION", name: "Service", expression: "ServiceName", sourceId: "<trace-source-id>" }
Expand Down
57 changes: 48 additions & 9 deletions packages/api/src/mcp/tools/dashboards/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
PromqlReducerSchema,
SearchConditionTrimmedLanguageSchema,
SQLIntervalSchema,
SqlQueryDashboardFilterSchema,
StaticListDashboardFilterSchema,
} from '@hyperdx/common-utils/dist/types';
import { z } from 'zod';
Expand Down Expand Up @@ -1765,6 +1766,33 @@ const mcpStaticListFilterSchema = z
'with the column passed explicitly, since the filter has no expression of its own.',
);

const mcpSqlQueryFilterSchema = z
.object({
...mcpDashboardFilterBaseShape,
type: z
.literal(DashboardFilterType.enum.SQL_QUERY)
.describe(
'Filter type discriminator for a filter whose dropdown values come from a ClickHouse SQL query.',
),
connectionId: objectIdSchema.describe(
'ClickHouse connection the query runs on. Get IDs from clickstack_list_sources.',
),
sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe(
'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' +
'string; nulls and duplicates dropped; row order kept, so add ORDER BY). ' +
'May use the time macros ($__timeFilter(<column>), $__fromTime, $__toTime) ' +
"and other filters' variables, ideally as $__filter(<expression>, $<variableName>). " +
'Cannot use $__sourceTable or $__filters: the filter has no source, so name ' +
'tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".',
),
})
.describe(
'A variable-only filter whose dropdown lists the first column of a ClickHouse ' +
'query. Use it for values in lookup tables, dictionaries, or joins that no ' +
'single source column covers. Tiles reference the selection as ' +
'$__filter(<expression>, $<variableName>) with the column passed explicitly.',
);

const mcpPromqlLabelFilterSchema = z
.object({
...mcpDashboardFilterBaseShape,
Expand Down Expand Up @@ -1795,25 +1823,28 @@ const mcpDashboardFilterSchema = z
.discriminatedUnion('type', [
mcpQueryExpressionFilterSchema,
mcpStaticListFilterSchema,
mcpSqlQueryFilterSchema,
])
.describe(
'A dashboard-level filter the user can adjust in the dashboard filter bar. ' +
'Two types: QUERY_EXPRESSION queries its dropdown values from a source column; ' +
'STATIC_LIST declares them inline and is always variable-only.',
'QUERY_EXPRESSION queries its dropdown values from a source column; ' +
'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query. ' +
'STATIC_LIST and SQL_QUERY are always variable-only.',
);

const mcpPromqlDashboardFilterSchema = z
.discriminatedUnion('type', [
mcpQueryExpressionFilterSchema,
mcpStaticListFilterSchema,
mcpSqlQueryFilterSchema,
mcpPromqlLabelFilterSchema,
])
.describe(
'A dashboard-level filter the user can adjust in the dashboard filter bar. ' +
'QUERY_EXPRESSION queries its dropdown values from a source column; ' +
'STATIC_LIST declares them inline and is always variable-only; ' +
'PROMETHEUS_LABEL lists the values of a label on a promql source and is ' +
'always variable-only.',
'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query; ' +
'PROMETHEUS_LABEL lists the values of a label on a promql source. All but ' +
'QUERY_EXPRESSION are always variable-only.',
);

export type McpDashboardFilter = z.infer<typeof mcpPromqlDashboardFilterSchema>;
Expand All @@ -1826,9 +1857,10 @@ const mcpFiltersParam = z
'If another tile\'s onClick targets THIS dashboard with `filters: [{ expression: "X", ... }]`, ' +
'this array MUST declare a filter whose `expression` is "X". Otherwise the value is ' +
'dropped on arrival and the destination opens unfiltered.\n\n' +
'Two filter types: QUERY_EXPRESSION queries its dropdown values from a source column, ' +
'while STATIC_LIST declares them inline via `options` and is always variable-only ' +
'(it has no expression to broadcast).\n\n' +
'Filter types: QUERY_EXPRESSION queries its dropdown values from a source column. ' +
'STATIC_LIST declares them inline via `options`, and SQL_QUERY reads them from the ' +
'first column of a ClickHouse query; both are always variable-only (they have no ' +
'expression to broadcast).\n\n' +
'A QUERY_EXPRESSION filter runs in one of two modes (or both). BROADCAST (the default) applies the ' +
'selected value as a condition on matching tiles with no per-tile wiring. VARIABLE ' +
'(isVariableEnabled) exposes the value to tile queries as $variableName, which the tile ' +
Expand Down Expand Up @@ -1867,6 +1899,13 @@ const mcpFiltersParam = z
'Always pass the column explicitly: the one-argument $__filter($env) form reuses the ' +
"filter's own expression, which a static filter does not have. Prefer STATIC_LIST when " +
'the values are a fixed business list or a curated subset rather than derivable from the data.\n\n' +
'Example (SQL query: values from a lookup table no source covers, narrowed by another filter):\n' +
'[\n' +
' { "type": "SQL_QUERY", "name": "Environment", "connectionId": "<connection-id>",\n' +
' "sqlTemplate": "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env",\n' +
' "variableName": "env" }\n' +
']\n' +
'Prefer QUERY_EXPRESSION when the values are a column of a registered source.\n\n' +
"Example (dependent filter: the Endpoint dropdown only lists the selected service's " +
'endpoints):\n' +
'[\n' +
Expand All @@ -1883,7 +1922,7 @@ const mcpFiltersParam = z
);

const PROMQL_FILTERS_EXAMPLE =
'\n\nA third type, PROMETHEUS_LABEL, lists the values of a label on a promql source ' +
'\n\nAnother type, PROMETHEUS_LABEL, lists the values of a label on a promql source ' +
'and is always variable-only.\n\n' +
'Example (PROMETHEUS_LABEL):\n' +
'[\n' +
Expand Down
Loading
Loading