diff --git a/.changeset/sql-query-dashboard-filter-api.md b/.changeset/sql-query-dashboard-filter-api.md new file mode 100644 index 0000000000..62fa1a7278 --- /dev/null +++ b/.changeset/sql-query-dashboard-filter-api.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +feat: accept SQL query dashboard filters in the API, external API, and MCP + +Saving a dashboard now rejects a `SQL_QUERY` filter whose connection doesn't belong to the team. The external API and MCP read and write the filter with a `connectionId`, and MCP warns about unknown variables and source-only macros in its SQL. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 8e883b9827..604e11c631 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -3951,6 +3951,9 @@ }, { "$ref": "#/components/schemas/PrometheusLabelFilterInput" + }, + { + "$ref": "#/components/schemas/SqlQueryFilterInput" } ], "discriminator": { @@ -3958,7 +3961,8 @@ "mapping": { "QUERY_EXPRESSION": "#/components/schemas/QueryExpressionFilterInput", "STATIC_LIST": "#/components/schemas/StaticListFilterInput", - "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput" + "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", + "SQL_QUERY": "#/components/schemas/SqlQueryFilterInput" } } }, @@ -4262,6 +4266,92 @@ } } }, + "SqlQueryFilterInput": { + "type": "object", + "description": "A filter whose dropdown lists the first column returned by a ClickHouse\nquery, cast to strings with nulls and duplicates removed. Supports\nvariable mode only.\n", + "required": [ + "type", + "name", + "connectionId", + "sqlTemplate" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "SQL_QUERY" + ], + "description": "Discriminator. Must be \"SQL_QUERY\".", + "example": "SQL_QUERY" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for the dashboard filter", + "example": "Environment" + }, + "connectionId": { + "type": "string", + "description": "ID of the ClickHouse connection the query runs on.", + "example": "65f5e4a3b9e77c001a567890" + }, + "sqlTemplate": { + "type": "string", + "minLength": 1, + "description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables, but not `$__sourceTable` or\n`$__filters`, since the filter has no source.\n", + "example": "SELECT DISTINCT env FROM default.envs ORDER BY env" + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + false + ], + "default": false, + "description": "Must be false or omitted.", + "example": false + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true if provided, and is true when omitted. Tiles reference\nthe selection as `$variableName`.\n", + "example": true + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token that tiles reference this filter's selected value by, as\n`$variableName`. Defaults to the display name with whitespace\nreplaced by underscores and remaining illegal characters removed.\nMust be unique across a dashboard's variable-enabled filters.\n", + "example": "env" + }, + "minSelections": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "default": 0, + "description": "Set to 1 to make the filter required.", + "example": 1 + }, + "isGlobalRequirement": { + "type": "boolean", + "default": false, + "description": "Widens a required filter's block to every tile on the dashboard.\nIgnored unless minSelections is 1.\n", + "example": true + }, + "maxSelections": { + "type": "integer", + "enum": [ + 1 + ], + "description": "Set to 1 to make the filter single-select.", + "example": 1 + } + } + }, "Filter": { "allOf": [ { diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index 260e5fd650..bd6ffe6a63 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -594,9 +594,22 @@ describe('mcpFiltersParam variable fields', () => { options: ['prod', 'staging'], variableName, }), + SQL_QUERY: (variableName: string) => ({ + type: 'SQL_QUERY', + name: 'Environment', + connectionId: sourceId, + sqlTemplate: 'SELECT DISTINCT env FROM envs', + variableName, + }), }; - describe.each(['QUERY_EXPRESSION', 'STATIC_LIST'] as const)('%s', type => { + const variableFilterTypes = [ + 'QUERY_EXPRESSION', + 'STATIC_LIST', + 'SQL_QUERY', + ] as const; + + describe.each(variableFilterTypes)('%s', type => { it.each([ ['a space', 'has space'], ['a leading digit', '1service'], @@ -921,4 +934,51 @@ describe('getFilterVariableWarnings', () => { expect(joined).toContain("unknown variable 'tenant'"); expect(joined).toContain('Available variables: service'); }); + + describe('SQL query filters', () => { + const sqlFilter = (sqlTemplate: string) => ({ + name: 'Environment', + sqlTemplate, + isVariableEnabled: true, + variableName: 'env', + }); + + it('accepts a query guarded by $__filter', () => { + expect( + getFilterVariableWarnings([ + serviceFilter, + sqlFilter( + 'SELECT env FROM envs WHERE $__filter(service, $service) AND $__timeFilter(ts)', + ), + ]), + ).toEqual([]); + }); + + it('checks the query as SQL', () => { + const warnings = getFilterVariableWarnings([ + serviceFilter, + sqlFilter('SELECT env FROM envs WHERE $__filter(service, $tenant)'), + ]); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("unknown variable 'tenant'"); + }); + + it('warns about a macro that needs a source', () => { + const warnings = getFilterVariableWarnings([ + sqlFilter('SELECT env FROM $__sourceTable'), + ]); + expect(warnings).toEqual([ + 'Filter "Environment": SQL query filters have no source, so $__sourceTable cannot be used', + ]); + }); + + it.each([ + ['a source macro', 'SELECT env FROM $__sourceTable(gauge'], + ['a time macro', 'SELECT env FROM envs WHERE $__timeFilter(ts'], + ])('warns about an unclosed %s instead of throwing', (_label, sql) => { + expect(getFilterVariableWarnings([sqlFilter(sql)])).toEqual([ + 'Filter "Environment": Failed to parse macro arguments', + ]); + }); + }); }); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 84c8199bb3..74e2d4e030 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -196,7 +196,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Using $__filter or $__conditionalAll in a Lucene where (they are matched as literal text; switch to whereLanguage: "sql", or write ServiceName:$var). - Chaining a filter's where off a filter that is not variable-enabled. Only isVariableEnabled: true publishes the $variableName token; without it the reference resolves to nothing and the downstream dropdown is unscoped or empty. - Referencing a filter's OWN variable in its where. That narrows its dropdown to the values already picked, so the rest of the options vanish from a multi-select. Reference a sibling filter's variable instead. -- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters. +- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate to SQL_QUERY filters. - Expecting a STATIC_LIST filter to broadcast. It cannot; a tile must reference its $variableName (via $__filter with the column passed explicitly) or nothing changes when the user picks a value.`; } @@ -1168,7 +1168,7 @@ Rules: Optional dashboard-level filter declarations. Each entry adds a dropdown to the dashboard header. Use this for focused per-dimension dashboards (per-service, per-tenant, per-endpoint) instead of hardcoding the dimension into every tile's where clause. -Two filter types: QUERY_EXPRESSION queries its dropdown values from a source column, while STATIC_LIST declares them inline as a hand-authored options list and is always variable-only (see the static filter shape below). +Filter types: QUERY_EXPRESSION queries its dropdown values from a source column. STATIC_LIST declares them inline as a hand-authored options list, and SQL_QUERY reads them from a ClickHouse query; both are always variable-only (see their shapes below). A QUERY_EXPRESSION filter does one of two things with the value the user picks, or both: @@ -1207,6 +1207,19 @@ Example (static list): { type: "STATIC_LIST", name: "Environment", options: ["prod", "staging", "dev"], variableName: "env" } ] +SQL query filter shape: + { type: "SQL_QUERY", name, connectionId, sqlTemplate, variableName? } + + connectionId ClickHouse connection the query runs on (from clickstack_list_sources). + sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables; cannot use $__sourceTable or $__filters, because the filter has no source. + + Use it when the values live in a lookup table, dictionary, or join that no single source column covers; otherwise prefer QUERY_EXPRESSION. Like a static filter it is variable-only, so tiles pass the column explicitly: $__filter(, $). + +Example (SQL query, narrowed by another filter's variable): + filters: [ + { type: "SQL_QUERY", name: "Environment", connectionId: "", sqlTemplate: "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env", variableName: "env" } + ] + Example (broadcast to every tile, the common case): filters: [ { type: "QUERY_EXPRESSION", name: "Service", expression: "ServiceName", sourceId: "" } diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index f9e40efe58..a195efa154 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -24,6 +24,7 @@ import { PromqlReducerSchema, SearchConditionTrimmedLanguageSchema, SQLIntervalSchema, + SqlQueryDashboardFilterSchema, StaticListDashboardFilterSchema, } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; @@ -1765,6 +1766,33 @@ const mcpStaticListFilterSchema = z 'with the column passed explicitly, since the filter has no expression of its own.', ); +const mcpSqlQueryFilterSchema = z + .object({ + ...mcpDashboardFilterBaseShape, + type: z + .literal(DashboardFilterType.enum.SQL_QUERY) + .describe( + 'Filter type discriminator for a filter whose dropdown values come from a ClickHouse SQL query.', + ), + connectionId: objectIdSchema.describe( + 'ClickHouse connection the query runs on. Get IDs from clickstack_list_sources.', + ), + sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( + 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + + 'string; nulls and duplicates dropped; row order kept, so add ORDER BY). ' + + 'May use the time macros ($__timeFilter(), $__fromTime, $__toTime) ' + + "and other filters' variables, ideally as $__filter(, $). " + + 'Cannot use $__sourceTable or $__filters: the filter has no source, so name ' + + 'tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".', + ), + }) + .describe( + 'A variable-only filter whose dropdown lists the first column of a ClickHouse ' + + 'query. Use it for values in lookup tables, dictionaries, or joins that no ' + + 'single source column covers. Tiles reference the selection as ' + + '$__filter(, $) with the column passed explicitly.', + ); + const mcpPromqlLabelFilterSchema = z .object({ ...mcpDashboardFilterBaseShape, @@ -1795,25 +1823,28 @@ const mcpDashboardFilterSchema = z .discriminatedUnion('type', [ mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, + mcpSqlQueryFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + - 'Two types: QUERY_EXPRESSION queries its dropdown values from a source column; ' + - 'STATIC_LIST declares them inline and is always variable-only.', + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + + 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query. ' + + 'STATIC_LIST and SQL_QUERY are always variable-only.', ); const mcpPromqlDashboardFilterSchema = z .discriminatedUnion('type', [ mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, + mcpSqlQueryFilterSchema, mcpPromqlLabelFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + - 'STATIC_LIST declares them inline and is always variable-only; ' + - 'PROMETHEUS_LABEL lists the values of a label on a promql source and is ' + - 'always variable-only.', + 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query; ' + + 'PROMETHEUS_LABEL lists the values of a label on a promql source. All but ' + + 'QUERY_EXPRESSION are always variable-only.', ); export type McpDashboardFilter = z.infer; @@ -1826,9 +1857,10 @@ const mcpFiltersParam = z 'If another tile\'s onClick targets THIS dashboard with `filters: [{ expression: "X", ... }]`, ' + 'this array MUST declare a filter whose `expression` is "X". Otherwise the value is ' + 'dropped on arrival and the destination opens unfiltered.\n\n' + - 'Two filter types: QUERY_EXPRESSION queries its dropdown values from a source column, ' + - 'while STATIC_LIST declares them inline via `options` and is always variable-only ' + - '(it has no expression to broadcast).\n\n' + + 'Filter types: QUERY_EXPRESSION queries its dropdown values from a source column. ' + + 'STATIC_LIST declares them inline via `options`, and SQL_QUERY reads them from the ' + + 'first column of a ClickHouse query; both are always variable-only (they have no ' + + 'expression to broadcast).\n\n' + 'A QUERY_EXPRESSION filter runs in one of two modes (or both). BROADCAST (the default) applies the ' + 'selected value as a condition on matching tiles with no per-tile wiring. VARIABLE ' + '(isVariableEnabled) exposes the value to tile queries as $variableName, which the tile ' + @@ -1867,6 +1899,13 @@ const mcpFiltersParam = z 'Always pass the column explicitly: the one-argument $__filter($env) form reuses the ' + "filter's own expression, which a static filter does not have. Prefer STATIC_LIST when " + 'the values are a fixed business list or a curated subset rather than derivable from the data.\n\n' + + 'Example (SQL query: values from a lookup table no source covers, narrowed by another filter):\n' + + '[\n' + + ' { "type": "SQL_QUERY", "name": "Environment", "connectionId": "",\n' + + ' "sqlTemplate": "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env",\n' + + ' "variableName": "env" }\n' + + ']\n' + + 'Prefer QUERY_EXPRESSION when the values are a column of a registered source.\n\n' + "Example (dependent filter: the Endpoint dropdown only lists the selected service's " + 'endpoints):\n' + '[\n' + @@ -1883,7 +1922,7 @@ const mcpFiltersParam = z ); const PROMQL_FILTERS_EXAMPLE = - '\n\nA third type, PROMETHEUS_LABEL, lists the values of a label on a promql source ' + + '\n\nAnother type, PROMETHEUS_LABEL, lists the values of a label on a promql source ' + 'and is always variable-only.\n\n' + 'Example (PROMETHEUS_LABEL):\n' + '[\n' + diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 987ab6ad2f..72f844579a 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -5,6 +5,7 @@ import { getDashboardVariableDeclarations, getFilterVariableName, isFilterVariableEnabled, + validateSqlQueryFilterTemplate, } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, @@ -21,6 +22,7 @@ import { DisplayType, QueryExpressionDashboardFilter, SavedChartConfig, + SqlQueryDashboardFilter, } from '@hyperdx/common-utils/dist/types'; import { getVariableReferences, @@ -154,11 +156,13 @@ export function getRawSqlTileMacroWarnings( } /** - * Minimal projection needed to check a filter's dropdown-values query. Static - * list filters have no dropdown query, so `where` / `whereLanguage` are optional. + * Minimal projection needed to check a filter's dropdown-values query: a + * queried filter's `where`, or a SQL query filter's `sqlTemplate`. Static list + * filters have no dropdown query, so all of these are optional. */ export type FilterForVariableValidation = FilterForVariableDeclaration & - Partial>; + Partial> & + Partial>; /** * Non-blocking checks on the dashboard variables references in a filter's `where`. @@ -202,11 +206,12 @@ export function getFilterVariableWarnings( const warnings: string[] = []; filters.forEach((filter, index) => { - const where = filter.where?.trim(); + const sqlTemplate = filter.sqlTemplate?.trim(); + const where = sqlTemplate || filter.where?.trim(); if (!where) return; const label = filter.name?.trim() || `filter #${index + 1}`; - const language = filter.whereLanguage ?? 'sql'; + const language = sqlTemplate ? 'sql' : (filter.whereLanguage ?? 'sql'); const result = validateVariableReferencesInTemplate(where, variables, { subject: 'The dropdown values query', @@ -214,6 +219,12 @@ export function getFilterVariableWarnings( }); const issues = [...result.errors, ...result.warnings]; + if (sqlTemplate) { + // Also reports variable macro errors, which `result` already holds. + const detail = validateSqlQueryFilterTemplate(sqlTemplate, variables); + if (detail && !issues.includes(detail)) issues.push(detail); + } + const referencedNames = new Set( getVariableReferences(where).map(reference => reference.name), ); diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index 2abf2baf64..e92d9bed16 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -17,6 +17,7 @@ import { makeTile, } from '@/fixtures'; import Alert, { AlertSource } from '@/models/alert'; +import Connection from '@/models/connection'; import Dashboard from '@/models/dashboard'; import PresetDashboardFilter from '@/models/presetDashboardFilter'; import { Source } from '@/models/source'; @@ -1847,6 +1848,126 @@ describe('dashboard router', () => { }); }); + describe('SQL query filters', () => { + let connectionId: string; + beforeEach(async () => { + const connection = await Connection.create({ + team: team._id, + name: 'Test Connection', + host: 'http://localhost:8123', + username: 'default', + }); + connectionId = connection._id.toString(); + }); + + const makeSqlFilter = (overrides = {}) => ({ + id: new Types.ObjectId().toString(), + type: 'SQL_QUERY' as const, + name: 'Environment', + connection: connectionId, + sqlTemplate: 'SELECT DISTINCT env FROM envs', + isBroadcastEnabled: false, + isVariableEnabled: true, + variableName: 'env', + ...overrides, + }); + + it('persists a SQL query filter on create', async () => { + const filter = makeSqlFilter(); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + expect(created.body.filters).toEqual([filter]); + }); + + it('rejects a connection that does not exist', async () => { + const connection = new Types.ObjectId().toString(); + + const response = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [makeSqlFilter({ connection })] }) + .expect(400); + + expect(response.body.message).toContain(connection); + }); + + it("rejects another team's connection", async () => { + const otherTeamConnection = await Connection.create({ + team: new Types.ObjectId(), + name: 'Other Team Connection', + host: 'http://localhost:8123', + username: 'default', + }); + + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeSqlFilter({ connection: otherTeamConnection._id.toString() }), + ], + }) + .expect(400); + }); + + // Same as PromQL sources: an accepted reference that later dangles must + // not block unrelated edits. + it("lets unrelated edits through after the filter's connection is deleted", async () => { + const filter = makeSqlFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + await Connection.findByIdAndDelete(connectionId); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ name: 'Renamed', filters: [filter] }) + .expect(200); + }); + + it('rejects a connection that PATCH introduces', async () => { + const filter = makeSqlFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ + filters: [{ ...filter, connection: new Types.ObjectId().toString() }], + }) + .expect(400); + }); + + it('accepts a PATCH that moves the filter to another team connection', async () => { + const filter = makeSqlFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: 'http://localhost:8123', + username: 'default', + }); + const moved = { ...filter, connection: otherConnection._id.toString() }; + + const updated = await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [moved] }) + .expect(200); + + expect(updated.body.filters).toEqual([moved]); + }); + }); + describe('ad hoc filters', () => { const createSource = (kind: SourceKind, teamId = team._id) => Source.create({ diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 544679ef92..0fb5d54f88 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -7,6 +7,7 @@ import { getAdhocFilterSourceKinds, isAdhocFilter, isPrometheusLabelFilter, + isSqlQueryFilter, } from '@hyperdx/common-utils/dist/filters'; import { DashboardFilter, @@ -23,6 +24,7 @@ import _ from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import { getConnectionsByTeam } from '@/controllers/connection'; import { createDashboard, deleteDashboard, @@ -122,6 +124,49 @@ async function validateFilterSources( return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${[...invalid].join(', ')}`; } +/** + * Rejects SQL_QUERY filters whose connection doesn't belong to the team. As in + * `validateFilterSources`, a connection the stored filter with the same ID + * already used is skipped. Returns an error message, or null. + */ +async function validateFilterConnections( + teamId: ObjectId, + filters: DashboardFilter[] = [], + existingFilters: DashboardFilter[] = [], +): Promise { + const existingConnectionById = new Map( + existingFilters + .filter(isSqlQueryFilter) + .map(filter => [filter.id, filter.connection]), + ); + const connectionIds = new Set( + filters + .filter(isSqlQueryFilter) + .filter( + filter => existingConnectionById.get(filter.id) !== filter.connection, + ) + .map(filter => filter.connection), + ); + if (connectionIds.size === 0) return null; + + const connections = await getConnectionsByTeam(teamId.toString()); + const knownIds = new Set(connections.map(c => c._id.toString())); + const missing = [...connectionIds].filter(id => !knownIds.has(id)); + if (missing.length === 0) return null; + return `The following filter connection IDs do not exist: ${missing.join(', ')}`; +} + +async function validateFilterReferences( + teamId: ObjectId, + filters?: DashboardFilter[], + existingFilters?: DashboardFilter[], +): Promise { + return ( + (await validateFilterSources(teamId, filters, existingFilters)) ?? + (await validateFilterConnections(teamId, filters, existingFilters)) + ); +} + /** * Heal legacy `chart-1`..`chart-10` tile colors from #2265 on the request * body *before* `validateRequest` runs `ChartPaletteTokenSchema`. Keeps the @@ -178,9 +223,12 @@ router.post( // dashboard to the provisioner. const dashboard = _.omit(req.body, 'provisioned'); - const sourceError = await validateFilterSources(teamId, req.body.filters); - if (sourceError != null) { - return res.status(400).json({ message: sourceError }); + const filterError = await validateFilterReferences( + teamId, + req.body.filters, + ); + if (filterError != null) { + return res.status(400).json({ message: filterError }); } const newDashboard = await createDashboard(teamId, dashboard, userId); @@ -223,13 +271,13 @@ router.patch( // null still signals field removal downstream. const updates = DashboardWithoutIdSchema.partial().parse(req.body); - const sourceError = await validateFilterSources( + const filterError = await validateFilterReferences( teamId, req.body.filters, dashboard.filters, ); - if (sourceError != null) { - return res.status(400).json({ message: sourceError }); + if (filterError != null) { + return res.status(400).json({ message: filterError }); } const updatedDashboard = await updateDashboard( diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 25d3f0b9c7..8772c5a2c7 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6285,6 +6285,60 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('SQL query filters', () => { + const sqlFilterInput = (overrides = {}) => ({ + id: new ObjectId().toString(), + type: 'SQL_QUERY' as const, + name: 'Environment', + connectionId: connection._id.toString(), + sqlTemplate: 'SELECT DISTINCT env FROM envs', + variableName: 'env', + ...overrides, + }); + + it('accepts a SQL query filter and round-trips it through GET', async () => { + const response = await sendFilters([sqlFilterInput()]); + expect(response.status).toBe(200); + + const [filter] = response.body.data.filters; + expect(filter).toMatchObject({ + type: 'SQL_QUERY', + name: 'Environment', + connectionId: connection._id.toString(), + sqlTemplate: 'SELECT DISTINCT env FROM envs', + isBroadcastEnabled: false, + isVariableEnabled: true, + variableName: 'env', + }); + expect(filter).not.toHaveProperty('connection'); + + const getResponse = await authRequest( + 'get', + `${BASE_URL}/${response.body.data.id}`, + ).expect(200); + expect(getResponse.body.data.filters).toEqual( + response.body.data.filters, + ); + }); + + it('rejects a connection that does not belong to the team', async () => { + const connectionId = new ObjectId().toString(); + const response = await sendFilters([sqlFilterInput({ connectionId })]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain(connectionId); + }); + + it.each([ + ['broadcast enabled', { isBroadcastEnabled: true }], + ['not variable-enabled', { isVariableEnabled: false }], + ['no SQL', { sqlTemplate: '' }], + ['a source', { sourceId: new ObjectId().toString() }], + ])('rejects a SQL query filter with %s', async (_label, overrides) => { + await expectFilters([sqlFilterInput(overrides)], 400); + }); + }); + describe('static-list filters', () => { it('accepts a static-list filter and round-trips it through GET', async () => { const response = await sendFilters([staticFilterInput()]); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 77d3b31361..3f47a774d3 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2368,12 +2368,14 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/QueryExpressionFilterInput' * - $ref: '#/components/schemas/StaticListFilterInput' * - $ref: '#/components/schemas/PrometheusLabelFilterInput' + * - $ref: '#/components/schemas/SqlQueryFilterInput' * discriminator: * propertyName: type * mapping: * QUERY_EXPRESSION: '#/components/schemas/QueryExpressionFilterInput' * STATIC_LIST: '#/components/schemas/StaticListFilterInput' * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' + * SQL_QUERY: '#/components/schemas/SqlQueryFilterInput' * * QueryExpressionFilterInput: * type: object @@ -2684,6 +2686,83 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * behavior. * example: 1 * + * SqlQueryFilterInput: + * type: object + * description: | + * A filter whose dropdown lists the first column returned by a ClickHouse + * query, cast to strings with nulls and duplicates removed. Supports + * variable mode only. + * required: + * - type + * - name + * - connectionId + * - sqlTemplate + * properties: + * type: + * type: string + * enum: [SQL_QUERY] + * description: Discriminator. Must be "SQL_QUERY". + * example: "SQL_QUERY" + * name: + * type: string + * minLength: 1 + * description: Display name for the dashboard filter + * example: "Environment" + * connectionId: + * type: string + * description: ID of the ClickHouse connection the query runs on. + * example: "65f5e4a3b9e77c001a567890" + * sqlTemplate: + * type: string + * minLength: 1 + * description: | + * Query whose first column populates the dropdown. May use the time + * macros and other filters' variables, but not `$__sourceTable` or + * `$__filters`, since the filter has no source. + * example: "SELECT DISTINCT env FROM default.envs ORDER BY env" + * isBroadcastEnabled: + * type: boolean + * enum: [false] + * default: false + * description: Must be false or omitted. + * example: false + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: | + * Must be true if provided, and is true when omitted. Tiles reference + * the selection as `$variableName`. + * example: true + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: | + * Token that tiles reference this filter's selected value by, as + * `$variableName`. Defaults to the display name with whitespace + * replaced by underscores and remaining illegal characters removed. + * Must be unique across a dashboard's variable-enabled filters. + * example: "env" + * minSelections: + * type: integer + * enum: [0, 1] + * default: 0 + * description: Set to 1 to make the filter required. + * example: 1 + * isGlobalRequirement: + * type: boolean + * default: false + * description: | + * Widens a required filter's block to every tile on the dashboard. + * Ignored unless minSelections is 1. + * example: true + * maxSelections: + * type: integer + * enum: [1] + * description: Set to 1 to make the filter single-select. + * example: 1 + * * Filter: * allOf: * - $ref: '#/components/schemas/FilterInput' diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 69e143e950..27af9366e4 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -13,10 +13,7 @@ import { validateDashboardFilterVariableNames, validateDashboardTileContainerRefs, } from '@hyperdx/common-utils/dist/dashboardValidation'; -import { - isPrometheusLabelFilter, - isStaticListFilter, -} from '@hyperdx/common-utils/dist/filters'; +import { isPrometheusLabelFilter } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, isHeatmapCompatibleSource, @@ -1309,8 +1306,7 @@ function getMissingSources( if (filters?.length) { for (const filter of filters) { - // Every variant but the static one names a source. - if (!isStaticListFilter(filter)) { + if ('sourceId' in filter) { sourceIds.add(filter.sourceId); } } @@ -1589,10 +1585,11 @@ async function getMissingOnClickDashboards( return [...dashboardIds].filter(id => !existingDashboardIds.has(id)); } -/** Returns connection IDs referenced in tiles that do not belong to the team */ +/** Returns connection IDs referenced in tiles or filters that do not belong to the team */ async function getMissingConnections( team: string | mongoose.Types.ObjectId, tiles: ExternalDashboardTileWithId[], + filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] = [], ): Promise { const connectionIds = new Set(); @@ -1601,6 +1598,11 @@ async function getMissingConnections( connectionIds.add(tile.config.connectionId); } } + for (const filter of filters) { + if ('connectionId' in filter) { + connectionIds.add(filter.connectionId); + } + } if (connectionIds.size === 0) return []; @@ -1763,7 +1765,7 @@ export async function validateDashboardTiles( invalidOnClickSearchSources, ] = await Promise.all([ fetchSourcesForValidation(teamId), - getMissingConnections(teamId, tiles), + getMissingConnections(teamId, tiles, filters), getMissingOnClickDashboards(teamId, tiles), getInvalidOnClickSearchSources(teamId, tiles), ]); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index c89015af03..4422bcf908 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -5,7 +5,6 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, - isSqlQueryFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -242,10 +241,7 @@ export function translateExternalChartToTileConfig( } /** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract< - DashboardFilter, - { type: 'ADHOC' | 'SQL_QUERY' } ->; +type InternalOnlyDashboardFilter = Extract; export type ExternalCompatibleDashboardFilter = Exclude< DashboardFilter, InternalOnlyDashboardFilter @@ -257,7 +253,7 @@ export function getExternalCompatibleFilters( ): ExternalCompatibleDashboardFilter[] { return (filters ?? []).filter( (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter) && !isSqlQueryFilter(filter), + !isAdhocFilter(filter), ); } @@ -287,6 +283,12 @@ export function translateFilterToExternalFilter( sourceId: filter.source.toString(), }; + case 'SQL_QUERY': + return { + ...omit(filter, 'connection', ...omittedKeys), + connectionId: filter.connection.toString(), + }; + case 'QUERY_EXPRESSION': { // Ignore variableName and appliesToSourceIds if the filter is not in a mode that uses them const ignoredKeys = [ @@ -318,6 +320,12 @@ export function translateExternalFilterToFilter( case 'PROMETHEUS_LABEL': return { ...omit(filter, 'sourceId'), source: filter.sourceId }; + case 'SQL_QUERY': + return { + ...omit(filter, 'connectionId'), + connection: filter.connectionId, + }; + case 'QUERY_EXPRESSION': return { ...omit(filter, 'sourceId'), source: filter.sourceId }; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index d14d34849d..2542a9d4a3 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -30,6 +30,7 @@ import { scheduleStartAtSchema, SearchConditionLanguageSchema as whereLanguageSchema, SQLIntervalSchema, + SqlQueryDashboardFilterSchema, StaticListDashboardFilterSchema, tagsSchema, TILE_DESCRIPTION_MAX_LENGTH, @@ -189,12 +190,22 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); +// `connectionId` rather than `connection`, matching raw SQL tiles. +const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ + connection: true, +}).extend({ + connectionId: objectIdSchema, + isBroadcastEnabled: z.literal(false).default(false), + isVariableEnabled: z.literal(true).default(true), +}); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ externalQueryExpressionFilterShape.strict(), externalStaticListFilterShape.strict(), externalPromqlLabelFilterShape.strict(), + externalSqlQueryFilterShape.strict(), ], ); @@ -211,6 +222,7 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalQueryExpressionFilterShape.omit({ id: true }).strict(), externalStaticListFilterShape.omit({ id: true }).strict(), externalPromqlLabelFilterShape.omit({ id: true }).strict(), + externalSqlQueryFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer<