Skip to content

Commit 989bc97

Browse files
committed
feat: add SQL_QUERY dashboard filter type to the schema
Add a variable-only dashboard filter whose dropdown values come from the first column of a ClickHouse query run on a connection. - Add SqlQueryDashboardFilterSchema and the isSqlQueryFilter guard - Validate the template's macros and variable references, rejecting $__sourceTable and $__filters - Export the filter's connection as a name in dashboard templates - Keep the type out of the external API and MCP for now Refs HDX-5524
1 parent 7b03880 commit 989bc97

10 files changed

Lines changed: 246 additions & 5 deletions

File tree

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
'@hyperdx/common-utils': patch
3+
'@hyperdx/api': patch
4+
'@hyperdx/app': patch
5+
---
6+
7+
feat: add the SQL query dashboard filter type to the dashboard schema
8+
9+
A `SQL_QUERY` filter stores a ClickHouse connection and a SQL template whose first column fills the filter's dropdown. It is variable-only. Validation flags templates that use `$__sourceTable` or `$__filters`, or whose macros and variable references can't expand. The filter is not yet exposed through the external API or MCP, and has no editor or values lookup yet.

‎packages/api/src/utils/externalApi.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
isFilterBroadcastEnabled,
66
isFilterRequired,
77
isFilterVariableEnabled,
8+
isSqlQueryFilter,
89
} from '@hyperdx/common-utils/dist/filters';
910
import {
1011
AlertErrorType,
@@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig(
241242
}
242243

243244
/** Filter types the external API and MCP don't expose yet. */
244-
type InternalOnlyDashboardFilter = Extract<DashboardFilter, { type: 'ADHOC' }>;
245+
type InternalOnlyDashboardFilter = Extract<
246+
DashboardFilter,
247+
{ type: 'ADHOC' | 'SQL_QUERY' }
248+
>;
245249
export type ExternalCompatibleDashboardFilter = Exclude<
246250
DashboardFilter,
247251
InternalOnlyDashboardFilter
@@ -253,7 +257,7 @@ export function getExternalCompatibleFilters(
253257
): ExternalCompatibleDashboardFilter[] {
254258
return (filters ?? []).filter(
255259
(filter): filter is ExternalCompatibleDashboardFilter =>
256-
!isAdhocFilter(filter),
260+
!isAdhocFilter(filter) && !isSqlQueryFilter(filter),
257261
);
258262
}
259263

‎packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
IconPencil,
2424
IconRefresh,
2525
IconSearch,
26+
IconSql,
2627
IconTrash,
2728
} from '@tabler/icons-react';
2829

@@ -91,6 +92,12 @@ function getValuesAttribute(
9192
tooltip: 'Sources the condition keys and values come from',
9293
label: getSourceNames(filter.sources, sources),
9394
};
95+
case 'SQL_QUERY':
96+
return {
97+
icon: <IconSql size={14} />,
98+
tooltip: 'The dropdown values come from a SQL query',
99+
label: 'SQL query',
100+
};
94101
default:
95102
filter satisfies never; // exhaustive check
96103
return { icon: <IconSearch size={14} />, label: '' };

‎packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ describe('toSavedFilter', () => {
326326

327327
/** The minimum a form needs to save, per filter type the form supports. */
328328
const byType: Record<
329-
Exclude<DashboardFilter['type'], 'ADHOC'>,
329+
Exclude<DashboardFilter['type'], 'ADHOC' | 'SQL_QUERY'>,
330330
Partial<FilterFormValues>
331331
> = {
332332
QUERY_EXPRESSION: {

‎packages/common-utils/src/__tests__/filters.test.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import type {
4444
Filter,
4545
PromqlConditionFilter,
4646
QueryExpressionDashboardFilter,
47+
SqlQueryDashboardFilter,
4748
StaticListDashboardFilter,
4849
} from '@/types';
4950
import {
@@ -740,6 +741,74 @@ describe('filters', () => {
740741
).toEqual([]);
741742
});
742743

744+
describe('SQL query filters', () => {
745+
const sqlFilter = (sqlTemplate: string): SqlQueryDashboardFilter => ({
746+
id: 'sql',
747+
type: 'SQL_QUERY',
748+
name: 'Environment',
749+
connection: 'conn-1',
750+
sqlTemplate,
751+
isBroadcastEnabled: false,
752+
isVariableEnabled: true,
753+
});
754+
const cloud = filter({
755+
id: 'cloud',
756+
name: 'cloud',
757+
isVariableEnabled: true,
758+
});
759+
760+
it('accepts time macros and references to other variables', () => {
761+
expect(
762+
validateDashboardFilterQueries([
763+
cloud,
764+
sqlFilter(
765+
'SELECT env FROM envs WHERE $__timeFilter(ts) AND $__filter(cloud, $cloud) AND region IN ($cloud)',
766+
),
767+
]),
768+
).toEqual([]);
769+
});
770+
771+
// Running it is the only reliable check of ClickHouse syntax.
772+
it('does not parse the SQL itself', () => {
773+
expect(
774+
validateDashboardFilterQueries([sqlFilter('SELEC env FRM')]),
775+
).toEqual([]);
776+
});
777+
778+
it.each([
779+
['$__sourceTable', 'SELECT env FROM $__sourceTable'],
780+
['$__filters', 'SELECT env FROM envs WHERE $__filters'],
781+
])(
782+
'flags %s, which needs a source the filter does not have',
783+
(macro, sql) => {
784+
expect(validateDashboardFilterQueries([sqlFilter(sql)])).toEqual([
785+
{
786+
filterId: 'sql',
787+
filterName: 'Environment',
788+
language: 'sql',
789+
where: sql,
790+
detail: `SQL query filters have no source, so ${macro} cannot be used`,
791+
},
792+
]);
793+
},
794+
);
795+
796+
it.each([
797+
['an undeclared variable macro', 'WHERE $__filter(env, $missing)'],
798+
['an unterminated macro', 'WHERE $__timeFilter(ts'],
799+
['an unknown variable format', 'WHERE env IN (${cloud:bogus})'],
800+
])('flags %s', (_label, sql) => {
801+
const issues = validateDashboardFilterQueries([cloud, sqlFilter(sql)]);
802+
803+
expect(issues).toHaveLength(1);
804+
expect(issues[0]).toMatchObject({
805+
filterId: 'sql',
806+
where: sql,
807+
detail: expect.any(String),
808+
});
809+
});
810+
});
811+
743812
it('accepts a valid lucene where clause', () => {
744813
expect(
745814
validateDashboardFilterQueries([

‎packages/common-utils/src/__tests__/types.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,48 @@ describe('DashboardFilterSchema static-list variant', () => {
678678
});
679679
});
680680

681+
describe('DashboardFilterSchema SQL query variant', () => {
682+
const sqlFilter = {
683+
id: 'f1',
684+
type: 'SQL_QUERY' as const,
685+
name: 'Environment',
686+
connection: 'conn-1',
687+
sqlTemplate: 'SELECT DISTINCT env FROM envs',
688+
isBroadcastEnabled: false,
689+
isVariableEnabled: true,
690+
variableName: 'env',
691+
};
692+
693+
it('parses a SQL query filter', () => {
694+
expect(DashboardFilterSchema.parse(sqlFilter)).toEqual(sqlFilter);
695+
});
696+
697+
it.each(['connection', 'sqlTemplate'] as const)(
698+
'requires a non-empty %s',
699+
field => {
700+
for (const value of [undefined, '']) {
701+
expect(
702+
DashboardFilterSchema.safeParse({ ...sqlFilter, [field]: value })
703+
.success,
704+
).toBe(false);
705+
}
706+
},
707+
);
708+
709+
it.each([
710+
['broadcast enabled', { isBroadcastEnabled: true }],
711+
['variables disabled', { isVariableEnabled: false }],
712+
])('rejects a SQL query filter with %s', (_label, overrides) => {
713+
const result = DashboardFilterSchema.safeParse({
714+
...sqlFilter,
715+
...overrides,
716+
});
717+
718+
expect(result.success).toBe(false);
719+
expect(result.error?.issues[0].path).toEqual([Object.keys(overrides)[0]]);
720+
});
721+
});
722+
681723
describe('DashboardFilterSchema query-expression variant', () => {
682724
const queryFilter = {
683725
id: 'f1',

‎packages/common-utils/src/__tests__/utils.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1606,6 +1606,39 @@ describe('utils', () => {
16061606
).toEqual(staticFilter);
16071607
});
16081608

1609+
it('should export a SQL query filter with its connection id remapped to a name', () => {
1610+
const sqlFilter = {
1611+
id: 'filter-sql',
1612+
type: 'SQL_QUERY' as const,
1613+
name: 'Environment',
1614+
connection: 'conn1',
1615+
sqlTemplate: 'SELECT DISTINCT env FROM envs',
1616+
isBroadcastEnabled: false as const,
1617+
isVariableEnabled: true as const,
1618+
};
1619+
const dashboard: z.infer<typeof DashboardSchema> = {
1620+
id: 'dashboard1',
1621+
name: 'SQL Filter Dashboard',
1622+
tags: [],
1623+
tiles: [],
1624+
filters: [sqlFilter],
1625+
};
1626+
const connections: Connection[] = [
1627+
{
1628+
id: 'conn1',
1629+
name: 'Production DB',
1630+
host: 'http://localhost:8123',
1631+
username: 'default',
1632+
},
1633+
];
1634+
1635+
const template = convertToDashboardTemplate(dashboard, [], connections);
1636+
1637+
expect(template.filters).toEqual([
1638+
{ ...sqlFilter, connection: 'Production DB' },
1639+
]);
1640+
});
1641+
16091642
it('should convert a dashboard without filters to a dashboard template', () => {
16101643
const dashboard: z.infer<typeof DashboardSchema> = {
16111644
id: 'dashboard1',

‎packages/common-utils/src/core/utils.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ export { default as objectHash } from 'object-hash';
99
import {
1010
isAdhocFilter,
1111
isQueryExpressionFilter,
12+
isSqlQueryFilter,
1213
isStaticListFilter,
1314
} from '@/filters';
1415
import {
@@ -944,12 +945,19 @@ export function convertToDashboardTemplate(
944945
const convertToFilterTemplate = (
945946
input: DashboardFilter,
946947
sources: TSource[],
948+
connections: Connection[],
947949
): DashboardFilter => {
948950
const filter = DashboardFilterSchema.parse(structuredClone(input));
949951

950952
// A static filter references nothing in the workspace
951953
if (isStaticListFilter(filter)) return filter;
952954

955+
if (isSqlQueryFilter(filter)) {
956+
filter.connection =
957+
connections.find(conn => conn.id === filter.connection)?.name ?? '';
958+
return filter;
959+
}
960+
953961
const getSourceName = (id: string) =>
954962
sources.find(source => source.id === id)?.name ?? '';
955963

@@ -975,7 +983,9 @@ export function convertToDashboardTemplate(
975983
if (input.filters) {
976984
output.filters = [];
977985
for (const filter of input.filters ?? []) {
978-
output.filters.push(convertToFilterTemplate(filter, sources));
986+
output.filters.push(
987+
convertToFilterTemplate(filter, sources, connections),
988+
);
979989
}
980990
}
981991

‎packages/common-utils/src/filters.ts‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as SQLParser from 'node-sql-parser';
22

33
import { escapeSqlString, replaceJsonExpressions } from '@/core/utils';
4+
import { getSourceDependentMacrosUsed, replaceMacros } from '@/macros';
45
import { parse } from '@/queryParser';
56
import {
67
AdhocDashboardFilter,
@@ -770,7 +771,7 @@ export type DashboardFilterQueryIssue = {
770771
filterName: string;
771772
/** Query language of the filter's `where` clause */
772773
language: 'lucene' | 'sql';
773-
/** The raw `where` clause that failed to parse */
774+
/** The raw `where` clause, or a SQL query filter's template, that failed to validate */
774775
where: string;
775776
/** Why it failed, when the clause itself parses but its variables don't resolve. */
776777
detail?: string;
@@ -803,6 +804,23 @@ export function validateDashboardFilterQueries(
803804
).map(declaration => ({ ...declaration, values: [] }));
804805

805806
for (const filter of filters) {
807+
if (isSqlQueryFilter(filter)) {
808+
const detail = validateSqlQueryFilterTemplate(
809+
filter.sqlTemplate,
810+
variables,
811+
);
812+
if (detail) {
813+
issues.push({
814+
filterId: filter.id,
815+
filterName: filter.name,
816+
language: 'sql',
817+
where: filter.sqlTemplate,
818+
detail,
819+
});
820+
}
821+
continue;
822+
}
823+
806824
// Only ClickHouse-queried filters carry a values query to validate.
807825
// A PromQL ad hoc filter may store a `where`, but nothing reads it.
808826
const isSqlAdhoc = isAdhocFilter(filter) && filter.sourceType === 'sql';
@@ -832,6 +850,28 @@ export function validateDashboardFilterQueries(
832850
return issues;
833851
}
834852

853+
/**
854+
* Why a SQL query filter's template can't expand, or undefined when it can.
855+
* The SQL itself is not parsed: ClickHouse syntax is only checked when it runs.
856+
*/
857+
function validateSqlQueryFilterTemplate(
858+
sqlTemplate: string,
859+
variables: ChartVariable[],
860+
): string | undefined {
861+
try {
862+
const sourceMacros = getSourceDependentMacrosUsed(sqlTemplate);
863+
if (sourceMacros.length > 0) {
864+
return `SQL query filters have no source, so ${sourceMacros
865+
.map(macro => `$__${macro}`)
866+
.join(' and ')} cannot be used`;
867+
}
868+
replaceMacros({ sqlTemplate, variables });
869+
return undefined;
870+
} catch (e) {
871+
return e instanceof Error ? e.message : String(e);
872+
}
873+
}
874+
835875
/** Derive the default `variableName` for a filter from its display name. */
836876
export function deriveVariableName(filterName: string): string {
837877
const sanitized = filterName
@@ -938,6 +978,13 @@ export function isAdhocFilter<T extends { type: DashboardFilterKind }>(
938978
return filter.type === 'ADHOC';
939979
}
940980

981+
/** Type guard for SQL_QUERY type filters. */
982+
export function isSqlQueryFilter<T extends { type: DashboardFilterKind }>(
983+
filter: T,
984+
): filter is Extract<T, { type: 'SQL_QUERY' }> {
985+
return filter.type === 'SQL_QUERY';
986+
}
987+
941988
/**
942989
* The sources an ADHOC filter broadcasts its conditions to: its
943990
* `appliesToSourceIds`, or its own `sources` when that is unset or empty.

0 commit comments

Comments
 (0)