diff --git a/.changeset/adhoc-filter-schemas.md b/.changeset/adhoc-filter-schemas.md new file mode 100644 index 0000000000..1a7777dc09 --- /dev/null +++ b/.changeset/adhoc-filter-schemas.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +feat: Add ad hoc dashboard filter schemas and types diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index dcaf898cd5..8a02026a0d 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1185,7 +1185,7 @@ describe('dashboard router', () => { .expect(200); const stored = await Dashboard.findById(created.body.id).lean(); - expect(stored?.filters?.[0].minSelections).toBe(1); + expect(stored?.filters?.[0]).toMatchObject({ minSelections: 1 }); }); // Absence is meaningful: the client reads a missing value as not required, @@ -1794,6 +1794,306 @@ describe('dashboard router', () => { }); }); + describe('ad hoc filters', () => { + const createSource = (kind: SourceKind, teamId = team._id) => + Source.create({ + kind, + name: `Test ${kind} source`, + team: teamId, + connection: new Types.ObjectId().toString(), + from: { databaseName: 'test_db', tableName: `${kind}_table` }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'body', + ...(kind === SourceKind.Metric && { + metricTables: { gauge: 'otel_metrics_gauge' }, + resourceAttributesExpression: 'ResourceAttributes', + }), + }); + + let logSourceId: string; + let traceSourceId: string; + let promqlSourceId: string; + beforeEach(async () => { + logSourceId = (await createSource(SourceKind.Log))._id.toString(); + traceSourceId = (await createSource(SourceKind.Trace))._id.toString(); + promqlSourceId = (await createSource(SourceKind.Promql))._id.toString(); + }); + + const makeAdhocFilter = (overrides = {}) => ({ + id: new Types.ObjectId().toString(), + type: 'ADHOC' as const, + name: 'Ad hoc', + sourceType: 'sql' as const, + sources: [logSourceId], + isVariableEnabled: true as const, + variableName: 'adhoc', + ...overrides, + }); + + const adhocValue = { + type: 'adhoc' as const, + name: 'adhoc', + conditions: [ + { key: 'ServiceName', operator: '=~' as const, value: 'api.*' }, + { + key: "LogAttributes['env']", + operator: 'NOT LIKE' as const, + value: 'dev', + }, + ], + }; + + it('persists a SQL ad hoc filter and its saved conditions', async () => { + const filter = makeAdhocFilter({ + sources: [logSourceId, traceSourceId], + appliesToSourceIds: [traceSourceId], + isBroadcastEnabled: false, + }); + + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [filter], + savedFilterValues: [adhocValue], + }) + .expect(200); + + expect(created.body.filters).toEqual([filter]); + expect(created.body.savedFilterValues).toEqual([adhocValue]); + + const stored = await Dashboard.findById(created.body.id).lean(); + expect(stored?.filters).toEqual([filter]); + expect(stored?.savedFilterValues).toEqual([adhocValue]); + + const list = await agent.get('/dashboards').expect(200); + const listed = list.body.find( + (d: { id: string }) => d.id === created.body.id, + ); + expect(listed.filters).toEqual([filter]); + expect(listed.savedFilterValues).toEqual([adhocValue]); + }); + + it('persists a PromQL ad hoc filter', async () => { + const filter = makeAdhocFilter({ + sourceType: 'promql', + sources: [promqlSourceId], + }); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + const stored = await Dashboard.findById(created.body.id).lean(); + expect(stored?.filters).toEqual([filter]); + }); + + it('persists saved conditions alongside other filter values on PATCH', async () => { + const filter = makeAdhocFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + const savedFilterValues = [ + { type: 'sql' as const, condition: "ServiceName IN ('api')" }, + { type: 'variable' as const, name: 'env', values: ['prod'] }, + adhocValue, + ]; + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ savedFilterValues }) + .expect(200); + + const stored = await Dashboard.findById(created.body.id).lean(); + expect(stored?.savedFilterValues).toEqual(savedFilterValues); + }); + + it.each([ + ['no source type', { sourceType: undefined }], + ['an unknown source type', { sourceType: 'lucene' }], + ['no sources', { sources: undefined }], + ['empty sources', { sources: [] }], + ['an empty source ID', { sources: [''] }], + ['an invalid variable name', { variableName: '1adhoc' }], + ['not variable-enabled', { isVariableEnabled: false }], + ['variables unset', { isVariableEnabled: undefined }], + ['a minimum selection', { minSelections: 1 }], + ['a maximum selection', { maxSelections: 1 }], + ['a global requirement', { isGlobalRequirement: true }], + ])('rejects an ad hoc filter with %s', async (_label, overrides) => { + await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [makeAdhocFilter(overrides)] }) + .expect(400); + }); + + it.each([ + [ + 'an unknown operator', + { conditions: [{ key: 'a', operator: '>', value: '1' }] }, + ], + [ + 'an empty key', + { conditions: [{ key: '', operator: '=', value: '1' }] }, + ], + [ + 'too many conditions', + { + conditions: Array.from({ length: 101 }, () => ({ + key: 'a', + operator: '=', + value: '1', + })), + }, + ], + ])('rejects saved conditions with %s', async (_label, overrides) => { + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeAdhocFilter()], + savedFilterValues: [{ ...adhocValue, ...overrides }], + }) + .expect(400); + }); + + it('rejects a variable name it shares with another filter', async () => { + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeAdhocFilter({ variableName: 'service' }), + { + id: new Types.ObjectId().toString(), + type: 'QUERY_EXPRESSION' as const, + name: 'Service', + expression: 'ServiceName', + source: logSourceId, + isVariableEnabled: true, + variableName: 'service', + }, + ], + }) + .expect(400); + }); + + describe('source validation', () => { + it('rejects a source that does not exist', async () => { + const source = new Types.ObjectId().toString(); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeAdhocFilter({ sources: [logSourceId, source] })], + }) + .expect(400); + + expect(response.body.message).toContain(source); + expect(response.body.message).not.toContain(logSourceId); + }); + + it.each([ + ['a PromQL source in a SQL filter', 'sql', SourceKind.Promql], + ['a metric source in a SQL filter', 'sql', SourceKind.Metric], + ['a log source in a PromQL filter', 'promql', SourceKind.Log], + ])('rejects %s', async (_label, sourceType, kind) => { + const source = (await createSource(kind))._id.toString(); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeAdhocFilter({ sourceType, sources: [source] })], + }) + .expect(400); + + expect(response.body.message).toContain(source); + }); + + it('rejects an applies-to source of the wrong type', async () => { + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeAdhocFilter({ appliesToSourceIds: [promqlSourceId] }), + ], + }) + .expect(400); + + expect(response.body.message).toContain(promqlSourceId); + }); + + it("rejects another team's source", async () => { + const otherTeamSource = await createSource( + SourceKind.Log, + new Types.ObjectId(), + ); + + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeAdhocFilter({ sources: [otherTeamSource._id.toString()] }), + ], + }) + .expect(400); + }); + + it('rejects a source added by PATCH, leaving the stored filter alone', async () => { + const filter = makeAdhocFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ + filters: [{ ...filter, sources: [logSourceId, promqlSourceId] }], + }) + .expect(400); + + const stored = await Dashboard.findById(created.body.id).lean(); + expect(stored?.filters).toEqual([filter]); + }); + + it("lets unrelated edits through after a filter's source is deleted", async () => { + const filter = makeAdhocFilter({ appliesToSourceIds: [traceSourceId] }); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + await Source.findByIdAndDelete(logSourceId); + await Source.findByIdAndDelete(traceSourceId); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ name: 'Renamed', filters: [{ ...filter, name: 'Renamed' }] }) + .expect(200); + }); + + it('rejects a PATCH changing the source type to one its sources do not match', async () => { + const filter = makeAdhocFilter(); + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [{ ...filter, sourceType: 'promql' }] }) + .expect(400); + }); + }); + }); + describe('preset dashboards', () => { const MOCK_SOURCE: Omit, 'id'> = { kind: SourceKind.Log, diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 9b8cde0b8c..778fd4914d 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -3,7 +3,11 @@ import { validateDashboardFilterOptionUniqueness, validateDashboardFilterVariableNames, } from '@hyperdx/common-utils/dist/dashboardValidation'; -import { isPrometheusLabelFilter } from '@hyperdx/common-utils/dist/filters'; +import { + getAdhocFilterSourceKinds, + isAdhocFilter, + isPrometheusLabelFilter, +} from '@hyperdx/common-utils/dist/filters'; import { DashboardFilter, DashboardSchema, @@ -11,6 +15,7 @@ import { PresetDashboard, PresetDashboardFilterSchema, resolveChartPaletteToken, + SourceKind, walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import express from 'express'; @@ -34,7 +39,6 @@ import { import { getSources } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import type { ObjectId } from '@/models'; -import { getPromqlLabelFilterSourceError } from '@/routers/external-api/v2/utils/dashboards'; import { objectIdSchema } from '@/utils/zod'; // create routes that will get and update dashboards @@ -60,40 +64,62 @@ const addFilterIssues = ( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); }; -/** Returns new and updated PROMETHEUS_LABEL filters. */ -function filterChangedPromqlLabelFilters( - filters: DashboardFilter[], - existingFilters: DashboardFilter[], -) { - const existingSourceById = new Map( - existingFilters - .filter(isPrometheusLabelFilter) - .map(filter => [filter.id, filter.source]), - ); - return filters - .filter(isPrometheusLabelFilter) - .filter(filter => existingSourceById.get(filter.id) !== filter.source); +/** The sources a filter reads from, each with the kinds it must be. */ +function getFilterSourceRefs( + filter: DashboardFilter, +): { id: string; kinds: SourceKind[] }[] { + if (isPrometheusLabelFilter(filter)) { + return [{ id: filter.source, kinds: [SourceKind.Promql] }]; + } + if (isAdhocFilter(filter)) { + const kinds = getAdhocFilterSourceKinds(filter.sourceType); + return [...filter.sources, ...(filter.appliesToSourceIds ?? [])].map( + id => ({ id, kinds }), + ); + } + return []; } /** - * Rejects PROMETHEUS_LABEL filters whose `source` is missing or is not a PromQL - * source. Returns an error message, or null when there is nothing to reject. + * Rejects PROMETHEUS_LABEL and ADHOC filters that reference a source that is + * missing or of the wrong kind. Resolving these filters needs a live source of + * the right kind, unlike other filter types. References the stored filter with + * the same ID already made are skipped, so a since-deleted source doesn't block + * unrelated saves. Returns an error message, or null when there is nothing to + * reject. */ -async function validatePromqlLabelFilterSources( +async function validateFilterSources( teamId: ObjectId, filters: DashboardFilter[] = [], - existingFilters?: DashboardFilter[], + existingFilters: DashboardFilter[] = [], ): Promise { - const promqlLabelFilters = existingFilters - ? filterChangedPromqlLabelFilters(filters, existingFilters) - : filters.filter(isPrometheusLabelFilter); - if (promqlLabelFilters.length === 0) return null; + const refKey = ({ id, kinds }: { id: string; kinds: SourceKind[] }) => + `${id}:${kinds.join(',')}`; + const existingRefsById = new Map( + existingFilters.map(filter => [ + filter.id, + new Set(getFilterSourceRefs(filter).map(refKey)), + ]), + ); + const newRefs = filters.flatMap(filter => + getFilterSourceRefs(filter).filter( + ref => !existingRefsById.get(filter.id)?.has(refKey(ref)), + ), + ); + if (newRefs.length === 0) return null; const sources = await getSources(teamId.toString()); - return getPromqlLabelFilterSourceError( - sources, - promqlLabelFilters.map(filter => filter.source), + const kindById = new Map(sources.map(s => [s._id.toString(), s.kind])); + const invalid = new Set( + newRefs + .filter(({ id, kinds }) => { + const kind = kindById.get(id); + return kind === undefined || !kinds.includes(kind); + }) + .map(({ id }) => id), ); + if (invalid.size === 0) return null; + return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${[...invalid].join(', ')}`; } /** @@ -152,10 +178,7 @@ router.post( // dashboard to the provisioner. const dashboard = _.omit(req.body, 'provisioned'); - const sourceError = await validatePromqlLabelFilterSources( - teamId, - req.body.filters, - ); + const sourceError = await validateFilterSources(teamId, req.body.filters); if (sourceError != null) { return res.status(400).json({ message: sourceError }); } @@ -193,10 +216,10 @@ router.patch( // `provisioned` is server-owned — see the POST handler above. const updates = _.omitBy(_.omit(req.body, 'provisioned'), _.isUndefined); - const sourceError = await validatePromqlLabelFilterSources( + const sourceError = await validateFilterSources( teamId, req.body.filters, - dashboard.filters ?? [], + dashboard.filters, ); if (sourceError != null) { return res.status(400).json({ message: sourceError }); diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index cdaa214268..67b841dae5 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8154,6 +8154,58 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('ad hoc filters', () => { + const adhocFilter = () => ({ + id: new ObjectId().toString(), + type: 'ADHOC' as const, + name: 'Ad hoc', + sourceType: 'sql' as const, + sources: [traceSource._id.toString()], + isVariableEnabled: true as const, + variableName: 'adhoc', + }); + const adhocValue = { + type: 'adhoc' as const, + name: 'adhoc', + conditions: [{ key: 'ServiceName', operator: '=', value: 'api' }], + }; + + it('hides ad hoc filters and their saved values from GET', async () => { + const dashboard = await createTestDashboard({ + filters: [adhocFilter()], + savedFilterValues: [adhocValue], + }); + + const response = await authRequest( + 'get', + `${BASE_URL}/${dashboard._id}`, + ).expect(200); + + expect(response.body.data.filters).toEqual([]); + expect(response.body.data.savedFilterValues).toEqual([]); + }); + + it('rejects an ad hoc filter in a POST', async () => { + await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + filters: [omit(adhocFilter(), 'id')], + }), + ) + .expect(400); + }); + + it('rejects ad hoc saved values in a POST', async () => { + await authRequest('post', BASE_URL) + .send( + createMockDashboard(traceSource._id.toString(), { + savedFilterValues: [adhocValue], + }), + ) + .expect(400); + }); + }); + describe('DELETE /:id', () => { it('should delete a dashboard', async () => { const dashboard = await createTestDashboard(); diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index c535ae9a69..2273d27370 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -54,6 +54,8 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { + getExternalCompatibleFilters, + getExternalCompatibleFilterValues, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, @@ -682,10 +684,14 @@ export function convertToExternalDashboard( .map(tile => convertTileToExternalChart(tile, containerById, dashboardId)) .filter(t => t !== undefined), tags: dashboard.tags || [], - filters: dashboard.filters?.map(translateFilterToExternalFilter) || [], + filters: getExternalCompatibleFilters(dashboard.filters).map( + translateFilterToExternalFilter, + ), savedQuery: dashboard.savedQuery ?? null, savedQueryLanguage: dashboard.savedQueryLanguage ?? null, - savedFilterValues: dashboard.savedFilterValues ?? [], + savedFilterValues: getExternalCompatibleFilterValues( + dashboard.savedFilterValues, + ), // Mongoose persists missing arrays as []. Only emit containers when // the user actually saved one or more, so dashboards without the // organization layer round-trip with the field absent. @@ -1145,7 +1151,7 @@ function getHeatmapTilesWithIncompatibleSources( * Returns an error message string if any of the referenced source IDs is not * a valid PromQL source, or null if all of them are. */ -export function getPromqlLabelFilterSourceError( +function getPromqlLabelFilterSourceError( sources: SourceForValidation[], referencedSourceIds: string[], ): string | null { diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index be0b3e0d1b..0f0721e11e 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -1,5 +1,7 @@ import { TIME_SERIES_DISPLAY_TYPE_BY_NAME } from '@hyperdx/common-utils/dist/core/utils'; import { + isAdhocFilter, + isAdhocFilterValue, isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, @@ -225,8 +227,32 @@ export function translateExternalChartToTileConfig( }; } +/** Filter types the external API and MCP don't expose yet. */ +type InternalOnlyDashboardFilter = Extract; +export type ExternalCompatibleDashboardFilter = Exclude< + DashboardFilter, + InternalOnlyDashboardFilter +>; + +/** The filters the external API and MCP may read. */ +export function getExternalCompatibleFilters( + filters: DashboardFilter[] | undefined, +): ExternalCompatibleDashboardFilter[] { + return (filters ?? []).filter( + (filter): filter is ExternalCompatibleDashboardFilter => + !isAdhocFilter(filter), + ); +} + +/** Saved filter values the external API and MCP may read. */ +export function getExternalCompatibleFilterValues( + values: IDashboard['savedFilterValues'], +): NonNullable { + return (values ?? []).filter(value => !isAdhocFilterValue(value)); +} + export function translateFilterToExternalFilter( - filter: DashboardFilter, + filter: ExternalCompatibleDashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 9050d2abd9..9e63561373 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -17,9 +17,10 @@ import { zodResolver } from '@hookform/resolvers/zod'; import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils'; import { type DashboardFilterQueryIssue, + isAdhocFilter, + isAdhocFilterValue, isPrometheusLabelFilter, isQueryExpressionFilter, - isStaticListFilter, QUERY_EXPRESSION_FILTER_SOURCE_KINDS, type SavedFilterValueIssue, type SavedQueryIssue, @@ -39,6 +40,8 @@ import { isOnClickDashboardById, isOnClickSearchById, isTraceSource, + type PromqlLabelDashboardFilter, + type QueryExpressionDashboardFilter, SavedChartConfig, SourceKind, TSource, @@ -392,9 +395,15 @@ function resolveAppliesToSources( }; } +/** Whether the filter names a single source that import must map. */ +const isSourceBackedTemplateFilter = ( + filter: DashboardFilter, +): filter is QueryExpressionDashboardFilter | PromqlLabelDashboardFilter => + isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter); + /** Returns the filter if it names a source, or `undefined` if it doesn't. */ const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) => - filter && !isStaticListFilter(filter) ? filter : undefined; + filter && isSourceBackedTemplateFilter(filter) ? filter : undefined; /** * The query-expression half of a template filter, or `undefined` for any other @@ -486,7 +495,7 @@ export function buildMappingFormSchema( }); input.filters?.forEach((filter, idx) => { - if (isStaticListFilter(filter)) return; + if (!isSourceBackedTemplateFilter(filter)) return; const mappedSourceId = data.filterSourceMappings?.[idx]; if (!mappedSourceId) { ctx.addIssue({ @@ -575,9 +584,10 @@ export function Mapping({ input }: { input: DashboardTemplate }) { }); const filterSourceMappings = input.filters?.map(filter => { - if (isStaticListFilter(filter)) return ''; + const sourceName = sourceBackedTemplateFilter(filter)?.source; + if (sourceName == null) return ''; const match = sources.find( - source => source.name.toLowerCase() === filter.source.toLowerCase(), + source => source.name.toLowerCase() === sourceName.toLowerCase(), ); return match?.id || ''; }); @@ -944,25 +954,31 @@ export function Mapping({ input }: { input: DashboardTemplate }) { }); // Zip the source mappings with the input filters - const zippedFilters = input.filters?.map((filter, idx) => { - const source = findSource(data.filterSourceMappings?.[idx]); - const appliesTo = data.filterAppliesToSourceMappings?.[idx]?.filter( - id => !!id?.length, - ); - return { - ...filter, - // A sourceless filter (`STATIC_LIST`) gets no mapping row, so - // `source` is undefined here - ...(source ? { source: source.id } : {}), - appliesToSourceIds: appliesTo?.length ? appliesTo : undefined, - }; - }); + const zippedFilters = input.filters + ?.map((filter, idx) => { + const source = findSource(data.filterSourceMappings?.[idx]); + const appliesTo = data.filterAppliesToSourceMappings?.[idx]?.filter( + id => !!id?.length, + ); + return { + ...filter, + // A sourceless filter (`STATIC_LIST`) gets no mapping row, so + // `source` is undefined here + ...(source ? { source: source.id } : {}), + appliesToSourceIds: appliesTo?.length ? appliesTo : undefined, + }; + }) + // Ad hoc filters aren't importable yet. + .filter(filter => !isAdhocFilter(filter)); // Format for server const output = convertToDashboardDocument({ ...input, tiles: zippedTiles, filters: zippedFilters, + savedFilterValues: input.savedFilterValues?.filter( + value => !isAdhocFilterValue(value), + ), name: data.dashboardName, tags: data.tags, }); @@ -1123,9 +1139,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { {/** Map filter sources */} {input.filters?.map((filter, i) => ( - {/* A static filter reads its values from the list stored on it, - so it has neither a source nor an applies-to list to map. */} - {!isStaticListFilter(filter) && ( + {isSourceBackedTemplateFilter(filter) && ( {filter.name} (Filter) Data Source diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 40520a3e07..29bd584a28 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -35,6 +35,7 @@ import { isTimeSeriesDisplayType, } from '@hyperdx/common-utils/dist/core/utils'; import { getBlockingRequiredFilterNames } from '@hyperdx/common-utils/dist/dashboardFilterValues'; +import { isAdhocFilter } from '@hyperdx/common-utils/dist/filters'; import { displayTypeRequiresSource, isBuilderChartConfig, @@ -1978,7 +1979,11 @@ function DBDashboardPage({ const [showFiltersModal, setShowFiltersModal] = useState(false); - const filters = dashboard?.filters ?? []; + // Ad hoc filters aren't rendered or applied yet + const filters = useMemo( + () => (dashboard?.filters ?? []).filter(filter => !isAdhocFilter(filter)), + [dashboard?.filters], + ); const { selectionByFilterId, setFilterValue, diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx index d75f2c69e4..4b97c8a53a 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx @@ -74,6 +74,9 @@ function getValuesAttribute( label: `${count} custom option${count === 1 ? '' : 's'}`, }; } + // Not shown in the dashboard UI yet + case 'ADHOC': + return { icon: , label: '' }; default: filter satisfies never; // exhaustive check return { icon: , label: '' }; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx index f7b55a1791..91d55296f3 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx @@ -1,4 +1,7 @@ -import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; +import { + DashboardFilter, + StaticListDashboardFilter, +} from '@hyperdx/common-utils/dist/types'; import { screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -369,7 +372,7 @@ describe('DashboardFiltersModal', () => { await user.click(screen.getByTestId('save-filter-button')); await waitFor(() => expect(onSaveFilter).toHaveBeenCalledTimes(1)); - const saved: DashboardFilter = onSaveFilter.mock.calls[0][0]; + const saved: StaticListDashboardFilter = onSaveFilter.mock.calls[0][0]; expect(saved.minSelections).toBe(1); expect(saved.isGlobalRequirement).toBeUndefined(); @@ -392,7 +395,7 @@ describe('DashboardFiltersModal', () => { await user.click(screen.getByTestId('save-filter-button')); await waitFor(() => expect(onSaveFilter).toHaveBeenCalledTimes(1)); - const saved: DashboardFilter = onSaveFilter.mock.calls[0][0]; + const saved: StaticListDashboardFilter = onSaveFilter.mock.calls[0][0]; expect(saved.maxSelections).toBe(1); expect(saved.minSelections).toBeUndefined(); @@ -433,7 +436,7 @@ describe('DashboardFiltersModal', () => { await user.click(screen.getByTestId('save-filter-button')); await waitFor(() => expect(onSaveFilter).toHaveBeenCalledTimes(1)); - const saved: DashboardFilter = onSaveFilter.mock.calls[0][0]; + const saved: StaticListDashboardFilter = onSaveFilter.mock.calls[0][0]; expect(saved.minSelections).toBe(1); expect(saved.isGlobalRequirement).toBe(true); diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index fac8468393..8b8d789d35 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -193,8 +193,11 @@ describe('toSavedFilter', () => { }); }); -/** The minimum a form needs to save, per filter type. */ -const byType: Record> = { +/** The minimum a form needs to save, per filter type the form supports. */ +const byType: Record< + Exclude, + Partial +> = { QUERY_EXPRESSION: { type: 'QUERY_EXPRESSION', name: 'Service', @@ -221,7 +224,7 @@ describe('minSelections', () => { (_type, filter) => { const saved = toSavedFilter(formValues({ ...filter, isRequired: true })); - expect(saved.minSelections).toBe(1); + expect(saved).toMatchObject({ minSelections: 1 }); expect(toFormValues(saved).isRequired).toBe(true); }, ); @@ -262,7 +265,7 @@ describe('isGlobalRequirement', () => { formValues({ ...filter, isRequired: true, isGlobalRequirement: true }), ); - expect(saved.isGlobalRequirement).toBe(true); + expect(saved).toMatchObject({ isGlobalRequirement: true }); expect(toFormValues(saved).isGlobalRequirement).toBe(true); }, ); @@ -307,7 +310,7 @@ describe('maxSelections', () => { formValues({ ...filter, isSingleSelect: true }), ); - expect(saved.maxSelections).toBe(1); + expect(saved).toMatchObject({ maxSelections: 1 }); expect(saved).not.toHaveProperty('isSingleSelect'); expect(toFormValues(saved).isSingleSelect).toBe(true); }, diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index 22a2dc5f7d..e103ab81e3 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -183,7 +183,8 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { values: selection ? Array.from(selection.included).map(String).sort() : [], - maxSelections: filter.maxSelections, + maxSelections: + 'maxSelections' in filter ? filter.maxSelections : undefined, }; }, ); diff --git a/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts b/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts index 5b24b62f21..f90a424e11 100644 --- a/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts +++ b/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts @@ -139,6 +139,24 @@ describe('dashboardFilterValues', () => { expect(parsed.byExpression).toEqual({ Env: included('prod') }); }); + it('carries ad hoc entries through a parse and serialize round trip', () => { + const adhoc: DashboardFilterValue = { + type: 'adhoc', + name: 'adhoc', + conditions: [{ key: 'ServiceName', operator: '=~', value: 'api.*' }], + }; + + const parsed = parseDashboardFilterValues([ + adhoc, + { type: 'variable', name: 'env', values: ['prod'] }, + ]); + + expect(serializeDashboardFilterValues(parsed)).toEqual([ + { type: 'variable', name: 'env', values: ['prod'] }, + adhoc, + ]); + }); + it('drops a sql entry it can extract nothing from, rather than carrying it', () => { // Carrying these would grow the URL without bound: a rebuild re-emits // every declared filter through `filtersToQuery`, so a filter whose diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index a85a649ddf..99945eceaa 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -1279,6 +1279,83 @@ describe('utils', () => { } }); + it('should remap ad hoc filter sources and applies-to sources from IDs to names', () => { + const sources: TSource[] = [ + { + id: 'source1', + name: 'Logs', + connection: 'connection1', + kind: SourceKind.Log, + from: { databaseName: 'db1', tableName: 'logs_table' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + { + id: 'source2', + name: 'Traces', + connection: 'connection1', + kind: SourceKind.Log, + from: { databaseName: 'db1', tableName: 'traces_table' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + ]; + + const dashboard: z.infer = { + id: 'dashboard1', + name: 'Ad hoc Dashboard', + tags: [], + tiles: [], + filters: [ + { + id: 'filter-scoped', + type: 'ADHOC', + name: 'Scoped', + sourceType: 'sql', + sources: ['source1', 'source2'], + appliesToSourceIds: ['source2', 'deleted-source-id'], + isVariableEnabled: true, + variableName: 'scoped', + }, + { + id: 'filter-unresolved', + type: 'ADHOC', + name: 'Unresolved', + sourceType: 'sql', + sources: ['source1'], + appliesToSourceIds: ['deleted-source-id'], + isVariableEnabled: true, + variableName: 'unresolved', + }, + ], + }; + + const template = convertToDashboardTemplate(dashboard, sources); + + expect(template.filters).toEqual([ + { + id: 'filter-scoped', + type: 'ADHOC', + name: 'Scoped', + sourceType: 'sql', + sources: ['Logs', 'Traces'], + appliesToSourceIds: ['Traces'], + isVariableEnabled: true, + variableName: 'scoped', + }, + { + id: 'filter-unresolved', + type: 'ADHOC', + name: 'Unresolved', + sourceType: 'sql', + sources: ['Logs'], + isVariableEnabled: true, + variableName: 'unresolved', + }, + ]); + expect(() => DashboardTemplateSchema.parse(template)).not.toThrow(); + }); + // Variable settings are dashboard-local — unlike `source` and // `appliesToSourceIds` they reference nothing in the workspace, so they need // no name↔ID remapping and must survive export verbatim. diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index e0a05c46ef..4440a21fe8 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -6,7 +6,11 @@ import { z } from 'zod'; export { default as objectHash } from 'object-hash'; -import { isQueryExpressionFilter, isStaticListFilter } from '@/filters'; +import { + isAdhocFilter, + isQueryExpressionFilter, + isStaticListFilter, +} from '@/filters'; import { isBuilderSavedChartConfig, isRawSqlSavedChartConfig } from '@/guards'; import { MacroExpansionError, MalformedMacroArgsError } from '@/macroErrors'; import { @@ -821,19 +825,20 @@ export function convertToDashboardTemplate( // A static filter references nothing in the workspace if (isStaticListFilter(filter)) return filter; - // Extract name from source or default to '' if not found - filter.source = - sources.find(source => source.id === filter.source)?.name ?? ''; - - if (isQueryExpressionFilter(filter)) { - if (filter.appliesToSourceIds?.length) { - const remapped = filter.appliesToSourceIds - .map(id => sources.find(source => source.id === id)?.name) - .filter((name): name is string => !!name && name.length > 0); - filter.appliesToSourceIds = remapped.length > 0 ? remapped : undefined; - } else { - filter.appliesToSourceIds = undefined; - } + const getSourceName = (id: string) => + sources.find(source => source.id === id)?.name ?? ''; + + if (isAdhocFilter(filter)) { + filter.sources = filter.sources.map(getSourceName); + } else { + filter.source = getSourceName(filter.source); + } + + if (isQueryExpressionFilter(filter) || isAdhocFilter(filter)) { + const remapped = (filter.appliesToSourceIds ?? []) + .map(getSourceName) + .filter(name => name.length > 0); + filter.appliesToSourceIds = remapped.length > 0 ? remapped : undefined; } return filter; }; diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index e270fa168a..33cdf604b6 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -3,6 +3,8 @@ import * as SQLParser from 'node-sql-parser'; import { escapeSqlString, replaceJsonExpressions } from '@/core/utils'; import { parse } from '@/queryParser'; import { + AdhocFilterSourceType, + AdhocFilterValue, ChartVariable, DASHBOARD_VARIABLE_NAME_MAX_LENGTH, DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, @@ -851,18 +853,20 @@ export function isFilterVariableEnabled(filter: { * Whether a filter must have at least one value selected before the tiles it * covers will load. */ -export function isFilterRequired(filter: { minSelections?: number }): boolean { - return (filter.minSelections ?? 0) > 0; +export function isFilterRequired( + filter: DashboardFilter | { minSelections?: number }, +): boolean { + return 'minSelections' in filter && (filter.minSelections ?? 0) > 0; } /** * Whether the given required filter blocks every tile on the dashboard, * rather than only the tiles that read it. */ -export function isFilterGlobalRequirement(filter: { - isGlobalRequirement?: boolean; -}): boolean { - return !!filter.isGlobalRequirement; +export function isFilterGlobalRequirement( + filter: DashboardFilter | { isGlobalRequirement?: boolean }, +): boolean { + return 'isGlobalRequirement' in filter && !!filter.isGlobalRequirement; } /** The discriminant every dashboard-filter shape carries. */ @@ -876,6 +880,21 @@ export const QUERY_EXPRESSION_FILTER_SOURCE_KINDS: SourceKind[] = [ SourceKind.Metric, ]; +/** + * The source kinds an ADHOC filter of the given source type can read keys + * from. Metric sources need a metric type to pick a table, so they are left out. + */ +export function getAdhocFilterSourceKinds( + sourceType: AdhocFilterSourceType, +): SourceKind[] { + switch (sourceType) { + case 'sql': + return [SourceKind.Log, SourceKind.Trace, SourceKind.Session]; + case 'promql': + return [SourceKind.Promql]; + } +} + /** Type guard for QUERY_EXPRESSION type filters. */ export function isQueryExpressionFilter< T extends { type: DashboardFilterKind }, @@ -897,6 +916,20 @@ export function isPrometheusLabelFilter< return filter.type === 'PROMETHEUS_LABEL'; } +/** Type guard for ADHOC type filters. */ +export function isAdhocFilter( + filter: T, +): filter is Extract { + return filter.type === 'ADHOC'; +} + +/** Type guard for the saved conditions of an ADHOC filter. */ +export function isAdhocFilterValue( + value: DashboardFilterValue, +): value is AdhocFilterValue { + return value.type === 'adhoc'; +} + /** The SQL expression associated with the filter, if any. */ export function getFilterExpression( filter: DashboardFilter, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 1d2ab24e29..51c8a272e0 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1247,10 +1247,49 @@ export const VariableFilterValueSchema = z.object({ export type VariableFilterValue = z.infer; +/** + * Operators an ad hoc filter condition can use, spelled as in PromQL so one set + * serves both source types. `LIKE` / `NOT LIKE` are offered for SQL sources only. + */ +export const AdhocFilterOperatorSchema = z.enum([ + '=', + '!=', + '=~', + '!~', + 'LIKE', + 'NOT LIKE', +]); + +export type AdhocFilterOperator = z.infer; + +/** One `key operator value` condition picked in an ad hoc filter. */ +export const AdhocFilterConditionSchema = z.object({ + /** A SQL expression (column or map access) or a PromQL label name. */ + key: z.string().min(1).max(10000), + operator: AdhocFilterOperatorSchema, + value: z.string().max(10000), +}); + +export type AdhocFilterCondition = z.infer; + +export const ADHOC_FILTER_MAX_CONDITIONS = 100; + +/** The conditions selected in an ad hoc filter, keyed by its variable name. */ +export const AdhocFilterValueSchema = z.object({ + type: z.literal('adhoc'), + name: VariableFilterValueSchema.shape.name, + conditions: z + .array(AdhocFilterConditionSchema) + .max(ADHOC_FILTER_MAX_CONDITIONS), +}); + +export type AdhocFilterValue = z.infer; + /** One entry in a dashboard's `filters=` param / `savedFilterValues`. */ export const DashboardFilterValueSchema = z.union([ FilterSchema, VariableFilterValueSchema, + AdhocFilterValueSchema, ]); export type DashboardFilterValue = z.infer; @@ -2092,6 +2131,7 @@ export const DashboardFilterType = z.enum([ 'QUERY_EXPRESSION', 'STATIC_LIST', 'PROMETHEUS_LABEL', + 'ADHOC', ]); /** Allowed variable names for dashboard filters. Alphanumeric + underscore, must start with a letter. */ @@ -2200,10 +2240,41 @@ export const PromqlLabelDashboardFilterSchema = isVariableEnabled: z.literal(true), }); +/** The kind of sources an ad hoc filter reads its keys from. */ +export const AdhocFilterSourceTypeSchema = z.enum(['sql', 'promql']); + +export type AdhocFilterSourceType = z.infer; + +/** + * A filter whose conditions (`key operator value`) are picked freely from the + * keys of its sources: columns and map keys for SQL sources, labels for PromQL + * sources. Selection limits and requirements have no meaning for a list of + * conditions, so the filter does not carry them. Strict because routes validate + * without stripping, so an omitted key would otherwise still be stored. + */ +export const AdhocDashboardFilterSchema = dashboardFilterBaseSchema + .omit({ minSelections: true, maxSelections: true, isGlobalRequirement: true }) + .extend({ + type: z.literal(DashboardFilterType.enum.ADHOC), + sourceType: AdhocFilterSourceTypeSchema, + /** IDs of the sources whose keys and values the filter offers, all of `sourceType`. */ + sources: z.array(z.string().min(1)).min(1), + /** + * Sources the conditions are broadcast to, all of `sourceType`. Undefined / + * empty means the filter's `sources`. + */ + appliesToSourceIds: z.array(z.string().min(1)).optional(), + /** Undefined / missing means ENABLED, as for QUERY_EXPRESSION filters. */ + isBroadcastEnabled: z.boolean().optional(), + isVariableEnabled: z.literal(true), + }) + .strict(); + export const DashboardFilterSchema = z.discriminatedUnion('type', [ QueryExpressionDashboardFilterSchema, StaticListDashboardFilterSchema, PromqlLabelDashboardFilterSchema, + AdhocDashboardFilterSchema, ]); export type QueryExpressionDashboardFilter = z.infer< @@ -2215,6 +2286,7 @@ export type StaticListDashboardFilter = z.infer< export type PromqlLabelDashboardFilter = z.infer< typeof PromqlLabelDashboardFilterSchema >; +export type AdhocDashboardFilter = z.infer; export type DashboardFilter = z.infer; export enum PresetDashboard { diff --git a/packages/common-utils/src/variables.ts b/packages/common-utils/src/variables.ts index 6d1aa7a34c..0c6a54d28c 100644 --- a/packages/common-utils/src/variables.ts +++ b/packages/common-utils/src/variables.ts @@ -21,6 +21,7 @@ import { ChartVariable, DASHBOARD_VARIABLE_NAME_PATTERN, DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, + DashboardFilter, DisplayType, SavedChartConfig, SearchConditionLanguage, @@ -41,10 +42,10 @@ export const VARIABLE_FORMATS = [ export type VariableFormat = (typeof VARIABLE_FORMATS)[number]; /** Whether at most one value can be selected for the filter (or its variable). */ -export function isFilterSingleSelect(filter: { - maxSelections?: number; -}): boolean { - return filter.maxSelections === 1; +export function isFilterSingleSelect( + filter: DashboardFilter | { maxSelections?: number }, +): boolean { + return 'maxSelections' in filter && filter.maxSelections === 1; } const isVariableFormat = (format: string): format is VariableFormat =>