diff --git a/.changeset/proud-walls-argue.md b/.changeset/proud-walls-argue.md new file mode 100644 index 0000000000..6ad800b8f6 --- /dev/null +++ b/.changeset/proud-walls-argue.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +feat: Support series mode heatmap in API and MCP diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 0a7b87fb39..fa589d62e1 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -2225,14 +2225,14 @@ } } }, - "HeatmapChartConfig": { + "HeatmapDistributionChartConfig": { "type": "object", "required": [ "displayType", "sourceId", "select" ], - "description": "Builder configuration for a heatmap tile. Heatmap is builder-only (no Raw SQL variant) and currently supports trace sources. The row-level filter lives at the chart-config level (where / whereLanguage), matching the HeatmapSeriesEditor in the UI.\n", + "description": "Builder configuration for a distribution-mode heatmap tile, which buckets a numeric value on the y-axis. Requires a trace source. The row-level filter lives at the chart-config level (where / whereLanguage), matching the HeatmapSeriesEditor in the UI.\n", "properties": { "displayType": { "type": "string", @@ -2242,6 +2242,14 @@ "description": "Display type discriminator. Must be \"heatmap\" for heatmap tiles.", "example": "heatmap" }, + "heatmapMode": { + "type": "string", + "enum": [ + "distribution" + ], + "description": "Heatmap mode. \"distribution\" is the default when omitted.", + "example": "distribution" + }, "sourceId": { "type": "string", "description": "ID of the data source to query.", @@ -2274,6 +2282,69 @@ } } }, + "HeatmapSeriesChartConfig": { + "type": "object", + "required": [ + "displayType", + "heatmapMode", + "sourceId", + "select" + ], + "description": "Builder configuration for a series-mode heatmap tile, which shows one aggregated series per groupBy value as a row on the y-axis. Requires a trace, log, or metric source. Any filter lives on the select item (where / whereLanguage).\n", + "properties": { + "displayType": { + "type": "string", + "enum": [ + "heatmap" + ], + "description": "Display type discriminator. Must be \"heatmap\" for heatmap tiles.", + "example": "heatmap" + }, + "heatmapMode": { + "type": "string", + "enum": [ + "series" + ], + "description": "Heatmap mode. Must be \"series\" for series heatmaps.", + "example": "series" + }, + "sourceId": { + "type": "string", + "description": "ID of the data source to query.", + "example": "65f5e4a3b9e77c001a111111" + }, + "select": { + "type": "array", + "minItems": 1, + "maxItems": 1, + "description": "Exactly one aggregated value used to color each cell.", + "items": { + "$ref": "#/components/schemas/SelectItem" + } + }, + "groupBy": { + "type": "string", + "maxLength": 10000, + "description": "Field expression to group results by (one row per group value).", + "example": "ServiceName" + }, + "numberFormat": { + "$ref": "#/components/schemas/NumberFormat", + "description": "Number formatting options for displayed values." + } + } + }, + "HeatmapChartConfig": { + "description": "Heatmap tile. Heatmap is builder-only (no Raw SQL variant). Omit heatmapMode (or set it to \"distribution\") for the distribution variant; set heatmapMode to \"series\" for the series variant.\n", + "oneOf": [ + { + "$ref": "#/components/schemas/HeatmapDistributionChartConfig" + }, + { + "$ref": "#/components/schemas/HeatmapSeriesChartConfig" + } + ] + }, "SearchChartConfig": { "type": "object", "required": [ diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts index ed4f69b7a0..eef1d08387 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts @@ -823,6 +823,77 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { expect(tile.config.seriesLimit).toBe(3); }); + it('should patch a series heatmap tile without losing its mode', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Logs', + }); + const sourceId = logSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Series Heatmap Patch Test', + tiles: [ + { + name: 'Original', + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId, + select: [{ aggFn: 'count' }], + groupBy: 'ServiceName', + }, + }, + ], + }, + ); + expect(createResult.isError).toBeFalsy(); + const created = JSON.parse(getFirstText(createResult)); + const tileId = created.tiles[0].id; + + // A valueExpression also satisfies the distribution schema, so this + // guards the patch union order. + const patchedConfig = { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId, + select: [ + { + aggFn: 'avg', + valueExpression: 'Duration', + where: 'SeverityText:error', + whereLanguage: 'lucene', + }, + ], + groupBy: 'SeverityText', + }; + const patchResult = await callTool( + ctx.client!, + 'clickstack_patch_dashboard', + { + dashboardId: created.id, + tileId, + tile: { name: 'Patched', config: patchedConfig }, + }, + ); + expect(patchResult.isError).toBeFalsy(); + + const getResult = await callTool( + ctx.client!, + 'clickstack_get_dashboard_tile', + { dashboardId: created.id, tileId }, + ); + expect(getResult.isError).toBeFalsy(); + const tile = JSON.parse(getFirstText(getResult)); + expect(tile.name).toBe('Patched'); + expect(tile.config).toMatchObject(patchedConfig); + }); + describe('raw SQL macro warnings', () => { // Patching a tile to a macro-less raw SQL config succeeds (non-blocking) // but surfaces an advisory `warnings` array on the response. diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts index 414877c3c1..d2c2ab73a9 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts @@ -438,6 +438,71 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => { expect(result.isError).toBeFalsy(); }); + it('returns time-bucketed rows for a series-mode heatmap tile', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Series Heatmap Logs', + bodyExpression: 'Body', + severityTextExpression: 'SeverityText', + }); + const now = new Date(); + const logs: Parameters[0] = []; + for (const minutesAgo of [10, 5]) { + for (const serviceName of ['svc-a', 'svc-b']) { + logs.push({ + Body: `series heatmap log ${serviceName} ${minutesAgo}`, + ServiceName: serviceName, + SeverityText: 'INFO', + Timestamp: new Date(now.getTime() - minutesAgo * 60 * 1000), + }); + } + } + await bulkInsertLogs(logs); + + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Series Heatmap Dashboard', + tiles: [ + { + name: 'Count by service', + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: logSource._id.toString(), + select: [{ aggFn: 'count' }], + groupBy: 'ServiceName', + }, + }, + ], + }, + ); + if (createResult.isError) { + throw new Error(getFirstText(createResult)); + } + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: new Date(now.getTime() - 15 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const parsed: { result: { data: Array> } } = + JSON.parse(getFirstText(result)); + expect(parsed.result.data).toHaveLength(4); + for (const row of parsed.result.data) { + expect(row).toHaveProperty('__hdx_time_bucket'); + } + }); + // Regression: a metric source rendered as a categorical (bar/pie) tile with a // groupBy runs the chart config through convertToCategoricalChartConfig, which // structuredClones it. `source.metricTables` is a live Mongoose subdocument diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 08a68e0110..1eea009b2d 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -525,6 +525,48 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { }); }); + it('should save and re-read a series heatmap tile on a Log source', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: ctx.team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: ctx.connection._id, + name: 'Logs', + }); + const config = { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: logSource._id.toString(), + select: [ + { + aggFn: 'avg', + valueExpression: 'Duration', + where: 'SeverityText:error', + whereLanguage: 'lucene', + alias: 'Avg duration', + }, + ], + groupBy: 'ServiceName', + }; + + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Series Heatmap Dashboard', + tiles: [{ name: 'Duration by service', config }], + }); + expect(result.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(result)); + expect(saved.tiles[0].config).toMatchObject(config); + + const get = await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: saved.id, + }); + expect(get.isError).toBeFalsy(); + expect(JSON.parse(getFirstText(get)).tiles[0].config).toMatchObject( + config, + ); + }); + it('should round-trip every MCP-specific heatmap field through save, get, update, and re-get', async () => { const sourceId = ctx.traceSource._id.toString(); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index ea8d2248c3..11a3130762 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -62,6 +62,7 @@ Use BUILDER tiles (with sourceId) for most cases: pie Proportional breakdowns (traffic share by service, errors by type). Keep slice count under 8. bar Categorical comparisons (request counts by service, errors by endpoint). One bar per group value; not a time series (use stacked_bar for that). heatmap Distribution of a numeric value over time (latency buckets, payload size). Trace sources only. Requires non-empty valueExpression. + With heatmapMode: "series": one row per groupBy value, colored by a normal select item. Trace, Log, or Metric sources. search Browse raw log/event rows (error logs, recent traces). markdown Use sparingly. The dashboard already shows its name in the title bar at the top; do NOT add a "About this dashboard" tile that repeats it. Markdown bodies render h1/h2/h3 headings at title-bar scale, so a single \`## Service Catalog\` line eats most of the tile and pushes real KPIs below the fold. Skip markdown tiles for starter dashboards. If you must add one, size it to fit the text (h: 2-3 for a line or two; h: 1 clips it), use plain prose, no \`#\`/\`##\`/\`###\` headings. Use containers/tabs for section grouping instead. @@ -88,7 +89,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 1. ONE QUESTION PER DASHBOARD. A dashboard answers a single observability question well. Split if the request mixes traces, logs, and metrics into unrelated views. -2. ALIAS EVERY SELECT ITEM. Every entry in select MUST carry an alias. Tables, lines, stacked_bars, pies, AND number tiles. The number-tile case is the one most often missed because the rendered UI uses the tile's name (not the column alias), so the absence looks invisible; the underlying query still emits a raw-expression column name (count(), quantile(0.95)(Duration)), which breaks orderBy references, CSV export, and any downstream onClick template that references the column by name. Treat "no alias" as a save-time bug, not a style nit. Heatmap is the only exception: heatmap select items take a valueExpression and no alias. +2. ALIAS EVERY SELECT ITEM. Every entry in select MUST carry an alias. Tables, lines, stacked_bars, pies, AND number tiles. The number-tile case is the one most often missed because the rendered UI uses the tile's name (not the column alias), so the absence looks invisible; the underlying query still emits a raw-expression column name (count(), quantile(0.95)(Duration)), which breaks orderBy references, CSV export, and any downstream onClick template that references the column by name. Treat "no alias" as a save-time bug, not a style nit. Distribution heatmaps are the only exception: their select items take a valueExpression and no alias. Number tile, correct: select: [{ aggFn: "count", alias: "Server Requests" }] Number tile, wrong: select: [{ aggFn: "count" }] @@ -139,7 +140,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 14. SIZE TILES TO FIT THEIR CONTENT. The layout w/h are not one-size-fits-all; a tile that is too short clips its content (a table loses rows below the fold, a number tile crops its label) and one that is too wide wastes the row. Match the size to the displayType: number tiles stay small (w 6-8, h 3-4) so three or four KPIs share a row; line / stacked_bar / pie / bar want w 8-12 and h 4-6; tables and search lists want the full row (w 24) and h 6-10 so rows are not cut off; heatmaps want w 12 and h 5-6; a markdown note wants h 2-3 (never h 1, which clips the text). The per-field w/h descriptions on the tile schema carry the same per-displayType ranges; reach for them instead of leaving every tile at the 12x4 default. -15. FILTER A BUILDER TILE ON THE SELECT ITEM, NOT THE TILE. To scope a table / line / stacked_bar / number / pie / bar tile to a subset of rows (a service, an error status), put the filter on EACH select item's where: select: [{ aggFn: "count", where: "ServiceName:payment", whereLanguage: "lucene", alias: "..." }]. The chart editor renders that per-series where as the tile's visible "Where" box, so the user can see and edit it. Do NOT put a filter at the tile config's top level for these types: the editor does not show it, so it is ignored. For a whole-dashboard scope use a dashboard-level filter (gotcha 9). The only display types with a tile-level where are search, heatmap, and event_patterns, where the editor does render it. +15. FILTER A BUILDER TILE ON THE SELECT ITEM, NOT THE TILE. To scope a table / line / stacked_bar / number / pie / bar tile to a subset of rows (a service, an error status), put the filter on EACH select item's where: select: [{ aggFn: "count", where: "ServiceName:payment", whereLanguage: "lucene", alias: "..." }]. The chart editor renders that per-series where as the tile's visible "Where" box, so the user can see and edit it. Do NOT put a filter at the tile config's top level for these types: the editor does not show it, so it is ignored. For a whole-dashboard scope use a dashboard-level filter (gotcha 9). The only display types with a tile-level where are search, distribution heatmap, and event_patterns, where the editor does render it. Series heatmaps filter on the select item like the other builder types. == ADAPT, DO NOT COPY == @@ -165,7 +166,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Multiple select items on number / pie / bar / heatmap tiles (each takes exactly one). - Missing level on aggFn "quantile" (must specify 0.5, 0.9, 0.95, or 0.99). - Assuming StatusCode or SeverityText values (always inspect lowCardinalityValues from clickstack_describe_source). -- Heatmap on a non-Trace source (heatmap is Trace-only today). +- Distribution heatmap on a non-Trace source, or series heatmap on a source that is not Trace, Log, or Metric. - Hardcoding a focus dimension into every tile's where clause (use a dashboard-level filter instead). - Putting a filter at the tile-config top level on a table / line / stacked_bar / number / pie / bar tile (ignored for these types; put the where on each select item instead, see gotcha 15). - Enabling both isBroadcastEnabled and isVariableEnabled on one filter (the picked value is then applied twice; set isBroadcastEnabled: false when you add a variable). The exception is rule 9c: a variable read only by another filter's where never reaches a tile, so that filter can keep broadcasting. diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 28a09cb92b..5c3a211e9e 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -927,7 +927,7 @@ const mcpHeatmapSelectItemSchema = z.object({ .describe('Color scale: "log" or "linear"'), }); -// Heatmap tiles are builder-only and currently restricted to Trace sources +// Distribution heatmap tiles are builder-only and restricted to Trace sources // (see HEATMAP_DISTRIBUTION_SOURCE_KINDS in `packages/common-utils/src/guards.ts`). // The save path runs `getHeatmapTilesWithIncompatibleSources` after schema // validation to enforce that, mirroring the REST handler. @@ -935,7 +935,14 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z .literal('heatmap') - .describe('Heatmap chart, requires a Trace source'), + .describe( + 'Heatmap chart. Without `heatmapMode` it is a distribution heatmap, ' + + 'which buckets a numeric value on the y-axis and requires a Trace source', + ), + heatmapMode: z + .literal('distribution') + .optional() + .describe('Omit for a distribution heatmap'), sourceId: z .string() .describe( @@ -965,6 +972,41 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ }), }); +const mcpHeatmapSeriesTileSchema = mcpTileLayoutSchema.extend({ + config: z.object({ + ...rejectedTileWhereFields, + displayType: z + .literal('heatmap') + .describe( + 'Series heatmap: one row per groupBy value, each cell colored by the ' + + 'aggregated value over time', + ), + heatmapMode: z + .literal('series') + .describe('Must be "series" for a series heatmap'), + sourceId: z + .string() + .describe( + 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources', + ), + select: z + .array(mcpTileSelectItemSchema) + .length(1) + .describe( + 'Defines the aggregate value displayed for each heatmap series.', + ), + groupBy: z + .string() + .optional() + .describe( + 'SQL GROUP BY expression defining the series shown in the heatmap.', + ), + numberFormat: mcpNumberFormatSchema + .optional() + .describe(tileLevelNumberFormatDescription), + }), +}); + const mcpSearchTileSchema = mcpTileLayoutSchema.extend({ config: z.object({ displayType: z.literal('search').describe('Log/event search results list'), @@ -1116,6 +1158,7 @@ const mcpTileSchema = z.union([ mcpNumberTileSchema, mcpPieTileSchema, mcpCategoricalBarTileSchema, + mcpHeatmapSeriesTileSchema, mcpHeatmapTileSchema, mcpSearchTileSchema, mcpEventPatternsTileSchema, @@ -1154,6 +1197,9 @@ const mcpPatchTileSchema = z.union([ mcpPatchTileLayoutSchema.extend({ config: mcpCategoricalBarTileSchema.shape.config, }), + mcpPatchTileLayoutSchema.extend({ + config: mcpHeatmapSeriesTileSchema.shape.config, + }), mcpPatchTileLayoutSchema.extend({ config: mcpHeatmapTileSchema.shape.config, }), diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index e04de47410..4848539693 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -2,6 +2,7 @@ import { ClickHouseError, type ClickHouseSettings, } from '@clickhouse/client-common'; +import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { convertToCategoricalChartConfig, @@ -553,11 +554,17 @@ export async function runConfigTile( // collapses to one row per group. Default to "auto" so the renderer // picks a bucket, mirroring the REST charts path // (packages/api/src/routers/external-api/v2/charts.ts:289). - // Search tiles intentionally have no granularity (handled above). + // Series heatmaps bucket by time like line charts; distribution heatmaps + // compute their own buckets. Search tiles intentionally have no + // granularity (handled above). + const isSeriesHeatmap = + builderConfig.displayType === DisplayType.Heatmap && + getHeatmapMode(builderConfig) === 'series'; const granularityOverride = !isSearch && (builderConfig.displayType === DisplayType.Line || - builderConfig.displayType === DisplayType.StackedBar) + builderConfig.displayType === DisplayType.StackedBar || + isSeriesHeatmap) ? { granularity: options?.granularity ?? 'auto' } : {}; diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 04eb023ef8..5e507a733e 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -2808,6 +2808,24 @@ describe('External API v2 Dashboards - new format', () => { select: [{ valueExpression: '' }], }, }, + { + label: 'series heatmap tile with multiple select items', + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: () => traceSource._id.toString(), + select: [{ aggFn: 'count' }, { aggFn: 'count' }], + }, + }, + { + label: 'heatmap tile with an unknown mode', + config: { + displayType: 'heatmap', + heatmapMode: 'bogus', + sourceId: () => traceSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + }, + }, { label: 'heatmap tile with multiple select items', config: { @@ -2880,13 +2898,12 @@ describe('External API v2 Dashboards - new format', () => { ); }); - it('skips series-mode heatmap tiles on GET', async () => { - // Series-mode heatmaps have no external representation yet; emitting - // them as distribution tiles would corrupt them on GET -> PUT. + it('returns series-mode heatmap tiles on GET', async () => { + const tileId = new ObjectId().toString(); const dashboard = await createTestDashboard({ tiles: [ { - id: new ObjectId().toString(), + id: tileId, x: 0, y: 0, w: 6, @@ -2896,7 +2913,12 @@ describe('External API v2 Dashboards - new format', () => { displayType: DisplayType.Heatmap, source: traceSource._id.toString(), select: [ - { aggFn: 'count', aggCondition: '', valueExpression: '' }, + { + aggFn: 'count', + aggCondition: 'StatusCode:Error', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, ], groupBy: 'ServiceName', where: '', @@ -2910,7 +2932,257 @@ describe('External API v2 Dashboards - new format', () => { 'get', `${BASE_URL}/${dashboard._id}`, ).expect(200); - expect(response.body.data.tiles).toEqual([]); + expect(response.body.data.tiles).toEqual([ + { + id: tileId, + name: 'Series heatmap', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: traceSource._id.toString(), + select: [ + { + aggFn: 'count', + valueExpression: '', + where: 'StatusCode:Error', + whereLanguage: 'lucene', + }, + ], + groupBy: 'ServiceName', + }, + }, + ]); + }); + + it('round-trips a series-mode heatmap tile on a Log source', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + const seriesHeatmap: ExternalDashboardTile = { + name: 'Errors by service', + x: 0, + y: 0, + w: 12, + h: 5, + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: logSource._id.toString(), + select: [ + { + aggFn: 'count', + valueExpression: '', + where: 'SeverityText:error', + whereLanguage: 'lucene', + }, + ], + groupBy: 'ServiceName', + numberFormat: { output: 'number' }, + }, + }; + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with series heatmap', + tiles: [seriesHeatmap], + tags: [], + }) + .expect(200); + expect(omit(response.body.data.tiles[0], ['id'])).toEqual(seriesHeatmap); + + const get = await authRequest( + 'get', + `${BASE_URL}/${response.body.data.id}`, + ).expect(200); + expect(omit(get.body.data.tiles[0], ['id'])).toEqual(seriesHeatmap); + + const dashboardInDb = await Dashboard.findById( + response.body.data.id, + ).lean(); + expect(dashboardInDb!.tiles[0].config).toMatchObject({ + displayType: DisplayType.Heatmap, + heatmap: { mode: 'series' }, + groupBy: 'ServiceName', + where: '', + }); + }); + + it('accepts a series-mode heatmap tile on a Metric source', async () => { + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with metric series heatmap', + tiles: [ + { + name: 'CPU by host', + x: 0, + y: 0, + w: 12, + h: 5, + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: metricSource._id.toString(), + select: [ + { + aggFn: 'avg', + metricType: MetricsDataType.Gauge, + metricName: 'system.cpu.utilization', + }, + ], + groupBy: "ResourceAttributes['host.name']", + }, + }, + ], + tags: [], + }) + .expect(200); + + expect(response.body.data.tiles[0].config).toMatchObject({ + heatmapMode: 'series', + select: [ + { + aggFn: 'avg', + metricType: MetricsDataType.Gauge, + metricName: 'system.cpu.utilization', + }, + ], + }); + }); + + it('rejects a series-mode heatmap tile on a Session source', async () => { + const sessionSource = await Source.create({ + kind: SourceKind.Session, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'rrweb_events' }, + timestampValueExpression: 'Timestamp', + traceSourceId: traceSource._id.toString(), + connection: connection._id, + name: 'Sessions', + }); + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with session series heatmap', + tiles: [ + { + name: 'Series heatmap', + x: 0, + y: 0, + w: 12, + h: 5, + config: { + displayType: 'heatmap', + heatmapMode: 'series', + sourceId: sessionSource._id.toString(), + select: [{ aggFn: 'count' }], + }, + }, + ], + tags: [], + }) + .expect(400); + + expect(response.body.message).toContain( + 'Series heatmap tiles require a Trace, Log, or Metric source', + ); + }); + + it('re-checks the source when an existing heatmap changes mode', async () => { + const logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + const tileId = new ObjectId().toString(); + const dashboard = await createTestDashboard({ + tiles: [ + { + id: tileId, + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Series heatmap', + displayType: DisplayType.Heatmap, + source: logSource._id.toString(), + select: [ + { aggFn: 'count', aggCondition: '', valueExpression: '' }, + ], + where: '', + heatmap: { mode: 'series' }, + }, + }, + ], + }); + + const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Dashboard', + tiles: [ + { + id: tileId, + name: 'Distribution heatmap', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'heatmap', + sourceId: logSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + }, + }, + ], + tags: [], + }) + .expect(400); + + expect(response.body.message).toContain( + 'Heatmap tiles require a Trace source', + ); + }); + + it('round-trips an explicit distribution heatmap mode', async () => { + const distributionHeatmap: ExternalDashboardTile = { + name: 'Latency', + x: 0, + y: 0, + w: 12, + h: 5, + config: { + displayType: 'heatmap', + heatmapMode: 'distribution', + sourceId: traceSource._id.toString(), + select: [{ valueExpression: 'Duration' }], + where: '', + whereLanguage: 'lucene', + }, + }; + + const response = await authRequest('post', BASE_URL) + .send({ + name: 'Dashboard with distribution heatmap', + tiles: [distributionHeatmap], + tags: [], + }) + .expect(200); + + expect(omit(response.body.data.tiles[0], ['id'])).toEqual( + distributionHeatmap, + ); }); it('round-trips a heatmap tile with only required fields', async () => { diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 51546ca632..14f1e85741 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -1061,15 +1061,15 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: Scale type used to bucket values on the y-axis. * example: "log" * - * HeatmapChartConfig: + * HeatmapDistributionChartConfig: * type: object * required: * - displayType * - sourceId * - select * description: > - * Builder configuration for a heatmap tile. Heatmap is builder-only - * (no Raw SQL variant) and currently supports trace sources. The + * Builder configuration for a distribution-mode heatmap tile, which + * buckets a numeric value on the y-axis. Requires a trace source. The * row-level filter lives at the chart-config level (where / * whereLanguage), matching the HeatmapSeriesEditor in the UI. * properties: @@ -1078,6 +1078,11 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * enum: [heatmap] * description: Display type discriminator. Must be "heatmap" for heatmap tiles. * example: "heatmap" + * heatmapMode: + * type: string + * enum: [distribution] + * description: Heatmap mode. "distribution" is the default when omitted. + * example: "distribution" * sourceId: * type: string * description: ID of the data source to query. @@ -1103,6 +1108,58 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * $ref: '#/components/schemas/NumberFormat' * description: Number formatting options for displayed values. * + * HeatmapSeriesChartConfig: + * type: object + * required: + * - displayType + * - heatmapMode + * - sourceId + * - select + * description: > + * Builder configuration for a series-mode heatmap tile, which shows + * one aggregated series per groupBy value as a row on the y-axis. + * Requires a trace, log, or metric source. Any filter lives + * on the select item (where / whereLanguage). + * properties: + * displayType: + * type: string + * enum: [heatmap] + * description: Display type discriminator. Must be "heatmap" for heatmap tiles. + * example: "heatmap" + * heatmapMode: + * type: string + * enum: [series] + * description: Heatmap mode. Must be "series" for series heatmaps. + * example: "series" + * sourceId: + * type: string + * description: ID of the data source to query. + * example: "65f5e4a3b9e77c001a111111" + * select: + * type: array + * minItems: 1 + * maxItems: 1 + * description: Exactly one aggregated value used to color each cell. + * items: + * $ref: '#/components/schemas/SelectItem' + * groupBy: + * type: string + * maxLength: 10000 + * description: Field expression to group results by (one row per group value). + * example: "ServiceName" + * numberFormat: + * $ref: '#/components/schemas/NumberFormat' + * description: Number formatting options for displayed values. + * + * HeatmapChartConfig: + * description: > + * Heatmap tile. Heatmap is builder-only (no Raw SQL variant). Omit + * heatmapMode (or set it to "distribution") for the distribution + * variant; set heatmapMode to "series" for the series variant. + * oneOf: + * - $ref: '#/components/schemas/HeatmapDistributionChartConfig' + * - $ref: '#/components/schemas/HeatmapSeriesChartConfig' + * * SearchChartConfig: * type: object * required: diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index ec6890d8fe..56a894efec 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -30,6 +30,7 @@ import { DashboardContainer, DashboardContainerSchema, DisplayType, + HeatmapMode, isLogSource, isOnClickDashboardById, isOnClickSearchById, @@ -95,6 +96,23 @@ export function isRawSqlExternalTileConfig( return 'configType' in config && config.configType === 'sql'; } +type ExternalHeatmapTileConfig = Extract< + ExternalDashboardTileConfig, + { displayType: 'heatmap' } +>; + +function isSeriesHeatmapExternalConfig( + config: ExternalHeatmapTileConfig, +): config is Extract { + return config.heatmapMode === 'series'; +} + +function getExternalHeatmapMode( + config: ExternalHeatmapTileConfig, +): HeatmapMode { + return getHeatmapMode({ heatmap: { mode: config.heatmapMode } }); +} + export function isConfigTile( tile: ExternalDashboardTileWithId, ): tile is ConfigTile { @@ -459,6 +477,21 @@ export const convertToExternalTileChartConfig = ( markdown: stringValueOrDefault(config.markdown, ''), }; case DisplayType.Heatmap: { + if (getHeatmapMode(config) === 'series') { + return { + displayType: DisplayType.Heatmap, + heatmapMode: 'series', + sourceId, + select: Array.isArray(config.select) + ? config.select.slice(0, 1).map(convertToExternalSelectItem) + : [DEFAULT_SELECT_ITEM], + groupBy: stringValueOrDefault(config.groupBy, undefined), + numberFormat: config.numberFormat, + }; + } + const heatmapField = config.heatmap + ? { heatmapMode: 'distribution' as const } + : {}; // The internal heatmap schema requires `select[0]` to be a builder // item with a non-empty `valueExpression`. Legacy/corrupted Mongo // docs that lack one would otherwise produce a tile that violates @@ -487,6 +520,7 @@ export const convertToExternalTileChartConfig = ( }; return { displayType: DisplayType.Heatmap, + ...heatmapField, sourceId, select: [placeholderItem], where: stringValueOrDefault(config.where, ''), @@ -496,6 +530,7 @@ export const convertToExternalTileChartConfig = ( } return { displayType: DisplayType.Heatmap, + ...heatmapField, sourceId, select: [convertToExternalHeatmapSelectItem(item)], where: stringValueOrDefault(config.where, ''), @@ -539,20 +574,6 @@ function convertTileToExternalChart( return undefined; } - // Series-mode heatmaps have no external schema yet. Emitting them as - // distribution tiles would corrupt them on a GET -> PUT round-trip. - if ( - isBuilderSavedChartConfig(tile.config) && - tile.config.displayType === DisplayType.Heatmap && - getHeatmapMode(tile.config) === 'series' - ) { - logger.warn( - { dashboardId, tileId: tile.id }, - 'Skipping series-mode heatmap tile in external API response (not yet supported)', - ); - return undefined; - } - // Returned in case of a failure converting the saved chart config const defaultTileConfig: ExternalDashboardTileConfig = isRawSqlSavedChartConfig(tile.config) @@ -863,6 +884,20 @@ export function convertToInternalTileConfig( } satisfies BuilderSavedChartConfig; break; case 'heatmap': { + if (isSeriesHeatmapExternalConfig(externalConfig)) { + // Matches the editor's `normalizeHeatmapFields`: series heatmaps + // have no chart-level where, so it is saved empty. + internalConfig = { + ...pick(externalConfig, ['groupBy', 'numberFormat']), + displayType: DisplayType.Heatmap, + heatmap: { mode: 'series' }, + select: [convertToInternalSelectItem(externalConfig.select[0])], + source: externalConfig.sourceId, + where: '', + name, + } satisfies BuilderSavedChartConfig; + break; + } // Heatmap is builder-only and uses a single select item with // its own shape: aggFn is the literal 'heatmap' on the external // surface, mapped to the internal 'count' aggFn that the editor @@ -873,6 +908,9 @@ export function convertToInternalTileConfig( const item = externalConfig.select[0]; internalConfig = { ...pick(externalConfig, ['numberFormat']), + ...(externalConfig.heatmapMode + ? { heatmap: { mode: externalConfig.heatmapMode } } + : {}), displayType: DisplayType.Heatmap, // Match the editor's `applyHeatmapDefaults` (in // `packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx`, @@ -1065,33 +1103,34 @@ function getMissingSources( /** * Returns source IDs referenced by heatmap tiles that exist but are not - * compatible with heatmap rendering. The heatmap UI gates the source picker - * via the same `HEATMAP_DISTRIBUTION_SOURCE_KINDS` set used here (see - * `packages/common-utils/src/guards.ts` and `ChartEditorControls.tsx`), so - * UI and API gates move together. + * compatible with heatmap rendering in the tile's mode, grouped by mode. The + * heatmap UI gates the source picker via the same `getHeatmapSourceKinds` + * sets used here (see `packages/common-utils/src/guards.ts` and + * `ChartEditorControls.tsx`), so UI and API gates move together. */ function getHeatmapTilesWithIncompatibleSources( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], -): string[] { - const heatmapSourceIds = new Set(); +): Record { + const sourceById = new Map(sources.map(s => [s._id.toString(), s])); + const distribution = new Set(); + const series = new Set(); for (const tile of tiles) { if ( - isConfigTile(tile) && - !isRawSqlExternalTileConfig(tile.config) && - tile.config.displayType === 'heatmap' && - tile.config.sourceId + !isConfigTile(tile) || + isRawSqlExternalTileConfig(tile.config) || + tile.config.displayType !== 'heatmap' || + !tile.config.sourceId ) { - heatmapSourceIds.add(tile.config.sourceId); + continue; + } + const mode = getExternalHeatmapMode(tile.config); + const source = sourceById.get(tile.config.sourceId); + if (source !== undefined && !isHeatmapCompatibleSource(source, mode)) { + (mode === 'series' ? series : distribution).add(tile.config.sourceId); } } - if (heatmapSourceIds.size === 0) return []; - - const sourceById = new Map(sources.map(s => [s._id.toString(), s])); - return [...heatmapSourceIds].filter(id => { - const source = sourceById.get(id); - return source !== undefined && !isHeatmapCompatibleSource(source); - }); + return { distribution: [...distribution], series: [...series] }; } /** @@ -1201,7 +1240,7 @@ function filterChangedFormulaTiles( * without being blocked when the underlying source's `kind` was * changed after the heatmap was originally accepted. New heatmap * tiles, tiles whose displayType just changed to heatmap, and tiles - * whose `sourceId` changed all flow through the check. + * whose `sourceId` or heatmap mode changed all flow through the check. */ function filterChangedHeatmapTiles( requestTiles: ExternalDashboardTileWithId[], @@ -1224,8 +1263,8 @@ function filterChangedHeatmapTiles( return true; } const existingConfig = existing.config; - if (isRawSqlSavedChartConfig(existingConfig)) { - // Existing tile was raw-SQL; user is converting to a heatmap. + if (!isBuilderSavedChartConfig(existingConfig)) { + // Existing tile was raw-SQL or PromQL; user is converting to a heatmap. return true; } if (existingConfig.displayType !== DisplayType.Heatmap) { @@ -1233,8 +1272,11 @@ function filterChangedHeatmapTiles( return true; } // Existing tile was already a heatmap. Re-check only when the - // source changed. - return existingConfig.source?.toString() !== tile.config.sourceId; + // source or mode changed. + return ( + existingConfig.source?.toString() !== tile.config.sourceId || + getHeatmapMode(existingConfig) !== getExternalHeatmapMode(tile.config) + ); }); } @@ -1497,12 +1539,15 @@ export async function validateDashboardTiles( const heatmapTilesToCheck = existingTiles ? filterChangedHeatmapTiles(tiles, existingTiles) : tiles; - const heatmapNonTraceSources = getHeatmapTilesWithIncompatibleSources( + const heatmapIncompatibleSources = getHeatmapTilesWithIncompatibleSources( sources, heatmapTilesToCheck, ); - if (heatmapNonTraceSources.length > 0) { - return `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapNonTraceSources.join(', ')}`; + if (heatmapIncompatibleSources.distribution.length > 0) { + return `Heatmap tiles require a Trace source. The following source IDs are not Trace sources: ${heatmapIncompatibleSources.distribution.join(', ')}`; + } + if (heatmapIncompatibleSources.series.length > 0) { + return `Series heatmap tiles require a Trace, Log, or Metric source. The following source IDs are not Trace, Log, or Metric sources: ${heatmapIncompatibleSources.series.join(', ')}`; } // Formula source-kind gate. On create (no existingTiles), validate all diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 78622d9c1a..c3dd169e8d 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -512,13 +512,14 @@ export type ExternalDashboardHeatmapSelectItem = z.infer< typeof externalDashboardHeatmapSelectItemSchema >; -// Heatmap exposes the row-level filter at the chart-config level (matching -// the editor: HeatmapSeriesEditor renders a single SearchWhereInput bound -// to the top-level `where` / `whereLanguage`). There is no groupBy in the -// heatmap UI (HeatmapSeriesEditor doesn't render one), so it is omitted -// from the schema. +// Distribution-mode heatmap. It exposes the row-level filter at the +// chart-config level (matching the editor: HeatmapSeriesEditor renders a +// single SearchWhereInput bound to the top-level `where` / `whereLanguage`). +// Distribution heatmaps have no groupBy, so it is omitted from the schema. +// `heatmapMode` is optional: an absent mode is 'distribution'. const externalDashboardHeatmapChartConfigSchema = z.object({ displayType: z.literal('heatmap'), + heatmapMode: z.literal('distribution').optional(), sourceId: objectIdSchema, select: z.array(externalDashboardHeatmapSelectItemSchema).length(1), where: z.string().max(10000).optional().default(''), @@ -530,6 +531,15 @@ const externalDashboardHeatmapChartConfigSchema = z.object({ numberFormat: NumberFormatSchema.optional(), }); +const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ + displayType: z.literal('heatmap'), + heatmapMode: z.literal('series'), + sourceId: objectIdSchema, + select: z.array(externalDashboardSelectItemSchema).length(1), + groupBy: z.string().max(10000).optional(), + numberFormat: NumberFormatSchema.optional(), +}); + const externalDashboardSearchChartConfigSchema = z.object({ displayType: z.literal('search'), sourceId: objectIdSchema, @@ -578,9 +588,25 @@ const externalDashboardBuilderTileConfigSchema = z.discriminatedUnion( ], ); -type ExternalDashboardBuilderTileConfig = z.infer< - typeof externalDashboardBuilderTileConfigSchema ->; +type ExternalDashboardBuilderTileConfig = + | z.infer + | z.infer; + +/** + * Both heatmap modes share `displayType: 'heatmap'`, which a discriminated + * union cannot hold twice, so series-mode heatmaps are routed to their own + * schema. Any other `heatmapMode` falls through to the distribution schema + * and fails its literal. + */ +const getExternalBuilderTileConfigSchema = (data: unknown) => + data !== null && + typeof data === 'object' && + 'displayType' in data && + data.displayType === 'heatmap' && + 'heatmapMode' in data && + data.heatmapMode === 'series' + ? externalDashboardHeatmapSeriesChartConfigSchema + : externalDashboardBuilderTileConfigSchema; const externalDashboardRawSqlTileConfigSchema = z.discriminatedUnion( 'displayType', @@ -707,7 +733,7 @@ const externalDashboardTileConfigSchema = z // than a generic union failure. const schema = isRawSqlRoutedConfig(data) ? externalDashboardRawSqlTileConfigSchema - : externalDashboardBuilderTileConfigSchema; + : getExternalBuilderTileConfigSchema(data); const result = schema.safeParse(data); if (!result.success) { @@ -728,7 +754,7 @@ const externalDashboardTileConfigSchema = z // so this is guaranteed to succeed. return isRawSqlRoutedConfig(data) ? externalDashboardRawSqlTileConfigSchema.parse(data) - : externalDashboardBuilderTileConfigSchema.parse(data); + : getExternalBuilderTileConfigSchema(data).parse(data); }); export type ExternalDashboardTileConfig = z.infer<