Skip to content

Commit d323769

Browse files
Merge branch 'main' into promql-query-attribution
2 parents 16096bf + 837c683 commit d323769

16 files changed

Lines changed: 973 additions & 68 deletions

File tree

‎.changeset/proud-walls-argue.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/api": patch
3+
---
4+
5+
feat: Support series mode heatmap in API and MCP

‎.changeset/text-index-key-order.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
'@hyperdx/common-utils': patch
3+
'@hyperdx/app': patch
4+
'@hyperdx/api': patch
5+
---
6+
7+
fix: rank Map keys by frequency when a text index serves key discovery
8+
9+
The two `mergeTreeTextIndex` paths in `getMapKeys` grouped keys and applied
10+
`LIMIT maxKeys` with no `ORDER BY`, so on a column with more keys than the
11+
limit the filter sidebar showed an arbitrary subset. They now order by
12+
`sum(cardinality) DESC, key`, which keeps the most frequent keys, as the key
13+
rollup path already does with `sum(count) DESC`.

‎packages/api/openapi.json‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2225,14 +2225,14 @@
22252225
}
22262226
}
22272227
},
2228-
"HeatmapChartConfig": {
2228+
"HeatmapDistributionChartConfig": {
22292229
"type": "object",
22302230
"required": [
22312231
"displayType",
22322232
"sourceId",
22332233
"select"
22342234
],
2235-
"description": "Builder configuration for a heatmap tile. Heatmap is builder-only (no Raw SQL variant) and currently supports trace sources. The row-level filter lives at the chart-config level (where / whereLanguage), matching the HeatmapSeriesEditor in the UI.\n",
2235+
"description": "Builder configuration for a distribution-mode heatmap tile, which buckets a numeric value on the y-axis. Requires a trace source. The row-level filter lives at the chart-config level (where / whereLanguage), matching the HeatmapSeriesEditor in the UI.\n",
22362236
"properties": {
22372237
"displayType": {
22382238
"type": "string",
@@ -2242,6 +2242,14 @@
22422242
"description": "Display type discriminator. Must be \"heatmap\" for heatmap tiles.",
22432243
"example": "heatmap"
22442244
},
2245+
"heatmapMode": {
2246+
"type": "string",
2247+
"enum": [
2248+
"distribution"
2249+
],
2250+
"description": "Heatmap mode. \"distribution\" is the default when omitted.",
2251+
"example": "distribution"
2252+
},
22452253
"sourceId": {
22462254
"type": "string",
22472255
"description": "ID of the data source to query.",
@@ -2274,6 +2282,69 @@
22742282
}
22752283
}
22762284
},
2285+
"HeatmapSeriesChartConfig": {
2286+
"type": "object",
2287+
"required": [
2288+
"displayType",
2289+
"heatmapMode",
2290+
"sourceId",
2291+
"select"
2292+
],
2293+
"description": "Builder configuration for a series-mode heatmap tile, which shows one aggregated series per groupBy value as a row on the y-axis. Requires a trace, log, or metric source. Any filter lives on the select item (where / whereLanguage).\n",
2294+
"properties": {
2295+
"displayType": {
2296+
"type": "string",
2297+
"enum": [
2298+
"heatmap"
2299+
],
2300+
"description": "Display type discriminator. Must be \"heatmap\" for heatmap tiles.",
2301+
"example": "heatmap"
2302+
},
2303+
"heatmapMode": {
2304+
"type": "string",
2305+
"enum": [
2306+
"series"
2307+
],
2308+
"description": "Heatmap mode. Must be \"series\" for series heatmaps.",
2309+
"example": "series"
2310+
},
2311+
"sourceId": {
2312+
"type": "string",
2313+
"description": "ID of the data source to query.",
2314+
"example": "65f5e4a3b9e77c001a111111"
2315+
},
2316+
"select": {
2317+
"type": "array",
2318+
"minItems": 1,
2319+
"maxItems": 1,
2320+
"description": "Exactly one aggregated value used to color each cell.",
2321+
"items": {
2322+
"$ref": "#/components/schemas/SelectItem"
2323+
}
2324+
},
2325+
"groupBy": {
2326+
"type": "string",
2327+
"maxLength": 10000,
2328+
"description": "Field expression to group results by (one row per group value).",
2329+
"example": "ServiceName"
2330+
},
2331+
"numberFormat": {
2332+
"$ref": "#/components/schemas/NumberFormat",
2333+
"description": "Number formatting options for displayed values."
2334+
}
2335+
}
2336+
},
2337+
"HeatmapChartConfig": {
2338+
"description": "Heatmap tile. Heatmap is builder-only (no Raw SQL variant). Omit heatmapMode (or set it to \"distribution\") for the distribution variant; set heatmapMode to \"series\" for the series variant.\n",
2339+
"oneOf": [
2340+
{
2341+
"$ref": "#/components/schemas/HeatmapDistributionChartConfig"
2342+
},
2343+
{
2344+
"$ref": "#/components/schemas/HeatmapSeriesChartConfig"
2345+
}
2346+
]
2347+
},
22772348
"SearchChartConfig": {
22782349
"type": "object",
22792350
"required": [

‎packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,77 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => {
823823
expect(tile.config.seriesLimit).toBe(3);
824824
});
825825

826+
it('should patch a series heatmap tile without losing its mode', async () => {
827+
const logSource = await Source.create({
828+
kind: SourceKind.Log,
829+
team: ctx.team._id,
830+
from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' },
831+
timestampValueExpression: 'Timestamp',
832+
connection: ctx.connection._id,
833+
name: 'Logs',
834+
});
835+
const sourceId = logSource._id.toString();
836+
const createResult = await callTool(
837+
ctx.client!,
838+
'clickstack_save_dashboard',
839+
{
840+
name: 'Series Heatmap Patch Test',
841+
tiles: [
842+
{
843+
name: 'Original',
844+
config: {
845+
displayType: 'heatmap',
846+
heatmapMode: 'series',
847+
sourceId,
848+
select: [{ aggFn: 'count' }],
849+
groupBy: 'ServiceName',
850+
},
851+
},
852+
],
853+
},
854+
);
855+
expect(createResult.isError).toBeFalsy();
856+
const created = JSON.parse(getFirstText(createResult));
857+
const tileId = created.tiles[0].id;
858+
859+
// A valueExpression also satisfies the distribution schema, so this
860+
// guards the patch union order.
861+
const patchedConfig = {
862+
displayType: 'heatmap',
863+
heatmapMode: 'series',
864+
sourceId,
865+
select: [
866+
{
867+
aggFn: 'avg',
868+
valueExpression: 'Duration',
869+
where: 'SeverityText:error',
870+
whereLanguage: 'lucene',
871+
},
872+
],
873+
groupBy: 'SeverityText',
874+
};
875+
const patchResult = await callTool(
876+
ctx.client!,
877+
'clickstack_patch_dashboard',
878+
{
879+
dashboardId: created.id,
880+
tileId,
881+
tile: { name: 'Patched', config: patchedConfig },
882+
},
883+
);
884+
expect(patchResult.isError).toBeFalsy();
885+
886+
const getResult = await callTool(
887+
ctx.client!,
888+
'clickstack_get_dashboard_tile',
889+
{ dashboardId: created.id, tileId },
890+
);
891+
expect(getResult.isError).toBeFalsy();
892+
const tile = JSON.parse(getFirstText(getResult));
893+
expect(tile.name).toBe('Patched');
894+
expect(tile.config).toMatchObject(patchedConfig);
895+
});
896+
826897
describe('raw SQL macro warnings', () => {
827898
// Patching a tile to a macro-less raw SQL config succeeds (non-blocking)
828899
// but surfaces an advisory `warnings` array on the response.

‎packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,71 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => {
438438
expect(result.isError).toBeFalsy();
439439
});
440440

441+
it('returns time-bucketed rows for a series-mode heatmap tile', async () => {
442+
const logSource = await Source.create({
443+
kind: SourceKind.Log,
444+
team: ctx.team._id,
445+
from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE },
446+
timestampValueExpression: 'Timestamp',
447+
connection: ctx.connection._id,
448+
name: 'Series Heatmap Logs',
449+
bodyExpression: 'Body',
450+
severityTextExpression: 'SeverityText',
451+
});
452+
const now = new Date();
453+
const logs: Parameters<typeof bulkInsertLogs>[0] = [];
454+
for (const minutesAgo of [10, 5]) {
455+
for (const serviceName of ['svc-a', 'svc-b']) {
456+
logs.push({
457+
Body: `series heatmap log ${serviceName} ${minutesAgo}`,
458+
ServiceName: serviceName,
459+
SeverityText: 'INFO',
460+
Timestamp: new Date(now.getTime() - minutesAgo * 60 * 1000),
461+
});
462+
}
463+
}
464+
await bulkInsertLogs(logs);
465+
466+
const createResult = await callTool(
467+
ctx.client!,
468+
'clickstack_save_dashboard',
469+
{
470+
name: 'Series Heatmap Dashboard',
471+
tiles: [
472+
{
473+
name: 'Count by service',
474+
config: {
475+
displayType: 'heatmap',
476+
heatmapMode: 'series',
477+
sourceId: logSource._id.toString(),
478+
select: [{ aggFn: 'count' }],
479+
groupBy: 'ServiceName',
480+
},
481+
},
482+
],
483+
},
484+
);
485+
if (createResult.isError) {
486+
throw new Error(getFirstText(createResult));
487+
}
488+
const dashboard = JSON.parse(getFirstText(createResult));
489+
490+
const result = await callTool(ctx.client!, 'clickstack_query_tile', {
491+
dashboardId: dashboard.id,
492+
tileId: dashboard.tiles[0].id,
493+
startTime: new Date(now.getTime() - 15 * 60 * 1000).toISOString(),
494+
endTime: new Date(now.getTime() + 60 * 1000).toISOString(),
495+
});
496+
497+
expect(result.isError).toBeFalsy();
498+
const parsed: { result: { data: Array<Record<string, string>> } } =
499+
JSON.parse(getFirstText(result));
500+
expect(parsed.result.data).toHaveLength(4);
501+
for (const row of parsed.result.data) {
502+
expect(row).toHaveProperty('__hdx_time_bucket');
503+
}
504+
});
505+
441506
// Regression: a metric source rendered as a categorical (bar/pie) tile with a
442507
// groupBy runs the chart config through convertToCategoricalChartConfig, which
443508
// structuredClones it. `source.metricTables` is a live Mongoose subdocument

‎packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,48 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => {
525525
});
526526
});
527527

528+
it('should save and re-read a series heatmap tile on a Log source', async () => {
529+
const logSource = await Source.create({
530+
kind: SourceKind.Log,
531+
team: ctx.team._id,
532+
from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' },
533+
timestampValueExpression: 'Timestamp',
534+
connection: ctx.connection._id,
535+
name: 'Logs',
536+
});
537+
const config = {
538+
displayType: 'heatmap',
539+
heatmapMode: 'series',
540+
sourceId: logSource._id.toString(),
541+
select: [
542+
{
543+
aggFn: 'avg',
544+
valueExpression: 'Duration',
545+
where: 'SeverityText:error',
546+
whereLanguage: 'lucene',
547+
alias: 'Avg duration',
548+
},
549+
],
550+
groupBy: 'ServiceName',
551+
};
552+
553+
const result = await callTool(ctx.client!, 'clickstack_save_dashboard', {
554+
name: 'Series Heatmap Dashboard',
555+
tiles: [{ name: 'Duration by service', config }],
556+
});
557+
expect(result.isError).toBeFalsy();
558+
const saved = JSON.parse(getFirstText(result));
559+
expect(saved.tiles[0].config).toMatchObject(config);
560+
561+
const get = await callTool(ctx.client!, 'clickstack_get_dashboard', {
562+
id: saved.id,
563+
});
564+
expect(get.isError).toBeFalsy();
565+
expect(JSON.parse(getFirstText(get)).tiles[0].config).toMatchObject(
566+
config,
567+
);
568+
});
569+
528570
it('should round-trip every MCP-specific heatmap field through save, get, update, and re-get', async () => {
529571
const sourceId = ctx.traceSource._id.toString();
530572

‎packages/api/src/mcp/prompts/dashboards/content.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ Use BUILDER tiles (with sourceId) for most cases:
6262
pie Proportional breakdowns (traffic share by service, errors by type). Keep slice count under 8.
6363
bar Categorical comparisons (request counts by service, errors by endpoint). One bar per group value; not a time series (use stacked_bar for that).
6464
heatmap Distribution of a numeric value over time (latency buckets, payload size). Trace sources only. Requires non-empty valueExpression.
65+
With heatmapMode: "series": one row per groupBy value, colored by a normal select item. Trace, Log, or Metric sources.
6566
search Browse raw log/event rows (error logs, recent traces).
6667
markdown Use sparingly. The dashboard already shows its name in the title bar at the top; do NOT add a "About this dashboard" tile that repeats it. Markdown bodies render h1/h2/h3 headings at title-bar scale, so a single \`## Service Catalog\` line eats most of the tile and pushes real KPIs below the fold. Skip markdown tiles for starter dashboards. If you must add one, size it to fit the text (h: 2-3 for a line or two; h: 1 clips it), use plain prose, no \`#\`/\`##\`/\`###\` headings. Use containers/tabs for section grouping instead.
6768
@@ -88,7 +89,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t
8889
8990
1. ONE QUESTION PER DASHBOARD. A dashboard answers a single observability question well. Split if the request mixes traces, logs, and metrics into unrelated views.
9091
91-
2. ALIAS EVERY SELECT ITEM. Every entry in select MUST carry an alias. Tables, lines, stacked_bars, pies, AND number tiles. The number-tile case is the one most often missed because the rendered UI uses the tile's name (not the column alias), so the absence looks invisible; the underlying query still emits a raw-expression column name (count(), quantile(0.95)(Duration)), which breaks orderBy references, CSV export, and any downstream onClick template that references the column by name. Treat "no alias" as a save-time bug, not a style nit. Heatmap is the only exception: heatmap select items take a valueExpression and no alias.
92+
2. ALIAS EVERY SELECT ITEM. Every entry in select MUST carry an alias. Tables, lines, stacked_bars, pies, AND number tiles. The number-tile case is the one most often missed because the rendered UI uses the tile's name (not the column alias), so the absence looks invisible; the underlying query still emits a raw-expression column name (count(), quantile(0.95)(Duration)), which breaks orderBy references, CSV export, and any downstream onClick template that references the column by name. Treat "no alias" as a save-time bug, not a style nit. Distribution heatmaps are the only exception: their select items take a valueExpression and no alias.
9293
9394
Number tile, correct: select: [{ aggFn: "count", alias: "Server Requests" }]
9495
Number tile, wrong: select: [{ aggFn: "count" }]
@@ -139,7 +140,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t
139140
140141
14. SIZE TILES TO FIT THEIR CONTENT. The layout w/h are not one-size-fits-all; a tile that is too short clips its content (a table loses rows below the fold, a number tile crops its label) and one that is too wide wastes the row. Match the size to the displayType: number tiles stay small (w 6-8, h 3-4) so three or four KPIs share a row; line / stacked_bar / pie / bar want w 8-12 and h 4-6; tables and search lists want the full row (w 24) and h 6-10 so rows are not cut off; heatmaps want w 12 and h 5-6; a markdown note wants h 2-3 (never h 1, which clips the text). The per-field w/h descriptions on the tile schema carry the same per-displayType ranges; reach for them instead of leaving every tile at the 12x4 default.
141142
142-
15. FILTER A BUILDER TILE ON THE SELECT ITEM, NOT THE TILE. To scope a table / line / stacked_bar / number / pie / bar tile to a subset of rows (a service, an error status), put the filter on EACH select item's where: select: [{ aggFn: "count", where: "ServiceName:payment", whereLanguage: "lucene", alias: "..." }]. The chart editor renders that per-series where as the tile's visible "Where" box, so the user can see and edit it. Do NOT put a filter at the tile config's top level for these types: the editor does not show it, so it is ignored. For a whole-dashboard scope use a dashboard-level filter (gotcha 9). The only display types with a tile-level where are search, heatmap, and event_patterns, where the editor does render it.
143+
15. FILTER A BUILDER TILE ON THE SELECT ITEM, NOT THE TILE. To scope a table / line / stacked_bar / number / pie / bar tile to a subset of rows (a service, an error status), put the filter on EACH select item's where: select: [{ aggFn: "count", where: "ServiceName:payment", whereLanguage: "lucene", alias: "..." }]. The chart editor renders that per-series where as the tile's visible "Where" box, so the user can see and edit it. Do NOT put a filter at the tile config's top level for these types: the editor does not show it, so it is ignored. For a whole-dashboard scope use a dashboard-level filter (gotcha 9). The only display types with a tile-level where are search, distribution heatmap, and event_patterns, where the editor does render it. Series heatmaps filter on the select item like the other builder types.
143144
144145
== ADAPT, DO NOT COPY ==
145146
@@ -165,7 +166,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie
165166
- Multiple select items on number / pie / bar / heatmap tiles (each takes exactly one).
166167
- Missing level on aggFn "quantile" (must specify 0.5, 0.9, 0.95, or 0.99).
167168
- Assuming StatusCode or SeverityText values (always inspect lowCardinalityValues from clickstack_describe_source).
168-
- Heatmap on a non-Trace source (heatmap is Trace-only today).
169+
- Distribution heatmap on a non-Trace source, or series heatmap on a source that is not Trace, Log, or Metric.
169170
- Hardcoding a focus dimension into every tile's where clause (use a dashboard-level filter instead).
170171
- Putting a filter at the tile-config top level on a table / line / stacked_bar / number / pie / bar tile (ignored for these types; put the where on each select item instead, see gotcha 15).
171172
- Enabling both isBroadcastEnabled and isVariableEnabled on one filter (the picked value is then applied twice; set isBroadcastEnabled: false when you add a variable). The exception is rule 9c: a variable read only by another filter's where never reaches a tile, so that filter can keep broadcasting.

0 commit comments

Comments
 (0)