From 7387a12054afc6dae129b8eba2a78dd7a1c8755a Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Tue, 20 Oct 2026 21:23:37 +1100 Subject: [PATCH 1/3] feat: distribution heatmaps for the MCP server, hdx and the Claude plugin Agents can now see how a value such as span latency is distributed over time. The new clickstack_heatmap MCP tool returns counts per time bucket and value range, and clickstack_query_tile answers saved distribution heatmap tiles instead of failing. hdx draws distribution heatmap tiles, and the Claude plugin draws both results. The bounds and widthBucket queries the web app ran in the browser move to common-utils, so the web, API and CLI share one copy. Caller-supplied granularity is validated, and a size that would flood the grid is coarsened with a hint, so a value like "1 week" can no longer stall bucket generation. --- .changeset/claude-charts-plugin.md | 4 +- .changeset/cli-heatmap-tiles.md | 5 + .changeset/mcp-distribution-heatmap.md | 10 + MCP.md | 1 + .../queryTilesSummarizeRows.test.ts | 15 ++ .../src/mcp/__tests__/heatmapTool.int.test.ts | 216 ++++++++++++++++ .../src/mcp/tools/dashboards/queryTiles.ts | 17 ++ .../api/src/mcp/tools/query/builderCatalog.ts | 1 + packages/api/src/mcp/tools/query/heatmap.ts | 95 +++++++ .../api/src/mcp/tools/query/heatmapResult.ts | 85 +++++++ packages/api/src/mcp/tools/query/helpers.ts | 49 ++++ packages/api/src/mcp/tools/query/index.ts | 2 + packages/app/package.json | 2 +- packages/app/src/ChartUtils.tsx | 110 +-------- .../DBHeatmapChart/heatmapBounds.ts | 29 +-- .../components/DBHeatmapChart/heatmapGrid.ts | 126 +--------- .../DBHeatmapChart/heatmapQueries.ts | 232 ++---------------- packages/claude-plugin/README.md | 8 +- .../src/__tests__/toTileResult.spec.ts | 50 ++++ packages/claude-plugin/src/renderChart.ts | 12 +- packages/claude-plugin/src/toTileResult.ts | 62 ++++- packages/claude-plugin/test/render.test.tsx | 47 ++++ packages/cli/AGENTS.md | 6 +- packages/cli/src/__tests__/termchart.test.ts | 61 +++++ packages/cli/src/__tests__/tileQuery.test.ts | 82 +++++++ packages/cli/src/__tests__/tileRender.test.ts | 34 +++ packages/cli/src/shared/chartData.ts | 15 ++ packages/cli/src/shared/tileConfig.ts | 1 + packages/cli/src/shared/tileQuery.ts | 43 ++++ packages/cli/src/shared/tileRender.ts | 16 ++ packages/cli/src/termchart/heatmap.ts | 106 ++++++++ packages/cli/src/termchart/index.ts | 2 + packages/cli/src/termchart/types.ts | 10 + packages/common-utils/package.json | 2 +- .../__tests__/distributionHeatmap.test.ts | 122 +++++++++ .../src/core/distributionHeatmap.ts | 115 +++++++++ packages/common-utils/src/core/heatmap.ts | 207 +++++++++++++++- .../common-utils/src/core/heatmapQuery.ts | 225 +++++++++++++++++ packages/common-utils/src/core/utils.ts | 112 +++++++++ 39 files changed, 1858 insertions(+), 479 deletions(-) create mode 100644 .changeset/cli-heatmap-tiles.md create mode 100644 .changeset/mcp-distribution-heatmap.md create mode 100644 packages/api/src/mcp/__tests__/heatmapTool.int.test.ts create mode 100644 packages/api/src/mcp/tools/query/heatmap.ts create mode 100644 packages/api/src/mcp/tools/query/heatmapResult.ts create mode 100644 packages/cli/src/__tests__/tileQuery.test.ts create mode 100644 packages/cli/src/termchart/heatmap.ts create mode 100644 packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts create mode 100644 packages/common-utils/src/core/distributionHeatmap.ts create mode 100644 packages/common-utils/src/core/heatmapQuery.ts diff --git a/.changeset/claude-charts-plugin.md b/.changeset/claude-charts-plugin.md index e0744e9e6a..a4d28a09ab 100644 --- a/.changeset/claude-charts-plugin.md +++ b/.changeset/claude-charts-plugin.md @@ -3,6 +3,6 @@ --- feat: add a Claude Code plugin that connects the ClickStack MCP server and draws -`clickstack_timeseries` and `clickstack_table` results as terminal charts, using -the CLI's renderers. Install with +`clickstack_timeseries`, `clickstack_table` and `clickstack_heatmap` results as +terminal charts, using the CLI's renderers. Install with `/plugin install hyperdx-charts --marketplace hyperdxio/hyperdx`. diff --git a/.changeset/cli-heatmap-tiles.md b/.changeset/cli-heatmap-tiles.md new file mode 100644 index 0000000000..f2d83ec9c3 --- /dev/null +++ b/.changeset/cli-heatmap-tiles.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/cli': patch +--- + +feat: `hdx` draws distribution heatmap tiles as shaded terminal charts. diff --git a/.changeset/mcp-distribution-heatmap.md b/.changeset/mcp-distribution-heatmap.md new file mode 100644 index 0000000000..a1ad6130d0 --- /dev/null +++ b/.changeset/mcp-distribution-heatmap.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': patch +'@hyperdx/common-utils': patch +--- + +feat: add a `clickstack_heatmap` MCP tool that returns how a value such as span +latency is distributed over time, as counts per time bucket and value range. +`clickstack_query_tile` now returns the same grid for distribution heatmap +tiles instead of failing. The distribution heatmap queries the web app runs +moved to common-utils so the API and CLI share them. diff --git a/MCP.md b/MCP.md index 2065874d08..1c9f1aa18f 100644 --- a/MCP.md +++ b/MCP.md @@ -136,6 +136,7 @@ with: | `clickstack_describe_metric` | Per-metric drill-down: kind(s), unit, description, attribute keys per map column, and sampled values | | `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart (works on log, trace, and metric sources) | | `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart (works on log, trace, and metric sources) | +| `clickstack_heatmap` | Show how a value such as span latency is distributed over time (trace sources) | | `clickstack_search` | Browse individual log, event, or trace rows | | `clickstack_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | | `clickstack_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts index 5708f699f9..0140dcdaaa 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts @@ -103,4 +103,19 @@ describe('withDeadline', () => { await expect(withDeadline(work, Date.now() + 1000)).resolves.toBe('done'); expect(captured?.aborted).toBe(true); }); + + it('summarizes a distribution heatmap by its time buckets', () => { + const counts = [ + [0, 2, 0], + [0, 0, 0], + ]; + expect(summarizeRows(JSON.stringify({ heatmap: { counts } }))).toEqual({ + hasData: true, + rowCount: 2, + }); + expect(summarizeRows(JSON.stringify({ heatmap: null }))).toEqual({ + hasData: false, + rowCount: 0, + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts new file mode 100644 index 0000000000..16a580f625 --- /dev/null +++ b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts @@ -0,0 +1,216 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { ObjectId } from 'mongodb'; + +import * as config from '@/config'; +import { + bulkInsertData, + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import { McpContext } from '@/mcp/tools/types'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +type HeatmapBody = { + heatmap: { + scale: string; + numberFormat?: { output: string; factor: number }; + times: string[]; + valueEdges: number[]; + counts: number[][]; + } | null; + hints: string[]; +}; + +describe('MCP Heatmap Tool', () => { + const server = getServer(); + let client: Client; + let traceSourceId: string; + let logSourceId: string; + // otel_traces is not truncated between tests, so each test filters to its own spans. + let where: string; + const now = Date.now(); + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const { team, user } = await getLoggedInAgent(server); + const connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_TRACES_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + }); + const logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + traceSourceId = traceSource._id.toString(); + logSourceId = logSource._id.toString(); + + const service = `heatmap-${new ObjectId().toString()}`; + where = `ServiceName:"${service}"`; + + // A fast mode (~1ms) and a slow tail (~1s), so the heatmap is bimodal. + const durations = [ + ...Array.from({ length: 50 }, (_, i) => 1e6 + i * 1e3), + ...Array.from({ length: 5 }, (_, i) => 1e9 + i * 1e6), + ]; + await bulkInsertData( + `${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE}`, + durations.map((Duration, i) => ({ + Timestamp: new Date(now - (i % 5) * 60_000 - 30_000), + TraceId: `trace-${i}`, + SpanId: `span-${i}`, + ParentSpanId: '', + SpanName: 'GET /api', + SpanKind: 'Server', + ServiceName: service, + Duration, + StatusCode: 'Ok', + StatusMessage: '', + SpanAttributes: {}, + ResourceAttributes: {}, + })), + ); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + it('returns counts per time bucket and value range', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + where, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + granularity: '1 minute', + }); + + expect(result.isError).toBeFalsy(); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + const heatmap = body.heatmap!; + expect(heatmap.scale).toBe('log'); + expect(heatmap.numberFormat).toEqual({ output: 'duration', factor: 1e-9 }); + expect(heatmap.counts).toHaveLength(heatmap.times.length); + // One count per bucket plus an under- and overflow count. + for (const column of heatmap.counts) { + expect(column).toHaveLength(heatmap.valueEdges.length + 1); + } + expect(heatmap.counts.flat().reduce((a, b) => a + b, 0)).toBe(55); + + // The slow tail lands in a higher value range than the fast mode. + const rowTotals = heatmap.counts.reduce((totals, column) => + totals.map((total, r) => total + (column.at(r) ?? 0)), + ); + const populated = rowTotals.flatMap((n, r) => (n > 0 ? [r] : [])); + // Row r counts [valueEdges[r - 1], valueEdges[r]). + expect(heatmap.valueEdges[populated.at(-1)! - 1]).toBeGreaterThan(1e8); + expect(heatmap.valueEdges[populated[0]]).toBeLessThan(1e7); + }); + + it('rejects a granularity the time buckets cannot step by', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + granularity: '1 week', + }); + expect(result.isError).toBe(true); + }); + + it('coarsens a granularity that would flood the grid, and says so', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + where, + startTime: new Date(now - 24 * 60 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + granularity: '1 second', + }); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + expect(body.heatmap!.times.length).toBeLessThanOrEqual(48 * 4); + expect(body.hints.join(' ')).toContain('Granularity "1 second" would make'); + }); + + it('answers clickstack_query_tile for a saved distribution heatmap tile', async () => { + const saved = await callTool(client, 'clickstack_save_dashboard', { + name: 'Latency', + tiles: [ + { + name: 'Span latency', + config: { + displayType: 'heatmap', + sourceId: traceSourceId, + select: [{ valueExpression: 'Duration' }], + where, + }, + }, + ], + }); + expect(saved.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(saved)); + + const result = await callTool(client, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + }); + expect(result.isError).toBeFalsy(); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + expect(body.heatmap!.counts.flat().reduce((a, b) => a + b, 0)).toBe(55); + }); + + it('rejects a non-trace source', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: logSourceId, + valueExpression: 'Duration', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('need a trace source'); + }); +}); diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 443076db76..d18a6b74ab 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -127,6 +127,23 @@ export function summarizeRows(text: string): { } catch { return {}; } + if (parsed != null && typeof parsed === 'object' && 'heatmap' in parsed) { + // A distribution heatmap: one row per time bucket, null when empty. + const { heatmap } = parsed; + const counts: unknown[] = + heatmap != null && + typeof heatmap === 'object' && + 'counts' in heatmap && + Array.isArray(heatmap.counts) + ? heatmap.counts + : []; + return { + hasData: counts.some( + column => Array.isArray(column) && column.some(n => n > 0), + ), + rowCount: counts.length, + }; + } if (parsed == null || typeof parsed !== 'object' || !('result' in parsed)) { return {}; } diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts index c4b1a77a91..6988b9d0b3 100644 --- a/packages/api/src/mcp/tools/query/builderCatalog.ts +++ b/packages/api/src/mcp/tools/query/builderCatalog.ts @@ -8,6 +8,7 @@ export const BUILDER_TOOLS_LIST = [ ' • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns', ' • clickstack_timeseries — trends / metrics over time', + ' • clickstack_heatmap — how a value (e.g. latency) is distributed over time', ' • clickstack_search — browsing individual log/trace rows', ' • clickstack_event_patterns — recurring log/event pattern discovery', ' • clickstack_event_deltas — attributes that differ between two row groups', diff --git a/packages/api/src/mcp/tools/query/heatmap.ts b/packages/api/src/mcp/tools/query/heatmap.ts new file mode 100644 index 0000000000..4978694bba --- /dev/null +++ b/packages/api/src/mcp/tools/query/heatmap.ts @@ -0,0 +1,95 @@ +import { SQLIntervalSchema } from '@hyperdx/common-utils/dist/types'; +import { z } from 'zod'; + +import type { ToolRegistrar } from '@/mcp/tools/types'; +import { mcpUserError } from '@/mcp/utils/errors'; + +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; +import { + endTimeSchema, + sourceIdSchema, + startTimeSchema, + whereLanguageSchema, + whereSchema, +} from './schemas'; + +const heatmapSchema = z.object({ + sourceId: sourceIdSchema.describe( + 'Trace source ID. Distribution heatmaps need a trace source; call clickstack_list_sources to find one.', + ), + valueExpression: z + .string() + .min(1) + .describe( + 'Numeric column or expression to bucket, e.g. "Duration" for span latency. ' + + 'Top-level columns are PascalCase; map attributes use bracket syntax.', + ), + countExpression: z + .string() + .optional() + .describe('What each cell counts. Default: count() (one per span).'), + scale: z + .enum(['log', 'linear']) + .optional() + .default('log') + .describe( + 'Value axis scale. "log" (default) suits latency, which spans orders of magnitude.', + ), + where: whereSchema.describe('Row filter, e.g. restrict to one service.'), + whereLanguage: whereLanguageSchema, + granularity: SQLIntervalSchema.optional().describe( + 'Time bucket size, " " where unit is second, minute, hour or day ' + + '(e.g. "5 minute"). Omit to pick one for ~48 buckets; a size that would make ' + + 'far more buckets is coarsened, with a hint.', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +export function registerHeatmap({ context, registerTool }: ToolRegistrar) { + const { teamId } = context; + + registerTool( + 'clickstack_heatmap', + { + title: 'Distribution Heatmap', + annotations: { readOnlyHint: true }, + description: + 'Show how a numeric value is distributed over time: counts of events per ' + + 'time bucket and value range, as on a latency heatmap. Use this to see ' + + 'bimodal latency, a slow tail appearing, or a shift in the whole ' + + 'distribution, which a single percentile line hides. ' + + 'Requires a trace source. Returns counts[t][r] with the time buckets and ' + + 'value-range edges they belong to.', + inputSchema: heatmapSchema, + }, + async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return mcpUserError(timeRange.error); + } + + const tile = buildTile('MCP Heatmap', 12, 6, { + displayType: 'heatmap', + sourceId: input.sourceId, + select: [ + { + valueExpression: input.valueExpression, + countExpression: input.countExpression, + heatmapScaleType: input.scale, + }, + ], + where: input.where, + whereLanguage: input.whereLanguage, + }); + + return runConfigTile( + teamId.toString(), + tile, + timeRange.startDate, + timeRange.endDate, + { granularity: input.granularity }, + ); + }, + ); +} diff --git a/packages/api/src/mcp/tools/query/heatmapResult.ts b/packages/api/src/mcp/tools/query/heatmapResult.ts new file mode 100644 index 0000000000..7901c15004 --- /dev/null +++ b/packages/api/src/mcp/tools/query/heatmapResult.ts @@ -0,0 +1,85 @@ +import type { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; +import { + heatmapCountColumns, + type HeatmapScaleType, +} from '@hyperdx/common-utils/dist/core/heatmap'; +import type { + BuilderSavedChartConfig, + NumberFormat, + TSource, +} from '@hyperdx/common-utils/dist/types'; + +import type { ToolResult } from '@/mcp/tools/types'; + +// Sized for an agent's context rather than a screen: ~48 x 22 counts. +export const MCP_HEATMAP_TIME_BUCKETS = 48; +export const MCP_HEATMAP_VALUE_BUCKETS = 20; + +function readingHint(scale: HeatmapScaleType) { + return ( + 'valueEdges are in the units of valueExpression (when numberFormat.factor is ' + + 'present, value x factor is seconds). counts[t] has one entry per range: ' + + 'counts[t][0] counts values below valueEdges[0], counts[t][r] counts values in ' + + '[valueEdges[r - 1], valueEdges[r]), and the last entry counts values at or ' + + 'above the last edge.' + + (scale === 'log' + ? ' On this log scale values below valueEdges[0] are counted in counts[t][1].' + : '') + ); +} + +/** A trace duration column formats as a duration unless the tile says otherwise. */ +export function heatmapNumberFormat( + config: Pick, + source: TSource, + valueExpression: string, +): NumberFormat | undefined { + if (config.numberFormat) return config.numberFormat; + if ( + 'durationExpression' in source && + valueExpression === source.durationExpression + ) { + return { + output: 'duration', + factor: Math.pow(10, -(source.durationPrecision ?? 9)), + }; + } + return undefined; +} + +/** A distribution heatmap as compact JSON an agent can read range by range. */ +export function formatHeatmapResult( + { + heatmap, + granularity, + granularityNote, + }: Awaited>, + { + valueExpression, + numberFormat, + }: { valueExpression: string; numberFormat?: NumberFormat }, +): ToolResult { + const notes = granularityNote ? [granularityNote] : []; + const body = heatmap + ? { + heatmap: { + valueExpression, + scale: heatmap.scaleType, + granularity, + ...(numberFormat ? { numberFormat } : {}), + times: heatmap.grid.times.map(t => new Date(t).toISOString()), + valueEdges: heatmap.bucketEdges.map(e => Number(e.toPrecision(6))), + counts: heatmapCountColumns(heatmap.grid), + }, + hints: [readingHint(heatmap.scaleType), ...notes], + } + : { + heatmap: null, + hints: [ + `Not enough distinct values of ${valueExpression} in the time range to bucket. ` + + 'Widen startTime/endTime or check the value expression.', + ...notes, + ], + }; + return { content: [{ type: 'text', text: JSON.stringify(body) }] }; +} diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 028164edbe..a9b6761f05 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -2,6 +2,7 @@ import { ClickHouseError, type ClickHouseSettings, } from '@clickhouse/client-common'; +import { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { @@ -12,6 +13,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderSavedChartConfig, + isHeatmapCompatibleSource, isPromqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { UnknownVariableError } from '@hyperdx/common-utils/dist/macroErrors'; @@ -47,6 +49,12 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +import { + formatHeatmapResult, + heatmapNumberFormat, + MCP_HEATMAP_TIME_BUCKETS, + MCP_HEATMAP_VALUE_BUCKETS, +} from './heatmapResult'; import { runEventPatterns } from './runEventPatterns'; // ─── Source body expression helpers ────────────────────────────────────────── @@ -596,6 +604,47 @@ export async function runConfigTile( variables: options?.variables, } satisfies ChartConfigWithDateRange; + // Distribution heatmaps bucket in two queries (value bounds, then + // widthBucket counts) that the generic renderer can't express. + if ( + builderConfig.displayType === DisplayType.Heatmap && + getHeatmapMode(builderConfig) === 'distribution' + ) { + if (!isHeatmapCompatibleSource(source)) { + return mcpUserError( + `Distribution heatmaps need a trace source; "${source.name}" is a ${source.kind} source.`, + ); + } + const valueExpression = Array.isArray(builderConfig.select) + ? (builderConfig.select[0]?.valueExpression ?? '') + : ''; + try { + const result = await queryDistributionHeatmap({ + client: clickhouseClient, + metadata: getMetadata(clickhouseClient), + config: chartConfig, + granularity: options?.granularity ?? builderConfig.granularity, + targetTimeBuckets: MCP_HEATMAP_TIME_BUCKETS, + nBuckets: MCP_HEATMAP_VALUE_BUCKETS, + querySettings: source.querySettings, + opts: { + clickhouse_settings: MCP_CLICKHOUSE_SETTINGS, + abort_signal: options?.abortSignal, + }, + }); + return formatHeatmapResult(result, { + valueExpression, + numberFormat: heatmapNumberFormat( + builderConfig, + source, + valueExpression, + ), + }); + } catch (e) { + return clickHouseErrorResult(e); + } + } + // Apply seriesLimit as LIMIT to categorical charts (pie/bar) const isCategorical = builderConfig.displayType === DisplayType.Pie || diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index 700a8a24ca..6af785b4d0 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -3,6 +3,7 @@ import type { ToolDefinition, ToolRegistrar } from '@/mcp/tools/types'; import { registerEmergingSignals } from './emergingSignals'; import { registerEventDeltas } from './eventDeltas'; import { registerEventPatterns } from './eventPatterns'; +import { registerHeatmap } from './heatmap'; import { registerSearch } from './search'; import { registerSql } from './sql'; import { registerTable } from './table'; @@ -11,6 +12,7 @@ import { registerTimeseries } from './timeseries'; const queryTools: ToolDefinition = (registrar: ToolRegistrar) => { registerTimeseries(registrar); registerTable(registrar); + registerHeatmap(registrar); registerSearch(registrar); registerEventPatterns(registrar); registerEventDeltas(registrar); diff --git a/packages/app/package.json b/packages/app/package.json index c6fc18f131..f9928aea73 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -14,7 +14,7 @@ "build:clickhouse": "NEXT_PUBLIC_THEME=clickstack NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_PUBLIC_CLICKHOUSE_BUILD=true next build --webpack && node scripts/prepare-clickhouse-build-export.js", "run:clickhouse": "test -d out && npx rimraf tmp && mkdir tmp && cp -r out tmp/clickstack && echo 'visit http://localhost:3000/clickstack to start' && npx serve tmp -l 3000 || echo 'run build:clickhouse first'", "start": "next start", - "lint": "npx eslint . --ext .ts,.tsx --max-warnings 568", + "lint": "npx eslint . --ext .ts,.tsx --max-warnings 567", "lint:fix": "npx eslint . --ext .ts,.tsx --fix", "lint:styles": "stylelint **/*/*.{css,scss}", "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:styles --quiet", diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index c2b72b654b..5b8fdd9e06 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -26,6 +26,7 @@ import { getPercentileAggFnLevel, Granularity, hasPositiveSeriesLimit, + isAggregateFunction, TIME_SERIES_DISPLAY_TYPE_BY_NAME, } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; @@ -391,114 +392,7 @@ export function timeBucketByGranularity( return buckets; } -export const isAggregateFunction = (value: string) => { - const fns = [ - // Basic aggregates - 'count', - 'countIf', - 'countDistinct', - 'sum', - 'sumIf', - 'avg', - 'avgIf', - 'min', - 'max', - 'any', - 'anyLast', - 'groupArray', - 'groupArrayInsertAt', - 'groupArrayMovingAvg', - 'groupArraySample', - 'groupUniqArray', - 'groupUniqArrayIf', - 'groupArrayIntersect', - 'groupArrayIntersectIf', - 'groupArrayReduce', - 'groupBitmap', - 'groupBitmapIf', - 'groupBitmapOr', - 'groupBitmapXor', - - // Quantiles - 'quantile', - 'quantileIf', - 'quantileExact', - 'quantileExactWeighted', - 'quantileTiming', - 'quantileTimingWeighted', - 'quantileTDigest', - 'quantileTDigestWeighted', - 'quantileBFloat16', - 'quantileBFloat16Weighted', - 'quantiles', - 'median', - 'medianExact', - 'medianTDigest', - 'medianBFloat16', - - // Statistical functions - 'stddevPop', - 'stddevPopIf', - 'stddevSamp', - 'stddevSampIf', - 'varPop', - 'varPopIf', - 'varSamp', - 'varSampIf', - 'covarPop', - 'covarSamp', - 'corr', - - // Combinators - 'uniq', - 'uniqExact', - 'uniqCombined', - 'uniqCombined64', - 'uniqHLL12', - 'uniqTheta', - - // Bit operations - 'groupBitAnd', - 'groupBitOr', - 'groupBitXor', - - // Map and tuple - 'groupArrayMap', - 'groupArrayTuple', - 'groupArraySorted', - 'topK', - 'topKIf', - 'topKWeighted', - - // Aggregate combinators - 'argMin', - 'argMax', - 'minMap', - 'maxMap', - - // Specialized aggregates - 'runningDifference', - 'retention', - 'sequenceCount', - 'sequenceMatch', - 'histogram', - 'simpleLinearRegression', - 'stochasticLinearRegression', - 'categoricalInformationValue', - 'sumMap', - 'sumMapFiltered', - 'sumWithOverflow', - 'entropy', - 'skewPop', - 'skewSamp', - 'kurtPop', - 'kurtSamp', - ]; - - // Make case-insensitive since ClickHouse function names are case-insensitive - const lowerValue = value.toLowerCase(); - return fns.some(fn => lowerValue.includes(fn.toLowerCase() + '(')); -}; +export { isAggregateFunction }; export const INTEGER_NUMBER_FORMAT: NumberFormat = { factor: 1, diff --git a/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts b/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts index 5d1db49594..cab9695854 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts @@ -1,27 +1,6 @@ -import type { HeatmapScaleType } from './heatmapGrid'; +export { + computeEffectiveMin, + heatmapLowQuantile, +} from '@hyperdx/common-utils/dist/core/heatmap'; export const HEATMAP_N_BUCKETS = 80; - -/** - * Quantile used for the y-axis lower bound. A quantile (rather than min) - * keeps near-zero outliers from stretching the log axis. The upper bound - * uses the actual max so that rare latency spikes stay visible; log scale - * already compresses wide ranges. - */ -export function heatmapLowQuantile(scaleType: HeatmapScaleType) { - return scaleType === 'log' ? 0.01 : 0.001; -} - -/** - * Lower bound actually used for bucketing. Log scale needs a positive - * minimum (log(0) is undefined), and the range is capped to ~4 orders of - * magnitude so the axis isn't dominated by a long empty tail of near-zero - * outliers. - */ -export function computeEffectiveMin( - min: number, - max: number, - scaleType: HeatmapScaleType, -) { - return scaleType === 'log' ? Math.max(min, max * 1e-4 || 1e-4) : min; -} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts index c1079471bd..571ac4ca0b 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts @@ -1,31 +1,16 @@ -import { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; - -export type HeatmapScaleType = 'log' | 'linear'; - -export type HeatmapYAxis = - | { - type: 'numeric'; - scale: HeatmapScaleType; - /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ - edges: number[]; - } - /** - * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per - * row; the top bound may be `Infinity` (a `+Inf` bucket). - */ - | { type: 'buckets'; bounds: number[] } - | { type: 'series'; labels: string[] }; - -/** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ -export type HeatmapGrid = { - /** Column centers, ms ascending. */ - times: number[]; - /** Column width, ms. */ - stepMs: number; - yAxis: HeatmapYAxis; - /** Row-major: `cells[timeIdx * rowCount + rowIdx]`. */ - cells: number[]; -}; +import { + type HeatmapGrid, + heatmapRowCount, + type HeatmapYAxis, +} from '@hyperdx/common-utils/dist/core/heatmap'; + +export { + gridFromBucketRows, + type HeatmapGrid, + heatmapRowCount, + type HeatmapScaleType, + type HeatmapYAxis, +} from '@hyperdx/common-utils/dist/core/heatmap'; /** * Numeric- and bucket-axis cells are counts, empty at 0. Series-axis cells @@ -48,17 +33,6 @@ export const EMPTY_HEATMAP_GRID: HeatmapGrid = { cells: [], }; -export function heatmapRowCount(yAxis: HeatmapYAxis) { - switch (yAxis.type) { - case 'series': - return yAxis.labels.length; - case 'buckets': - return yAxis.bounds.length; - case 'numeric': - return Math.max(0, yAxis.edges.length - 1); - } -} - /** * Lower and upper bound of each row in plot space. Series and histogram * bucket rows are unit-height bands. The uPlot y scale is linear, so log-scale @@ -121,80 +95,6 @@ export function gridToPlotData(grid: HeatmapGrid): HeatmapPlotData { return [xs, ys, grid.cells, x0s, x1s, y0s, y1s]; } -/** - * Grid for the rows of the server-side `widthBucket` query. widthBucket - * returns buckets 0 (underflow) through nBuckets + 1 (overflow), so each time - * bucket has nBuckets + 2 rows. - */ -export function gridFromBucketRows({ - data, - timestampColumn, - generatedTsBuckets, - scaleType, - effectiveMin, - max, - nBuckets, -}: { - data: Record[]; - timestampColumn: ColumnMetaType; - generatedTsBuckets: Date[]; - scaleType: HeatmapScaleType; - effectiveMin: number; - max: number; - nBuckets: number; -}): HeatmapGrid { - const isLog = scaleType === 'log' && effectiveMin > 0 && max > effectiveMin; - const bucketValue = (j: number) => - isLog - ? effectiveMin * Math.pow(max / effectiveMin, j / nBuckets) - : effectiveMin + j * ((max - effectiveMin) / nBuckets); - - // Row j is centered on bucketValue(j), which is the upper bound of - // widthBucket's bucket j, so the row boundaries sit half a bucket either - // side of it. - const rows = nBuckets + 2; - const edges = Array.from({ length: rows + 1 }, (_, k) => - bucketValue(k - 0.5), - ); - - const times = generatedTsBuckets.map(d => d.getTime()); - const cells: number[] = []; - - let dataIndex = 0; - for (const generatedTs of times) { - for (let j = 0; j < rows; j++) { - const row = data[dataIndex]; - - if ( - row != null && - new Date(row[timestampColumn.name]).getTime() == generatedTs && - row['x_bucket'] == j - ) { - cells.push(Number.parseInt(row['count'], 10)); // UInt64 returns as string - - // Skip duplicate buckets (from unmerged distributed table results) - while ( - dataIndex < data.length && - new Date(data[dataIndex][timestampColumn.name]).getTime() == - generatedTs && - data[dataIndex]['x_bucket'] == j - ) { - dataIndex++; - } - } else { - cells.push(0); - } - } - } - - return { - times, - stepMs: times.length > 1 ? times[1] - times[0] : 0, - yAxis: { type: 'numeric', scale: isLog ? 'log' : 'linear', edges }, - cells, - }; -} - /** * Cumulative share (0-100) of events at or below each row, keyed by row * index and pooled across all time buckets. diff --git a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts index 948df6920f..21ea61c90c 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts @@ -1,10 +1,17 @@ +import type { HeatmapScaleType } from '@hyperdx/common-utils/dist/core/heatmap'; import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; +import { + type HeatmapChartConfig, + toDistributionHeatmapConfig, +} from '@hyperdx/common-utils/dist/core/heatmapQuery'; import { getAlignedDateRange } from '@hyperdx/common-utils/dist/core/utils'; -import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + isBuilderChartConfig, + isPromqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, - DisplayType, PromqlConfigWithDateRange, SQLInterval, } from '@hyperdx/common-utils/dist/types'; @@ -12,31 +19,13 @@ import { import { convertToTimeChartConfig, getTimeChartGranularity, - isAggregateFunction, } from '@/ChartUtils'; -import { heatmapLowQuantile } from './heatmapBounds'; -import type { HeatmapScaleType } from './heatmapGrid'; - -export type HeatmapChartConfig = { - displayType: DisplayType.Heatmap; - select: [ - { - aggFn: 'heatmap'; - valueExpression: string; - countExpression?: string; - }, - ]; - from: BuilderChartConfigWithDateRange['from']; - where: BuilderChartConfigWithDateRange['where']; - dateRange: BuilderChartConfigWithDateRange['dateRange']; - granularity: BuilderChartConfigWithDateRange['granularity']; - timestampValueExpression: BuilderChartConfigWithDateRange['timestampValueExpression']; - numberFormat?: BuilderChartConfigWithDateRange['numberFormat']; - filters?: BuilderChartConfigWithDateRange['filters']; - connection: string; - with?: BuilderChartConfigWithDateRange['with']; -}; +export { + buildHeatmapBoundsConfig, + buildHeatmapBucketConfig, + type HeatmapChartConfig, +} from '@hyperdx/common-utils/dist/core/heatmapQuery'; /** The configs a series-mode heatmap can query. */ export type HeatmapSeriesChartConfig = @@ -112,26 +101,7 @@ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { }; } - const firstSelect = Array.isArray(config.select) - ? config.select[0] - : undefined; - return { - mode: 'distribution', - config: { - ...config, - displayType: DisplayType.Heatmap, - select: [ - { - aggFn: 'heatmap' as const, - valueExpression: firstSelect?.valueExpression ?? '', - countExpression: firstSelect?.countExpression, - }, - ], - granularity: config.granularity, - numberFormat: config.numberFormat, - }, - scaleType: firstSelect?.heatmapScaleType ?? 'log', - }; + return { mode: 'distribution', ...toDistributionHeatmapConfig(config) }; } /** @@ -168,175 +138,3 @@ export function buildHeatmapSeriesConfig( heatmap: undefined, }); } - -/** - * Build the bounds (min/max) ChartConfig that runs first. Result feeds - * `effectiveMin`/`max` into `buildHeatmapBucketConfig`. - */ -export function buildHeatmapBoundsConfig({ - config, - scaleType, - granularity, -}: { - config: HeatmapChartConfig; - scaleType: HeatmapScaleType; - granularity: SQLInterval; -}): BuilderChartConfigWithDateRange { - const valueExpression = config.select[0].valueExpression; - const isAggregateExpression = isAggregateFunction(valueExpression); - const qLo = heatmapLowQuantile(scaleType); - - return isAggregateExpression - ? { - ...config, - where: '', - orderBy: undefined, - granularity: undefined, - select: [ - { - aggFn: 'quantile' as const, - level: qLo, - aggCondition: `value_calc >= 0`, - aggConditionLanguage: 'sql', - valueExpression: 'value_calc', - alias: 'min', - }, - { - aggFn: 'max' as const, - valueExpression: 'value_calc', - alias: 'max', - }, - ], - with: [ - { - name: 'min_max_calc', - chartConfig: { - ...config, - select: [{ valueExpression, alias: 'value_calc' }], - orderBy: undefined, - // Emits the __hdx_time_bucket column the outer query filters on. - granularity, - }, - }, - ], - timestampValueExpression: '__hdx_time_bucket', - // The first bucket is labelled before an unaligned start, so align - // this filter on bucket labels. The inner query keeps the original - // range so no events outside it are counted. - dateRange: getAlignedDateRange(config.dateRange, granularity), - from: { databaseName: '', tableName: 'min_max_calc' }, - } - : { - ...config, - orderBy: undefined, - granularity: undefined, - select: [ - { - aggFn: 'quantile' as const, - level: qLo, - valueExpression, - aggCondition: `${valueExpression} >= 0`, - aggConditionLanguage: 'sql', - alias: 'min', - }, - { - aggFn: 'max' as const, - valueExpression, - alias: 'max', - }, - ], - }; -} - -/** - * Build the bucketed-counts ChartConfig that runs second. `effectiveMin`/`max` - * are usually numbers (resolved from the bounds query), but accept strings so - * callers — like the editor's SQL preview — can pass placeholder tokens - * (e.g. `'{min}'`) before the bounds are known. - */ -export function buildHeatmapBucketConfig({ - config, - scaleType, - effectiveMin, - max, - granularity, - nBuckets, -}: { - config: HeatmapChartConfig; - scaleType: HeatmapScaleType; - effectiveMin: string | number; - max: string | number; - granularity: SQLInterval; - nBuckets: number; -}): BuilderChartConfigWithDateRange { - const valueExpression = config.select[0].valueExpression; - // The chart editor saves a cleared Count input as ''. - const countExpression = config.select[0].countExpression?.trim() || 'count()'; - const isAggregateExpression = isAggregateFunction(valueExpression); - - const bucketExprAgg = - scaleType === 'log' - ? `widthBucket(log(greatest(toFloat64(value_calc), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` - : `widthBucket(value_calc, ${effectiveMin}, ${max}, ${nBuckets})`; - const bucketExprDirect = - scaleType === 'log' - ? `widthBucket(log(greatest(toFloat64(${valueExpression}), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` - : `widthBucket(${valueExpression}, ${effectiveMin}, ${max}, ${nBuckets})`; - - return isAggregateExpression - ? { - ...config, - where: '', - select: [ - { - valueExpression: 'sum(value_count)', - alias: 'count', - }, - ], - groupBy: [ - { - valueExpression: bucketExprAgg, - alias: 'x_bucket', - }, - ], - with: [ - { - name: 'bucket_calc', - chartConfig: { - ...config, - select: [ - { valueExpression, alias: 'value_calc' }, - { - valueExpression: countExpression, - alias: 'value_count', - }, - ], - granularity, - orderBy: undefined, - }, - }, - ], - timestampValueExpression: '__hdx_time_bucket', - dateRange: getAlignedDateRange(config.dateRange, granularity), - from: { databaseName: '', tableName: 'bucket_calc' }, - orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], - granularity, - } - : { - ...config, - select: [ - { - valueExpression: countExpression, - alias: 'count', - }, - ], - groupBy: [ - { - valueExpression: bucketExprDirect, - alias: 'x_bucket', - }, - ], - orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], - granularity, - }; -} diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index da84395e5c..c76c51d04c 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -1,9 +1,11 @@ # @hyperdx/claude-plugin A Claude Code plugin that draws ClickStack MCP query results as charts in the -terminal. When Claude runs `clickstack_timeseries` or `clickstack_table`, the -tool row shows a line, stacked bar, bar, pie, number or table chart in place of -the raw JSON. Claude still reads the full result. +terminal. When Claude runs `clickstack_timeseries`, `clickstack_table` or +`clickstack_heatmap`, the tool row shows a line, stacked bar, bar, pie, number, +table or heatmap chart in place of the raw JSON. Claude still reads the full +result. `clickstack_query_tile` results are drawn when the tile is a +distribution heatmap. Charts are drawn by the same renderers as the `hdx` CLI, so a query looks the same in both. diff --git a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts index e3cd5665de..3ac0c12c8d 100644 --- a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts +++ b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts @@ -169,4 +169,54 @@ describe('toTileResult', () => { ), ).toBeUndefined(); }); + + it('turns a heatmap result into a grid, for the heatmap and query_tile tools', () => { + const heatmap = { + scale: 'log', + numberFormat: { output: 'duration', factor: 1e-9 }, + times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], + valueEdges: [1, 10, 100], + counts: [ + [0, 3, 0, 0], + [0, 1, 2, 0], + ], + }; + const out = [{ type: 'text', text: JSON.stringify({ heatmap }) }]; + for (const tool of ['clickstack_heatmap', 'clickstack_query_tile']) { + const result = toTileResult( + `mcp__clickstack__${tool}`, + {}, + out, + calledAt, + ); + expect(result).toMatchObject({ + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + numberFormat: heatmap.numberFormat, + }, + grid: { + stepMs: 60_000, + // Under- and overflow rows get one bucket's height on either side. + yAxis: { + type: 'numeric', + scale: 'log', + edges: [0.1, 1, 10, 100, 1000], + }, + cells: [0, 3, 0, 0, 0, 1, 2, 0], + }, + }); + } + }); + + it('leaves a query_tile result alone unless it is a heatmap', () => { + expect( + toTileResult( + 'mcp__clickstack__clickstack_query_tile', + {}, + output, + calledAt, + ), + ).toBeUndefined(); + }); }); diff --git a/packages/claude-plugin/src/renderChart.ts b/packages/claude-plugin/src/renderChart.ts index c1f991f268..4e4f75ff51 100644 --- a/packages/claude-plugin/src/renderChart.ts +++ b/packages/claude-plugin/src/renderChart.ts @@ -9,6 +9,7 @@ import { isTrimmed, toTileResult } from './toTileResult'; const TIME_CHART_ROWS = 14; const MAX_TABLE_ROWS = 20; +const HEATMAP_ROWS = 18; const TRIMMED_NOTE = 'The server trimmed this result to fit its size limit; the chart shows only the rows it returned.'; @@ -29,9 +30,14 @@ export function renderChart( const height = displayType === DisplayType.Number ? 3 - : isTimeSeriesDisplayType(displayType) - ? TIME_CHART_ROWS - : Math.min(result.data.data.length + 2, MAX_TABLE_ROWS); + : displayType === DisplayType.Heatmap + ? HEATMAP_ROWS + : isTimeSeriesDisplayType(displayType) + ? TIME_CHART_ROWS + : Math.min( + (result.status === 'ok' ? result.data.data.length : 0) + 2, + MAX_TABLE_ROWS, + ); try { const lines = ansiLines( diff --git a/packages/claude-plugin/src/toTileResult.ts b/packages/claude-plugin/src/toTileResult.ts index b68a77fbb4..b9b1570e37 100644 --- a/packages/claude-plugin/src/toTileResult.ts +++ b/packages/claude-plugin/src/toTileResult.ts @@ -1,6 +1,10 @@ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; -import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import type { HeatmapGrid } from '@hyperdx/common-utils/dist/core/heatmap'; import { TIME_SERIES_DISPLAY_TYPE_BY_NAME } from '@hyperdx/common-utils/dist/core/utils'; +import type { + ChartConfigWithDateRange, + NumberFormat, +} from '@hyperdx/common-utils/dist/types'; import { DisplayType } from '@hyperdx/common-utils/dist/types'; import type { TileQueryResult } from '@hyperdx/cli/src/shared/tileQuery'; @@ -19,6 +23,10 @@ const DISPLAY_TYPES: Record> = { }, }; +// Tools answering with a `heatmap` grid. A query_tile result for any other +// tile carries no display type, so it is left alone. +const HEATMAP_TOOLS = ['clickstack_heatmap', 'clickstack_query_tile']; + type ToolInput = { shape?: string; select?: unknown[]; @@ -28,12 +36,22 @@ type ToolInput = { endTime?: string; }; -type ChartResult = Extract; +type HeatmapBody = { + numberFormat?: NumberFormat; + times: string[]; + valueEdges: number[]; + counts: number[][]; + scale: 'log' | 'linear'; +}; + +type ChartResult = Extract; /** Matches by MCP tool name suffix, so it works whatever the user named the server. */ export function chartedTool(toolName: string): string | undefined { const suffix = toolName.split('__').pop() ?? ''; - return suffix in DISPLAY_TYPES ? suffix : undefined; + return suffix in DISPLAY_TYPES || HEATMAP_TOOLS.includes(suffix) + ? suffix + : undefined; } /** MCP output arrives as text blocks holding one JSON document. */ @@ -84,6 +102,20 @@ export function toTileResult( ? new Date(args.startTime) : new Date(end.getTime() - DEFAULT_WINDOW_MS); + if (HEATMAP_TOOLS.includes(tool)) { + const heatmap = parseBody(output)?.heatmap as HeatmapBody | undefined; + if (!heatmap?.counts?.length) return undefined; + return { + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + dateRange: [start, end], + numberFormat: heatmap.numberFormat, + } as unknown as ChartConfigWithDateRange, + grid: toGrid(heatmap), + }; + } + const data = parseResponse(output); if (!data || data.data.length === 0) return undefined; const shapes = DISPLAY_TYPES[tool]!; @@ -110,3 +142,27 @@ export function toTileResult( return { status: 'ok', queriedConfig, data }; } +function toGrid({ + times, + valueEdges, + counts, + scale, +}: HeatmapBody): HeatmapGrid { + const ms = times.map(t => Date.parse(t)); + return { + times: ms, + stepMs: ms.length > 1 ? ms[1]! - ms[0]! : 0, + yAxis: { type: 'numeric', scale, edges: withOuterRows(valueEdges, scale) }, + cells: counts.flat(), + }; +} + +// The server reports the bucket bounds, with an under- and overflow count on +// either side; give those two rows one bucket's height so they can be drawn. +function withOuterRows(edges: number[], scale: 'log' | 'linear'): number[] { + const [first, second] = [edges[0]!, edges[1] ?? edges[0]!]; + const [beforeLast, last] = [edges.at(-2) ?? edges.at(-1)!, edges.at(-1)!]; + return scale === 'log' && first > 0 + ? [(first * first) / second, ...edges, (last * last) / beforeLast] + : [first - (second - first), ...edges, last + (last - beforeLast)]; +} diff --git a/packages/claude-plugin/test/render.test.tsx b/packages/claude-plugin/test/render.test.tsx index 3d4fe99779..b99e4eabb2 100644 --- a/packages/claude-plugin/test/render.test.tsx +++ b/packages/claude-plugin/test/render.test.tsx @@ -130,6 +130,53 @@ test( }, ); +test('draws a heatmap result as shaded cells', { options }, async ($, on) => { + const heatmap = { + scale: 'log', + times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], + valueEdges: [1, 10, 100, 1000], + counts: [ + [0, 9, 1, 0, 0], + [0, 0, 2, 30, 0], + ], + }; + let id = ''; + on('tool.call', (_, e) => { + id = e.tool_use_id; + return { result: [{ type: 'text', text: JSON.stringify({ heatmap }) }] }; + }); + on('ui.render', ($, e) => { + const { Text } = $.ui.resolve(e); + return engine row; + }); + const tool = 'mcp__clickstack__clickstack_heatmap'; + await $.tool.call({ tool, sourceId: 's', valueExpression: 'Duration' }); + const ui = await $.ui.mount({ + plugin: 'hyperdx-charts', + surface: 'terminal', + component: 'ToolGroup', + props: { + calls: [ + { + tool_use_id: id, + tool, + input: {}, + isRunning: false, + isErrored: false, + isInterrupted: false, + }, + ], + isActive: false, + isExpanded: false, + }, + viewport: { columns: 100, rows: 40 }, + }); + expect( + await ui.find({ type: 'Text', text: /events per cell, max 30/ }), + ).toBeDefined(); + expect(await ui.find({ type: 'Text', text: /[░▒▓█]/ })).toBeDefined(); +}); + test( 'draws a timeseries call drawn as its own tool row', { options }, diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index d74077b978..84de88d0c1 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -161,8 +161,10 @@ dashboard**: module (only dependency: chalk) — number formatting is injected via callbacks, and it must never import from the rest of the CLI. -Supported display types: line, stacked*bar, number, table, bar, pie, markdown. -Heatmap / search / event patterns / PromQL show a placeholder. The web's +Supported display types: line, stacked*bar, number, table, bar, pie, markdown, +and distribution heatmaps (queried through common-utils' +`queryDistributionHeatmap`, the same two queries the web runs). Series +heatmaps / search / event patterns / PromQL show a placeholder. The web's previous-period comparison overlay is not rendered; tiles with `compareToPreviousPeriod` set show a dim callout instead. Time-series charts stretch to the full terminal width via peak-preserving resampling diff --git a/packages/cli/src/__tests__/termchart.test.ts b/packages/cli/src/__tests__/termchart.test.ts index 60c051aaac..38a2f5d42a 100644 --- a/packages/cli/src/__tests__/termchart.test.ts +++ b/packages/cli/src/__tests__/termchart.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from '@jest/globals'; import { renderLegend } from '@/termchart/ansi'; import { niceTicks, + renderHeatmapChart, renderLineChart, renderStackedBarChart, resampleSeries, @@ -386,6 +387,66 @@ describe('renderStackedBarChart', () => { }); }); +describe('renderHeatmapChart', () => { + // Two time buckets, four value rows (edges 1..16 on a log axis); the second + // bucket has a dense band in the top row. + const data = { + times: [1_700_000_000, 1_700_000_060], + rowEdges: [1, 2, 4, 8, 16], + counts: [ + [0, 5, 0, 0], + [0, 1, 0, 50], + ], + }; + + it('shades denser cells darker and labels the value axis', () => { + const output = stripAnsi( + renderHeatmapChart({ data, width: 40, height: 12 }), + ); + const lines = output.split('\n'); + expect(lines[0]).toContain('16'); + expect(lines[0]).toContain('█'); + expect(output).toContain('max 50'); + const bottomRow = lines.at(-4)!; + expect(bottomRow).toMatch(/[░▒]/); + expect(bottomRow).not.toContain('█'); + }); + + it('trims empty value rows so the populated range fills the plot', () => { + const output = stripAnsi( + renderHeatmapChart({ + data: { + ...data, + counts: [ + [0, 5, 5, 0], + [0, 5, 5, 0], + ], + }, + width: 40, + height: 12, + }), + ); + expect(output.split('\n')[0]).toContain('8'); + }); + + it('says so when every cell is empty', () => { + const output = stripAnsi( + renderHeatmapChart({ + data: { + ...data, + counts: [ + [0, 0, 0, 0], + [0, 0, 0, 0], + ], + }, + width: 40, + height: 12, + }), + ); + expect(output).toContain('No data'); + }); +}); + describe('renderLegend', () => { const series = Array.from({ length: 8 }, (_, i) => ({ displayName: `service-${i}-${'x'.repeat(12)}`, diff --git a/packages/cli/src/__tests__/tileQuery.test.ts b/packages/cli/src/__tests__/tileQuery.test.ts new file mode 100644 index 0000000000..0dc15a6150 --- /dev/null +++ b/packages/cli/src/__tests__/tileQuery.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, jest } from '@jest/globals'; + +import type { SavedChartConfig } from '@hyperdx/common-utils/dist/types'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import type { ProxyClickhouseClient, SourceResponse } from '@/api/client'; +import { fetchTileData } from '@/shared/tileQuery'; + +const traceSource = { + id: 'src-2', + kind: 'trace', + name: 'Traces', + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, +} as unknown as SourceResponse; + +const dateRange: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-13T00:02:00Z'), +]; + +function heatmapTile(extra: Record = {}) { + return { + displayType: DisplayType.Heatmap, + source: 'src-2', + select: [{ aggFn: 'count', valueExpression: 'Duration' }], + where: '', + ...extra, + } as unknown as SavedChartConfig; +} + +describe('fetchTileData heatmap tiles', () => { + it('runs a distribution heatmap through the bounds and bucket queries', async () => { + const queryChartConfig = jest + .fn<() => Promise>() + .mockResolvedValueOnce({ meta: [], data: [{ min: '1', max: '100' }] }) + .mockResolvedValueOnce({ + meta: [{ name: '__hdx_time_bucket', type: 'DateTime' }], + data: [ + { + __hdx_time_bucket: '2026-10-13T00:00:00Z', + x_bucket: 2, + count: '4', + }, + ], + }); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile(), + source: traceSource, + dateRange, + granularity: '1 minute', + }); + + expect(queryChartConfig).toHaveBeenCalledTimes(2); + expect(result.status).toBe('heatmap'); + if (result.status !== 'heatmap') return; + expect(result.grid?.times).toHaveLength(2); + expect(result.grid?.cells.reduce((a, b) => a + b, 0)).toBe(4); + }); + + it('reports series heatmaps as unsupported without querying', async () => { + const queryChartConfig = jest.fn(); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile({ heatmap: { mode: 'series' } }), + source: traceSource, + dateRange, + }); + expect(result.status).toBe('unsupported'); + expect(queryChartConfig).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/__tests__/tileRender.test.ts b/packages/cli/src/__tests__/tileRender.test.ts index 5a015d7cd9..d2c55574c6 100644 --- a/packages/cli/src/__tests__/tileRender.test.ts +++ b/packages/cli/src/__tests__/tileRender.test.ts @@ -86,4 +86,38 @@ describe('renderTileContent', () => { ); expect(render(DisplayType.StackedLine)).not.toBe(render(DisplayType.Line)); }); + + it('draws a distribution heatmap with its duration axis', () => { + const heatmapConfig = { + displayType: DisplayType.Heatmap, + dateRange, + numberFormat: { output: 'duration', factor: 0.001 }, + } as unknown as ChartConfigWithDateRange; + const result: TileQueryResult = { + status: 'heatmap', + queriedConfig: heatmapConfig, + grid: { + times: [dateRange[0].getTime(), dateRange[0].getTime() + 60_000], + stepMs: 60_000, + yAxis: { type: 'numeric', scale: 'log', edges: [1, 10, 100, 1000] }, + cells: [9, 1, 0, 0, 2, 30], + }, + }; + const output = stripAnsi( + renderTileContent({ result, source: undefined, width: 60, height: 12 }), + ); + expect(output).toContain('events per cell, max 30'); + expect(output).toMatch(/[░▒▓█]/); + + expect( + stripAnsi( + renderTileContent({ + result: { ...result, grid: undefined }, + source: undefined, + width: 60, + height: 12, + }), + ), + ).toContain('Not enough data points'); + }); }); diff --git a/packages/cli/src/shared/chartData.ts b/packages/cli/src/shared/chartData.ts index 3dca9f9d94..70f06dc359 100644 --- a/packages/cli/src/shared/chartData.ts +++ b/packages/cli/src/shared/chartData.ts @@ -20,6 +20,10 @@ import { type ColumnMetaType, type ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; +import { + heatmapCountColumns, + type HeatmapGrid, +} from '@hyperdx/common-utils/dist/core/heatmap'; import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; import type { ChartConfigWithOptDateRange, @@ -29,6 +33,7 @@ import type { import type { SourceResponse } from '@/api/client'; import type { CategoricalEntry, + HeatmapData, TableColumn, TimeChartSeries, } from '@/termchart'; @@ -551,3 +556,13 @@ export function deriveTableColumns({ isGroupColumn: groupByKeys.includes(key), })); } + +/** A heatmap grid as termchart draws it: per-column counts, seconds timestamps. */ +export function formatHeatmapGrid(grid: HeatmapGrid): HeatmapData | undefined { + if (grid.yAxis.type !== 'numeric') return undefined; + return { + times: grid.times.map(t => t / 1000), + rowEdges: grid.yAxis.edges, + counts: heatmapCountColumns(grid), + }; +} diff --git a/packages/cli/src/shared/tileConfig.ts b/packages/cli/src/shared/tileConfig.ts index b8b1fea940..9c02ce0481 100644 --- a/packages/cli/src/shared/tileConfig.ts +++ b/packages/cli/src/shared/tileConfig.ts @@ -401,5 +401,6 @@ export const CLI_SUPPORTED_DISPLAY_TYPES: ReadonlySet = new Set([ DisplayType.Table, DisplayType.Pie, DisplayType.Bar, + DisplayType.Heatmap, DisplayType.Markdown, ]); diff --git a/packages/cli/src/shared/tileQuery.ts b/packages/cli/src/shared/tileQuery.ts index 80b90de471..6271241307 100644 --- a/packages/cli/src/shared/tileQuery.ts +++ b/packages/cli/src/shared/tileQuery.ts @@ -5,7 +5,13 @@ */ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { + getHeatmapMode, + type HeatmapGrid, +} from '@hyperdx/common-utils/dist/core/heatmap'; +import { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import type { ChartConfigWithDateRange, SavedChartConfig, @@ -27,6 +33,12 @@ export type TileQueryResult = queriedConfig: ChartConfigWithDateRange; data: ResponseJSON>; } + | { + status: 'heatmap'; + queriedConfig: ChartConfigWithDateRange; + /** Undefined when the range has too few distinct values to bucket */ + grid: HeatmapGrid | undefined; + } | { status: 'markdown'; /** Markdown tiles render statically without querying */ @@ -38,6 +50,9 @@ export type TileQueryResult = resolution: Extract; }; +// Value buckets per heatmap column; the renderer merges them to fit its rows. +const HEATMAP_VALUE_BUCKETS = 40; + export interface FetchTileDataParams { clickhouseClient: ProxyClickhouseClient; metadata: Metadata; @@ -91,6 +106,34 @@ export async function fetchTileData({ return { status: 'unresolved', resolution }; } + if (displayType === DisplayType.Heatmap) { + const heatmapConfig = resolution.config; + if ( + !isBuilderChartConfig(heatmapConfig) || + getHeatmapMode(heatmapConfig) === 'series' + ) { + return { + status: 'unsupported', + message: 'Series heatmap tiles are not supported in the CLI yet.', + }; + } + const result = await queryDistributionHeatmap({ + client: clickhouseClient, + metadata, + config: heatmapConfig, + granularity: heatmapConfig.granularity, + targetTimeBuckets: maxTimeBuckets, + nBuckets: HEATMAP_VALUE_BUCKETS, + querySettings: source?.querySettings, + opts: { abort_signal: signal }, + }); + return { + status: 'heatmap', + queriedConfig: heatmapConfig, + grid: result.heatmap?.grid, + }; + } + const queriedConfig = convertTileConfigForQuery(resolution.config, { maxTimeBuckets, }); diff --git a/packages/cli/src/shared/tileRender.ts b/packages/cli/src/shared/tileRender.ts index 4b62c77ab7..a50a3ab340 100644 --- a/packages/cli/src/shared/tileRender.ts +++ b/packages/cli/src/shared/tileRender.ts @@ -15,6 +15,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import type { SourceResponse } from '@/api/client'; import { deriveTableColumns, + formatHeatmapGrid, formatResponseForCategoricalChart, formatResponseForTimeChart, getNumberChartValue, @@ -30,6 +31,7 @@ import type { TileQueryResult } from '@/shared/tileQuery'; import { defaultFormatValue, renderCategoricalChart, + renderHeatmapChart, renderLineChart, renderMarkdown, renderNumberChart, @@ -62,6 +64,20 @@ export function renderTileContent({ if (result.status === 'unresolved') { return chalk.yellow(result.resolution.message); } + if (result.status === 'heatmap') { + const data = result.grid && formatHeatmapGrid(result.grid); + if (!data) return chalk.dim('Not enough data points to draw a heatmap.'); + const { chartFormat } = resolveChartNumberFormats( + result.queriedConfig, + source, + ); + return renderHeatmapChart({ + data, + width, + height, + formatTick: axisTickFormatter(chartFormat), + }); + } const { queriedConfig, data } = result; const displayType = queriedConfig.displayType ?? DisplayType.Line; diff --git a/packages/cli/src/termchart/heatmap.ts b/packages/cli/src/termchart/heatmap.ts new file mode 100644 index 0000000000..bc1d91e099 --- /dev/null +++ b/packages/cli/src/termchart/heatmap.ts @@ -0,0 +1,106 @@ +/** + * Heatmap renderer: time across, value buckets down, each cell shaded by + * how many events landed in it. + */ + +import chalk from 'chalk'; + +import { + chalkColor, + defaultFormatValue, + NO_DATA_TEXT, + renderTimeAxis, + truncate, +} from '@/termchart/ansi'; +import { resampleLinear } from '@/termchart/scale'; +import type { HeatmapData, ValueFormatter } from '@/termchart/types'; + +const SHADES = ['░', '▒', '▓', '█']; +const SHADE_COLORS = ['blue', 'cyan', 'yellow', 'red']; +const LABEL_WIDTH = 10; + +/** The source index range each of `length` output slots covers, summed when shrinking. */ +function slotRange(slot: number, length: number, from: number, count: number) { + const start = from + Math.floor((slot * count) / length); + const end = Math.max( + start + 1, + from + Math.floor(((slot + 1) * count) / length), + ); + return [start, end] as const; +} + +export function renderHeatmapChart({ + data, + width, + height, + formatTick = defaultFormatValue, +}: { + data: HeatmapData; + width: number; + height: number; + formatTick?: ValueFormatter; +}): string { + const { times, rowEdges, counts } = data; + const rowCount = rowEdges.length - 1; + const populated = Array.from({ length: rowCount }, (_, r) => + counts.some(column => (column[r] ?? 0) > 0), + ); + const first = populated.indexOf(true); + const last = populated.lastIndexOf(true); + if (first === -1 || times.length === 0) return chalk.dim(NO_DATA_TEXT); + + const gutterWidth = LABEL_WIDTH + 2; + const plotWidth = Math.max(10, width - gutterWidth); + const plotHeight = Math.max(2, height - 3); + const visibleRows = last - first + 1; + + const cells = Array.from({ length: plotHeight }, (_, row) => { + const [r0, r1] = slotRange(row, plotHeight, first, visibleRows); + return Array.from({ length: plotWidth }, (_, col) => { + const [t0, t1] = slotRange(col, plotWidth, 0, times.length); + let sum = 0; + for (let t = t0; t < t1; t++) + for (let r = r0; r < r1; r++) sum += counts[t]?.[r] ?? 0; + return sum; + }); + }); + const max = Math.max(...cells.flat()); + + // Event counts are heavy-tailed, so shade on a log scale or one dense + // bucket washes out everything else. + const shade = (v: number) => { + if (v <= 0) return ' '; + const level = Math.min( + 3, + Math.floor((Math.log1p(v) / Math.log1p(max)) * 4 - 1e-9), + ); + return chalkColor(SHADE_COLORS[level])(SHADES[level]); + }; + + const labels = new Map([ + [plotHeight - 1, rowEdges[last + 1]], + [Math.floor(plotHeight / 2), rowEdges[first + Math.round(visibleRows / 2)]], + [0, rowEdges[first]], + ]); + const lines = cells + .map((columns, row) => { + const value = labels.get(row); + const label = + value === undefined + ? ' '.repeat(LABEL_WIDTH) + : truncate(formatTick(value), LABEL_WIDTH).padStart(LABEL_WIDTH); + return `${chalk.dim(label)} ${chalk.dim('┤')}${columns.map(shade).join('')}`; + }) + .reverse(); + + const axis = renderTimeAxis({ + timestamps: resampleLinear(times, plotWidth).map(Math.round), + gutterWidth, + plotWidth, + }); + const legend = `${SHADES.map((s, i) => chalkColor(SHADE_COLORS[i])(s)).join('')} ${chalk.dim( + `events per cell, max ${defaultFormatValue(max)}`, + )}`; + + return [...lines, axis, '', legend].join('\n'); +} diff --git a/packages/cli/src/termchart/index.ts b/packages/cli/src/termchart/index.ts index a84c69697e..597c3b2055 100644 --- a/packages/cli/src/termchart/index.ts +++ b/packages/cli/src/termchart/index.ts @@ -15,11 +15,13 @@ export { renderNumberChart, renderTableChart, } from '@/termchart/charts'; +export { renderHeatmapChart } from '@/termchart/heatmap'; export { niceTicks, resampleSeries } from '@/termchart/scale'; export { renderStackedBarChart } from '@/termchart/stackedBar'; export { renderLineChart } from '@/termchart/timeseries'; export type { CategoricalEntry, + HeatmapData, TableColumn, TimeChartData, TimeChartSeries, diff --git a/packages/cli/src/termchart/types.ts b/packages/cli/src/termchart/types.ts index 0badcb0303..1bf71d0e6a 100644 --- a/packages/cli/src/termchart/types.ts +++ b/packages/cli/src/termchart/types.ts @@ -51,3 +51,13 @@ export interface TableColumn { /** Group-by columns render left-aligned and skip numeric formatting */ isGroupColumn: boolean; } + +/** A heatmap: one column per time bucket, one row per value range. */ +export interface HeatmapData { + /** Column timestamps (unix seconds), ascending. */ + times: number[]; + /** `rows + 1` ascending row boundaries in data space. */ + rowEdges: number[]; + /** `counts[timeIdx][rowIdx]`: events in that time bucket and value range. */ + counts: number[][]; +} diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 1a2aee6743..eb63dcb67f 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -50,7 +50,7 @@ "dev:build": "tsup && tsc --emitDeclarationOnly --declaration && tsc-alias", "build": "tsup", "ci:build": "tsup", - "lint": "npx eslint . --ext .ts --max-warnings 80", + "lint": "npx eslint . --ext .ts --max-warnings 83", "lint:fix": "npx eslint . --ext .ts --fix", "ci:lint": "yarn lint && yarn tsc --noEmit", "ci:unit": "jest --ci --coverage", diff --git a/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts new file mode 100644 index 0000000000..6ad277853c --- /dev/null +++ b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts @@ -0,0 +1,122 @@ +import { queryDistributionHeatmap } from '@/core/distributionHeatmap'; +import { + heatmapBucketEdges, + resolveDistributionGranularity, +} from '@/core/heatmap'; +import { timeBucketByGranularity } from '@/core/utils'; +import { DisplayType } from '@/types'; + +const dateRange: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-13T00:02:00Z'), +]; +const config = { + displayType: DisplayType.Heatmap, + select: [{ aggFn: 'count' as const, valueExpression: 'Duration' }], + from: { databaseName: 'default', tableName: 'otel_traces' }, + where: '', + timestampValueExpression: 'Timestamp', + connection: 'c1', + dateRange, +}; + +function fakeClient( + bounds: Record, + buckets: Record[], +) { + const queryChartConfig = jest + .fn() + .mockResolvedValueOnce({ meta: [], data: [bounds] }) + .mockResolvedValueOnce({ + meta: [ + { name: '__hdx_time_bucket', type: 'DateTime' }, + { name: 'x_bucket', type: 'UInt32' }, + { name: 'count', type: 'UInt64' }, + ], + data: buckets, + }); + return { queryChartConfig }; +} + +const run = (client: ReturnType, granularity = '1 minute') => + queryDistributionHeatmap({ + client, + metadata: {} as never, + config, + granularity, + targetTimeBuckets: 48, + nBuckets: 4, + querySettings: undefined, + }); + +describe('queryDistributionHeatmap', () => { + it('buckets between the bounds query result and shapes a grid', async () => { + const client = fakeClient({ min: '10', max: '1000' }, [ + { __hdx_time_bucket: '2026-10-13T00:00:00Z', x_bucket: '1', count: '7' }, + { __hdx_time_bucket: '2026-10-13T00:01:00Z', x_bucket: '4', count: '3' }, + ]); + const { heatmap } = await run(client); + + const bucketQuery = client.queryChartConfig.mock.calls[1][0].config; + expect(JSON.stringify(bucketQuery)).toContain('widthBucket('); + expect(heatmap?.scaleType).toBe('log'); + expect(heatmap?.grid.times).toHaveLength(2); + // nBuckets + under/overflow rows per time bucket + expect(heatmap?.grid.cells).toEqual([0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0]); + expect(heatmap?.bucketEdges[0]).toBeCloseTo(10); + expect(heatmap?.bucketEdges.at(-1)).toBeCloseTo(1000); + }); + + it('skips the bucket query when the range has no spread', async () => { + const client = fakeClient({ min: '5', max: '5' }, []); + const result = await run(client); + expect(result.heatmap).toBeUndefined(); + expect(client.queryChartConfig).toHaveBeenCalledTimes(1); + }); + + it('replaces an unsupported granularity instead of looping on it', async () => { + const client = fakeClient({ min: '5', max: '5' }, []); + const result = await run(client, '1 week'); + expect(result.granularity).not.toBe('1 week'); + expect(result.granularityNote).toContain('not supported'); + }); +}); + +describe('resolveDistributionGranularity', () => { + const day: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-14T00:00:00Z'), + ]; + + it('keeps a supported granularity that fits the bucket budget', () => { + expect(resolveDistributionGranularity('1 hour', day, 48)).toEqual({ + granularity: '1 hour', + }); + }); + + it('coarsens a granularity that would flood the grid', () => { + const { granularity, note } = resolveDistributionGranularity( + '1 second', + day, + 48, + ); + expect(granularity).not.toBe('1 second'); + expect(note).toContain('86400 time buckets'); + }); +}); + +describe('heatmapBucketEdges', () => { + it('returns the widthBucket bounds, not the half-bucket display edges', () => { + expect(heatmapBucketEdges('linear', 0, 100, 4)).toEqual([ + 0, 25, 50, 75, 100, + ]); + }); +}); + +describe('timeBucketByGranularity', () => { + it('returns no buckets for a unit it cannot step by', () => { + expect( + timeBucketByGranularity(dateRange[0], dateRange[1], '1 week' as never), + ).toEqual([]); + }); +}); diff --git a/packages/common-utils/src/core/distributionHeatmap.ts b/packages/common-utils/src/core/distributionHeatmap.ts new file mode 100644 index 0000000000..71eb7c526f --- /dev/null +++ b/packages/common-utils/src/core/distributionHeatmap.ts @@ -0,0 +1,115 @@ +import type { BaseClickhouseClient } from '@/clickhouse'; +import { inferTimestampColumn } from '@/clickhouse'; +import type { HeatmapGrid, HeatmapScaleType } from '@/core/heatmap'; +import { + computeEffectiveMin, + gridFromBucketRows, + heatmapBucketEdges, + resolveDistributionGranularity, +} from '@/core/heatmap'; +import { + buildHeatmapBoundsConfig, + buildHeatmapBucketConfig, + toDistributionHeatmapConfig, +} from '@/core/heatmapQuery'; +import type { Metadata } from '@/core/metadata'; +import { timeBucketByGranularity } from '@/core/utils'; +import type { + BuilderChartConfigWithDateRange, + QuerySettings, + SQLInterval, +} from '@/types'; + +/** + * Runs a distribution heatmap outside the browser: the bounds query, then the + * `widthBucket` query, shaped into a grid. `heatmap` is absent when the range + * holds too few distinct values to bucket. + */ +export async function queryDistributionHeatmap({ + client, + metadata, + config, + granularity: requestedGranularity, + targetTimeBuckets, + nBuckets, + querySettings, + opts, +}: { + client: Pick; + metadata: Metadata; + config: BuilderChartConfigWithDateRange; + granularity: string | undefined; + targetTimeBuckets: number; + nBuckets: number; + querySettings: QuerySettings | undefined; + opts?: Parameters[0]['opts']; +}): Promise<{ + granularity: SQLInterval; + granularityNote?: string; + heatmap?: { + grid: HeatmapGrid; + scaleType: HeatmapScaleType; + bucketEdges: number[]; + }; +}> { + const { granularity, note: granularityNote } = resolveDistributionGranularity( + requestedGranularity, + config.dateRange, + targetTimeBuckets, + ); + const { config: heatmapConfig, scaleType } = + toDistributionHeatmapConfig(config); + + const bounds = await client.queryChartConfig({ + config: buildHeatmapBoundsConfig({ + config: heatmapConfig, + scaleType, + granularity, + }), + metadata, + querySettings, + opts, + }); + // quantile returns floats; UInt64 come back as strings + const min = Number.parseFloat(String(bounds.data?.[0]?.['min'] ?? '0')); + const max = Number.parseFloat(String(bounds.data?.[0]?.['max'] ?? '0')); + const effectiveMin = computeEffectiveMin(min, max, scaleType); + if (!(max > effectiveMin)) return { granularity, granularityNote }; + + const buckets = await client.queryChartConfig({ + config: buildHeatmapBucketConfig({ + config: heatmapConfig, + scaleType, + effectiveMin, + max, + granularity, + nBuckets, + }), + metadata, + querySettings, + opts, + }); + const timestampColumn = inferTimestampColumn(buckets.meta ?? []); + if (timestampColumn == null) { + throw new Error('Heatmap bucket query returned no time bucket column'); + } + + const [start, end] = config.dateRange; + return { + granularity, + granularityNote, + heatmap: { + grid: gridFromBucketRows({ + data: buckets.data, + timestampColumn, + generatedTsBuckets: timeBucketByGranularity(start, end, granularity), + scaleType, + effectiveMin, + max, + nBuckets, + }), + scaleType, + bucketEdges: heatmapBucketEdges(scaleType, effectiveMin, max, nBuckets), + }, + }; +} diff --git a/packages/common-utils/src/core/heatmap.ts b/packages/common-utils/src/core/heatmap.ts index 2018299a00..551b0a955a 100644 --- a/packages/common-utils/src/core/heatmap.ts +++ b/packages/common-utils/src/core/heatmap.ts @@ -1,4 +1,10 @@ -import type { PromqlHeatmapMode } from '@/types'; +import type { ColumnMetaType } from '@/clickhouse'; +import { + convertDateRangeToGranularityString, + convertGranularityToSeconds, +} from '@/core/utils'; +import type { PromqlHeatmapMode, SQLInterval } from '@/types'; +import { SQLIntervalSchema } from '@/types'; /** A heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ export function getHeatmapMode(config: { @@ -6,3 +12,202 @@ export function getHeatmapMode(config: { }): M | 'distribution' { return config.heatmap?.mode ?? 'distribution'; } + +export type HeatmapScaleType = 'log' | 'linear'; + +export type HeatmapYAxis = + | { + type: 'numeric'; + scale: HeatmapScaleType; + /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ + edges: number[]; + } + /** + * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per + * row; the top bound may be `Infinity` (a `+Inf` bucket). + */ + | { type: 'buckets'; bounds: number[] } + | { type: 'series'; labels: string[] }; + +/** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ +export type HeatmapGrid = { + /** Column centers, ms ascending. */ + times: number[]; + /** Column width, ms. */ + stepMs: number; + yAxis: HeatmapYAxis; + /** Row-major: `cells[timeIdx * rowCount + rowIdx]`. */ + cells: number[]; +}; + +export function heatmapRowCount(yAxis: HeatmapYAxis) { + switch (yAxis.type) { + case 'series': + return yAxis.labels.length; + case 'buckets': + return yAxis.bounds.length; + case 'numeric': + return Math.max(0, yAxis.edges.length - 1); + } +} + +/** + * Quantile used for the y-axis lower bound. A quantile (rather than min) + * keeps near-zero outliers from stretching the log axis. The upper bound + * uses the actual max so that rare latency spikes stay visible; log scale + * already compresses wide ranges. + */ +export function heatmapLowQuantile(scaleType: HeatmapScaleType) { + return scaleType === 'log' ? 0.01 : 0.001; +} + +/** + * Lower bound actually used for bucketing. Log scale needs a positive + * minimum (log(0) is undefined), and the range is capped to ~4 orders of + * magnitude so the axis isn't dominated by a long empty tail of near-zero + * outliers. + */ +export function computeEffectiveMin( + min: number, + max: number, + scaleType: HeatmapScaleType, +) { + return scaleType === 'log' ? Math.max(min, max * 1e-4 || 1e-4) : min; +} + +function isLogBuckets(scaleType: HeatmapScaleType, min: number, max: number) { + return scaleType === 'log' && min > 0 && max > min; +} + +/** Upper bound of `widthBucket` bucket j (bucket 0 is the underflow). */ +function bucketValueFn( + scaleType: HeatmapScaleType, + effectiveMin: number, + max: number, + nBuckets: number, +) { + return isLogBuckets(scaleType, effectiveMin, max) + ? (j: number) => effectiveMin * Math.pow(max / effectiveMin, j / nBuckets) + : (j: number) => effectiveMin + j * ((max - effectiveMin) / nBuckets); +} + +/** + * The `nBuckets + 1` bounds `widthBucket` counted between: bucket j covers + * [edges[j - 1], edges[j]). The grid's edges are display edges, half a bucket off. + */ +export function heatmapBucketEdges( + scaleType: HeatmapScaleType, + effectiveMin: number, + max: number, + nBuckets: number, +): number[] { + const bucketValue = bucketValueFn(scaleType, effectiveMin, max, nBuckets); + return Array.from({ length: nBuckets + 1 }, (_, j) => bucketValue(j)); +} + +/** Cells as `[timeIdx][rowIdx]`. */ +export function heatmapCountColumns(grid: HeatmapGrid): number[][] { + const rows = heatmapRowCount(grid.yAxis); + return grid.times.map((_, t) => grid.cells.slice(t * rows, (t + 1) * rows)); +} + +/** + * A time bucket size for a heatmap outside the browser: the requested one when + * it is supported and not so fine it floods the grid, else one sized for + * `targetBuckets`, with a note saying why. + */ +export function resolveDistributionGranularity( + granularity: string | undefined, + dateRange: [Date, Date], + targetBuckets: number, +): { granularity: SQLInterval; note?: string } { + const auto = convertDateRangeToGranularityString(dateRange, targetBuckets); + if (!granularity || granularity === 'auto') return { granularity: auto }; + if (!SQLIntervalSchema.safeParse(granularity).success) { + return { + granularity: auto, + note: `Granularity "${granularity}" is not supported (use " second|minute|hour|day"); used "${auto}".`, + }; + } + const seconds = convertGranularityToSeconds(granularity as SQLInterval); + const buckets = + (dateRange[1].getTime() - dateRange[0].getTime()) / 1000 / seconds; + if (!(seconds > 0) || buckets > targetBuckets * 4) { + return { + granularity: auto, + note: `Granularity "${granularity}" would make ${Math.round(buckets)} time buckets; used "${auto}".`, + }; + } + return { granularity: granularity as SQLInterval }; +} + +/** + * Grid for the rows of the server-side `widthBucket` query. widthBucket + * returns buckets 0 (underflow) through nBuckets + 1 (overflow), so each time + * bucket has nBuckets + 2 rows. + */ +export function gridFromBucketRows({ + data, + timestampColumn, + generatedTsBuckets, + scaleType, + effectiveMin, + max, + nBuckets, +}: { + data: Record[]; + timestampColumn: ColumnMetaType; + generatedTsBuckets: Date[]; + scaleType: HeatmapScaleType; + effectiveMin: number; + max: number; + nBuckets: number; +}): HeatmapGrid { + const isLog = isLogBuckets(scaleType, effectiveMin, max); + const bucketValue = bucketValueFn(scaleType, effectiveMin, max, nBuckets); + + // Row j is centered on bucketValue(j), which is the upper bound of + // widthBucket's bucket j, so the row boundaries sit half a bucket either + // side of it. + const rows = nBuckets + 2; + const edges = Array.from({ length: rows + 1 }, (_, k) => + bucketValue(k - 0.5), + ); + + const times = generatedTsBuckets.map(d => d.getTime()); + const cells: number[] = []; + + let dataIndex = 0; + for (const generatedTs of times) { + for (let j = 0; j < rows; j++) { + const row = data[dataIndex]; + + if ( + row != null && + new Date(row[timestampColumn.name]).getTime() == generatedTs && + row['x_bucket'] == j + ) { + cells.push(Number.parseInt(row['count'], 10)); // UInt64 returns as string + + // Skip duplicate buckets (from unmerged distributed table results) + while ( + dataIndex < data.length && + new Date(data[dataIndex][timestampColumn.name]).getTime() == + generatedTs && + data[dataIndex]['x_bucket'] == j + ) { + dataIndex++; + } + } else { + cells.push(0); + } + } + } + + return { + times, + stepMs: times.length > 1 ? times[1] - times[0] : 0, + yAxis: { type: 'numeric', scale: isLog ? 'log' : 'linear', edges }, + cells, + }; +} diff --git a/packages/common-utils/src/core/heatmapQuery.ts b/packages/common-utils/src/core/heatmapQuery.ts new file mode 100644 index 0000000000..d1e924abfc --- /dev/null +++ b/packages/common-utils/src/core/heatmapQuery.ts @@ -0,0 +1,225 @@ +import type { HeatmapScaleType } from '@/core/heatmap'; +import { heatmapLowQuantile } from '@/core/heatmap'; +import { getAlignedDateRange, isAggregateFunction } from '@/core/utils'; +import type { BuilderChartConfigWithDateRange, SQLInterval } from '@/types'; +import { DisplayType } from '@/types'; + +export type HeatmapChartConfig = { + displayType: DisplayType.Heatmap; + select: [ + { + aggFn: 'heatmap'; + valueExpression: string; + countExpression?: string; + }, + ]; + from: BuilderChartConfigWithDateRange['from']; + where: BuilderChartConfigWithDateRange['where']; + dateRange: BuilderChartConfigWithDateRange['dateRange']; + granularity: BuilderChartConfigWithDateRange['granularity']; + timestampValueExpression: BuilderChartConfigWithDateRange['timestampValueExpression']; + numberFormat?: BuilderChartConfigWithDateRange['numberFormat']; + filters?: BuilderChartConfigWithDateRange['filters']; + connection: string; + with?: BuilderChartConfigWithDateRange['with']; +}; + +/** + * The query a distribution heatmap runs, from a builder config whose first + * select item names the value to bucket. + */ +export function toDistributionHeatmapConfig( + config: BuilderChartConfigWithDateRange, +): { config: HeatmapChartConfig; scaleType: HeatmapScaleType } { + const firstSelect = Array.isArray(config.select) + ? config.select[0] + : undefined; + return { + config: { + ...config, + displayType: DisplayType.Heatmap, + select: [ + { + aggFn: 'heatmap' as const, + valueExpression: firstSelect?.valueExpression ?? '', + countExpression: firstSelect?.countExpression, + }, + ], + granularity: config.granularity, + numberFormat: config.numberFormat, + }, + scaleType: firstSelect?.heatmapScaleType ?? 'log', + }; +} + +/** + * Build the bounds (min/max) ChartConfig that runs first. Result feeds + * `effectiveMin`/`max` into `buildHeatmapBucketConfig`. + */ +export function buildHeatmapBoundsConfig({ + config, + scaleType, + granularity, +}: { + config: HeatmapChartConfig; + scaleType: HeatmapScaleType; + granularity: SQLInterval; +}): BuilderChartConfigWithDateRange { + const valueExpression = config.select[0].valueExpression; + const isAggregateExpression = isAggregateFunction(valueExpression); + const qLo = heatmapLowQuantile(scaleType); + + return isAggregateExpression + ? { + ...config, + where: '', + orderBy: undefined, + granularity: undefined, + select: [ + { + aggFn: 'quantile' as const, + level: qLo, + aggCondition: `value_calc >= 0`, + aggConditionLanguage: 'sql', + valueExpression: 'value_calc', + alias: 'min', + }, + { + aggFn: 'max' as const, + valueExpression: 'value_calc', + alias: 'max', + }, + ], + with: [ + { + name: 'min_max_calc', + chartConfig: { + ...config, + select: [{ valueExpression, alias: 'value_calc' }], + orderBy: undefined, + // Emits the __hdx_time_bucket column the outer query filters on. + granularity, + }, + }, + ], + timestampValueExpression: '__hdx_time_bucket', + // The first bucket is labelled before an unaligned start, so align + // this filter on bucket labels. The inner query keeps the original + // range so no events outside it are counted. + dateRange: getAlignedDateRange(config.dateRange, granularity), + from: { databaseName: '', tableName: 'min_max_calc' }, + } + : { + ...config, + orderBy: undefined, + granularity: undefined, + select: [ + { + aggFn: 'quantile' as const, + level: qLo, + valueExpression, + aggCondition: `${valueExpression} >= 0`, + aggConditionLanguage: 'sql', + alias: 'min', + }, + { + aggFn: 'max' as const, + valueExpression, + alias: 'max', + }, + ], + }; +} + +/** + * Build the bucketed-counts ChartConfig that runs second. `effectiveMin`/`max` + * are usually numbers (resolved from the bounds query), but accept strings so + * callers — like the editor's SQL preview — can pass placeholder tokens + * (e.g. `'{min}'`) before the bounds are known. + */ +export function buildHeatmapBucketConfig({ + config, + scaleType, + effectiveMin, + max, + granularity, + nBuckets, +}: { + config: HeatmapChartConfig; + scaleType: HeatmapScaleType; + effectiveMin: string | number; + max: string | number; + granularity: SQLInterval; + nBuckets: number; +}): BuilderChartConfigWithDateRange { + const valueExpression = config.select[0].valueExpression; + // The chart editor saves a cleared Count input as ''. + const countExpression = config.select[0].countExpression?.trim() || 'count()'; + const isAggregateExpression = isAggregateFunction(valueExpression); + + const bucketExprAgg = + scaleType === 'log' + ? `widthBucket(log(greatest(toFloat64(value_calc), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` + : `widthBucket(value_calc, ${effectiveMin}, ${max}, ${nBuckets})`; + const bucketExprDirect = + scaleType === 'log' + ? `widthBucket(log(greatest(toFloat64(${valueExpression}), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` + : `widthBucket(${valueExpression}, ${effectiveMin}, ${max}, ${nBuckets})`; + + return isAggregateExpression + ? { + ...config, + where: '', + select: [ + { + valueExpression: 'sum(value_count)', + alias: 'count', + }, + ], + groupBy: [ + { + valueExpression: bucketExprAgg, + alias: 'x_bucket', + }, + ], + with: [ + { + name: 'bucket_calc', + chartConfig: { + ...config, + select: [ + { valueExpression, alias: 'value_calc' }, + { + valueExpression: countExpression, + alias: 'value_count', + }, + ], + granularity, + orderBy: undefined, + }, + }, + ], + timestampValueExpression: '__hdx_time_bucket', + dateRange: getAlignedDateRange(config.dateRange, granularity), + from: { databaseName: '', tableName: 'bucket_calc' }, + orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], + granularity, + } + : { + ...config, + select: [ + { + valueExpression: countExpression, + alias: 'count', + }, + ], + groupBy: [ + { + valueExpression: bucketExprDirect, + alias: 'x_bucket', + }, + ], + orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], + granularity, + }; +} diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index e249239c9c..e634aad6e3 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -709,6 +709,8 @@ export function timeBucketByGranularity( let current = toStartOfInterval(start, granularity); const granularitySeconds = convertGranularityToSeconds(granularity); + // Units beyond day convert to 0 seconds, which would never advance. + if (!(granularitySeconds > 0)) return buckets; while (current < end) { buckets.push(current); current = fnsAdd(current, { @@ -1877,3 +1879,113 @@ export type MetadataMVQueryOptions = Map< string, Map<'NativeColumn' | string, string[]> // map from column name to keys >; + +/** Whether a SQL expression calls an aggregate function. */ +export const isAggregateFunction = (value: string) => { + const fns = [ + // Basic aggregates + 'count', + 'countIf', + 'countDistinct', + 'sum', + 'sumIf', + 'avg', + 'avgIf', + 'min', + 'max', + 'any', + 'anyLast', + 'groupArray', + 'groupArrayInsertAt', + 'groupArrayMovingAvg', + 'groupArraySample', + 'groupUniqArray', + 'groupUniqArrayIf', + 'groupArrayIntersect', + 'groupArrayIntersectIf', + 'groupArrayReduce', + 'groupBitmap', + 'groupBitmapIf', + 'groupBitmapOr', + 'groupBitmapXor', + + // Quantiles + 'quantile', + 'quantileIf', + 'quantileExact', + 'quantileExactWeighted', + 'quantileTiming', + 'quantileTimingWeighted', + 'quantileTDigest', + 'quantileTDigestWeighted', + 'quantileBFloat16', + 'quantileBFloat16Weighted', + 'quantiles', + 'median', + 'medianExact', + 'medianTDigest', + 'medianBFloat16', + + // Statistical functions + 'stddevPop', + 'stddevPopIf', + 'stddevSamp', + 'stddevSampIf', + 'varPop', + 'varPopIf', + 'varSamp', + 'varSampIf', + 'covarPop', + 'covarSamp', + 'corr', + + // Combinators + 'uniq', + 'uniqExact', + 'uniqCombined', + 'uniqCombined64', + 'uniqHLL12', + 'uniqTheta', + + // Bit operations + 'groupBitAnd', + 'groupBitOr', + 'groupBitXor', + + // Map and tuple + 'groupArrayMap', + 'groupArrayTuple', + 'groupArraySorted', + 'topK', + 'topKIf', + 'topKWeighted', + + // Aggregate combinators + 'argMin', + 'argMax', + 'minMap', + 'maxMap', + + // Specialized aggregates + 'runningDifference', + 'retention', + 'sequenceCount', + 'sequenceMatch', + 'histogram', + 'simpleLinearRegression', + 'stochasticLinearRegression', + 'categoricalInformationValue', + 'sumMap', + 'sumMapFiltered', + 'sumWithOverflow', + 'entropy', + 'skewPop', + 'skewSamp', + 'kurtPop', + 'kurtSamp', + ]; + + // Make case-insensitive since ClickHouse function names are case-insensitive + const lowerValue = value.toLowerCase(); + return fns.some(fn => lowerValue.includes(fn.toLowerCase() + '(')); +}; From f75c01a06a576eac81e15721d7d6a246ae7e834f Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Tue, 20 Oct 2026 21:23:48 +1100 Subject: [PATCH 2/3] fix: align heatmap output across the MCP server, hdx and the plugin Address review feedback on #3401: - hdx labels heatmap rows with the bucket bounds widthBucket counted (the plugin's edge helper moves to common-utils), formats saved Duration tiles as durations, refuses non-trace sources, keeps a saved tile's granularity, and includes the heatmap in --json output. - One HeatmapResultSchema and builder in common-utils shape the MCP result, hdx --json and what the plugin validates; value edges keep full precision. - Drop the query_tile granularity fallback, which external heatmap tiles never carry. - The plugin caches only results it can draw, so query_tile calls on other tiles no longer evict charts. --- .../api/src/mcp/tools/query/heatmapResult.ts | 12 ++-- packages/api/src/mcp/tools/query/helpers.ts | 2 +- packages/claude-plugin/hooks/chart.d.ts | 5 ++ packages/claude-plugin/hooks/register.tsx | 25 +++++--- .../src/__tests__/toTileResult.spec.ts | 53 ++++++++++++++++ packages/claude-plugin/src/index.ts | 2 +- packages/claude-plugin/src/toTileResult.ts | 52 ++++++++-------- packages/claude-plugin/test/render.test.tsx | 2 + packages/cli/AGENTS.md | 3 +- packages/cli/src/__tests__/tileQuery.test.ts | 62 +++++++++++++++++++ packages/cli/src/__tests__/tileRender.test.ts | 27 ++++++++ packages/cli/src/cli.tsx | 3 + packages/cli/src/shared/formatNumber.ts | 18 ++++++ packages/cli/src/shared/tileQuery.ts | 52 ++++++++++++++-- packages/cli/src/shared/tileRender.ts | 8 +-- .../__tests__/distributionHeatmap.test.ts | 35 +++++++++++ packages/common-utils/src/core/heatmap.ts | 61 +++++++++++++++++- 17 files changed, 365 insertions(+), 57 deletions(-) diff --git a/packages/api/src/mcp/tools/query/heatmapResult.ts b/packages/api/src/mcp/tools/query/heatmapResult.ts index 7901c15004..35447cee0c 100644 --- a/packages/api/src/mcp/tools/query/heatmapResult.ts +++ b/packages/api/src/mcp/tools/query/heatmapResult.ts @@ -1,7 +1,7 @@ import type { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import { - heatmapCountColumns, type HeatmapScaleType, + toHeatmapResult, } from '@hyperdx/common-utils/dist/core/heatmap'; import type { BuilderSavedChartConfig, @@ -62,15 +62,11 @@ export function formatHeatmapResult( const notes = granularityNote ? [granularityNote] : []; const body = heatmap ? { - heatmap: { + heatmap: toHeatmapResult(heatmap, { valueExpression, - scale: heatmap.scaleType, granularity, - ...(numberFormat ? { numberFormat } : {}), - times: heatmap.grid.times.map(t => new Date(t).toISOString()), - valueEdges: heatmap.bucketEdges.map(e => Number(e.toPrecision(6))), - counts: heatmapCountColumns(heatmap.grid), - }, + numberFormat, + }), hints: [readingHint(heatmap.scaleType), ...notes], } : { diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index a9b6761f05..0c7b7b8815 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -623,7 +623,7 @@ export async function runConfigTile( client: clickhouseClient, metadata: getMetadata(clickhouseClient), config: chartConfig, - granularity: options?.granularity ?? builderConfig.granularity, + granularity: options?.granularity, targetTimeBuckets: MCP_HEATMAP_TIME_BUCKETS, nBuckets: MCP_HEATMAP_VALUE_BUCKETS, querySettings: source.querySettings, diff --git a/packages/claude-plugin/hooks/chart.d.ts b/packages/claude-plugin/hooks/chart.d.ts index 94e6b16a98..b4ae0de7fe 100644 --- a/packages/claude-plugin/hooks/chart.d.ts +++ b/packages/claude-plugin/hooks/chart.d.ts @@ -10,6 +10,11 @@ export type Segment = { }; export function chartedTool(toolName: string): string | undefined; +export function isChartable( + toolName: string, + input: unknown, + output: unknown, +): boolean; export function renderChart( call: ChartCall, columns: number, diff --git a/packages/claude-plugin/hooks/register.tsx b/packages/claude-plugin/hooks/register.tsx index 40e0cb7d53..2e94e70294 100644 --- a/packages/claude-plugin/hooks/register.tsx +++ b/packages/claude-plugin/hooks/register.tsx @@ -8,7 +8,12 @@ import type { import type { ChartCall } from '../types'; -import { chartedTool, renderChart, stripControlChars } from './chart.js'; +import { + chartedTool, + isChartable, + renderChart, + stripControlChars, +} from './chart.js'; const KEEP = 20; const MAX_COLUMNS = 140; @@ -82,21 +87,23 @@ export const register: Register = on => { // The server fixes its default window before the query runs. const calledAt = Date.now(); const ran = await next(e); - if (!chartedTool(e.tool) || ran.deny !== undefined || ran.isError) - return ran; + if (ran.deny !== undefined || ran.isError) return ran; const call: ChartCall = { tool: e.tool, input: e, output: ran.result, calledAt, }; - // Chart bookkeeping must never fail the call it watched. + // Chart bookkeeping must never fail the call it watched. Only results that + // draw are kept, so query_tile calls on other tiles don't evict charts. try { - await update($, calls, all => - Object.fromEntries( - [...Object.entries(all), [e.tool_use_id, call]].slice(-KEEP), - ), - ); + if (isChartable(e.tool, e, ran.result)) { + await update($, calls, all => + Object.fromEntries( + [...Object.entries(all), [e.tool_use_id, call]].slice(-KEEP), + ), + ); + } } catch { // The row keeps the engine's own rendering. } diff --git a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts index 3ac0c12c8d..91e52fef40 100644 --- a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts +++ b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts @@ -3,6 +3,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { chartedTool, + isChartable, isTrimmed, parseResponse, toTileResult, @@ -172,6 +173,8 @@ describe('toTileResult', () => { it('turns a heatmap result into a grid, for the heatmap and query_tile tools', () => { const heatmap = { + valueExpression: 'Duration', + granularity: '1 minute', scale: 'log', numberFormat: { output: 'duration', factor: 1e-9 }, times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], @@ -219,4 +222,54 @@ describe('toTileResult', () => { ), ).toBeUndefined(); }); + + it('rejects a heatmap body that does not match the shared schema', () => { + const bad = (heatmap: object) => [ + { type: 'text', text: JSON.stringify({ heatmap }) }, + ]; + const body = { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + times: ['2026-10-13T00:00:00.000Z'], + valueEdges: [1, 10], + counts: [[0, 1, 0]], + }; + const tool = 'mcp__clickstack__clickstack_heatmap'; + expect(toTileResult(tool, {}, bad(body), calledAt)).toBeDefined(); + const { granularity: _omit, ...noGranularity } = body; + expect( + toTileResult(tool, {}, bad(noGranularity), calledAt), + ).toBeUndefined(); + expect( + toTileResult(tool, {}, bad({ ...body, counts: 'x' }), calledAt), + ).toBeUndefined(); + }); + + it('keeps only results it can draw for caching', () => { + const heatmapBody = [ + { + type: 'text', + text: JSON.stringify({ + heatmap: { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + times: ['2026-10-13T00:00:00.000Z'], + valueEdges: [1, 10], + counts: [[0, 1, 0]], + }, + }), + }, + ]; + expect( + isChartable('mcp__clickstack__clickstack_query_tile', {}, heatmapBody), + ).toBe(true); + expect( + isChartable('mcp__clickstack__clickstack_query_tile', {}, output), + ).toBe(false); + expect(isChartable('mcp__clickstack__clickstack_search', {}, output)).toBe( + false, + ); + }); }); diff --git a/packages/claude-plugin/src/index.ts b/packages/claude-plugin/src/index.ts index d627d028bb..43563c173b 100644 --- a/packages/claude-plugin/src/index.ts +++ b/packages/claude-plugin/src/index.ts @@ -1,3 +1,3 @@ export { stripControlChars } from './ansiLines'; export { renderChart } from './renderChart'; -export { chartedTool } from './toTileResult'; +export { chartedTool, isChartable } from './toTileResult'; diff --git a/packages/claude-plugin/src/toTileResult.ts b/packages/claude-plugin/src/toTileResult.ts index b9b1570e37..f16a1fd1c1 100644 --- a/packages/claude-plugin/src/toTileResult.ts +++ b/packages/claude-plugin/src/toTileResult.ts @@ -1,10 +1,12 @@ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; -import type { HeatmapGrid } from '@hyperdx/common-utils/dist/core/heatmap'; +import { + type HeatmapGrid, + type HeatmapResult, + HeatmapResultSchema, + heatmapRowEdges, +} from '@hyperdx/common-utils/dist/core/heatmap'; import { TIME_SERIES_DISPLAY_TYPE_BY_NAME } from '@hyperdx/common-utils/dist/core/utils'; -import type { - ChartConfigWithDateRange, - NumberFormat, -} from '@hyperdx/common-utils/dist/types'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { DisplayType } from '@hyperdx/common-utils/dist/types'; import type { TileQueryResult } from '@hyperdx/cli/src/shared/tileQuery'; @@ -36,14 +38,6 @@ type ToolInput = { endTime?: string; }; -type HeatmapBody = { - numberFormat?: NumberFormat; - times: string[]; - valueEdges: number[]; - counts: number[][]; - scale: 'log' | 'linear'; -}; - type ChartResult = Extract; /** Matches by MCP tool name suffix, so it works whatever the user named the server. */ @@ -69,6 +63,15 @@ function parseBody(output: unknown): Record | undefined { } } +/** Whether a call's result can be drawn, so only those are kept for drawing. */ +export function isChartable( + toolName: string, + input: unknown, + output: unknown, +): boolean { + return toTileResult(toolName, input, output, Date.now()) !== undefined; +} + /** The server trims results over its size limit and says so in `note`. */ export function isTrimmed(output: unknown): boolean { return typeof parseBody(output)?.note === 'string'; @@ -103,8 +106,9 @@ export function toTileResult( : new Date(end.getTime() - DEFAULT_WINDOW_MS); if (HEATMAP_TOOLS.includes(tool)) { - const heatmap = parseBody(output)?.heatmap as HeatmapBody | undefined; - if (!heatmap?.counts?.length) return undefined; + const parsed = HeatmapResultSchema.safeParse(parseBody(output)?.heatmap); + if (!parsed.success || parsed.data.counts.length === 0) return undefined; + const heatmap = parsed.data; return { status: 'heatmap', queriedConfig: { @@ -147,22 +151,16 @@ function toGrid({ valueEdges, counts, scale, -}: HeatmapBody): HeatmapGrid { +}: HeatmapResult): HeatmapGrid { const ms = times.map(t => Date.parse(t)); return { times: ms, stepMs: ms.length > 1 ? ms[1]! - ms[0]! : 0, - yAxis: { type: 'numeric', scale, edges: withOuterRows(valueEdges, scale) }, + yAxis: { + type: 'numeric', + scale, + edges: heatmapRowEdges(valueEdges, scale), + }, cells: counts.flat(), }; } - -// The server reports the bucket bounds, with an under- and overflow count on -// either side; give those two rows one bucket's height so they can be drawn. -function withOuterRows(edges: number[], scale: 'log' | 'linear'): number[] { - const [first, second] = [edges[0]!, edges[1] ?? edges[0]!]; - const [beforeLast, last] = [edges.at(-2) ?? edges.at(-1)!, edges.at(-1)!]; - return scale === 'log' && first > 0 - ? [(first * first) / second, ...edges, (last * last) / beforeLast] - : [first - (second - first), ...edges, last + (last - beforeLast)]; -} diff --git a/packages/claude-plugin/test/render.test.tsx b/packages/claude-plugin/test/render.test.tsx index b99e4eabb2..ea890b7872 100644 --- a/packages/claude-plugin/test/render.test.tsx +++ b/packages/claude-plugin/test/render.test.tsx @@ -132,6 +132,8 @@ test( test('draws a heatmap result as shaded cells', { options }, async ($, on) => { const heatmap = { + valueExpression: 'Duration', + granularity: '1 minute', scale: 'log', times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], valueEdges: [1, 10, 100, 1000], diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index 84de88d0c1..8a22c1d84a 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -184,7 +184,8 @@ same pipeline in three modes: dashboard tiles (`-d`), ad-hoc builder charts `shared/adhocChart.ts`), and ad-hoc raw SQL (`--sql` with `$__timeFilter` / `$__timeInterval` macros). ANSI colors are stripped automatically when stdout is not a TTY (`--color auto|always|never`), and `--json` emits raw rows + column -metadata. +metadata (distribution heatmap tiles emit a `heatmap` object in the same shape +as the MCP `clickstack_heatmap` result). Do NOT change resolution/shaping rules without checking the web components first (see the alignment table below). diff --git a/packages/cli/src/__tests__/tileQuery.test.ts b/packages/cli/src/__tests__/tileQuery.test.ts index 0dc15a6150..393791b2e8 100644 --- a/packages/cli/src/__tests__/tileQuery.test.ts +++ b/packages/cli/src/__tests__/tileQuery.test.ts @@ -79,4 +79,66 @@ describe('fetchTileData heatmap tiles', () => { expect(result.status).toBe('unsupported'); expect(queryChartConfig).not.toHaveBeenCalled(); }); + + it('uses the saved granularity and labels rows with widthBucket bounds', async () => { + const queryChartConfig = jest + .fn<() => Promise>() + .mockResolvedValueOnce({ meta: [], data: [{ min: '0', max: '100' }] }) + .mockResolvedValueOnce({ + meta: [{ name: '__hdx_time_bucket', type: 'DateTime' }], + data: [], + }); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile({ + granularity: '1 minute', + select: [ + { + aggFn: 'count', + valueExpression: 'Duration', + heatmapScaleType: 'linear', + }, + ], + }), + source: traceSource, + dateRange, + }); + + const bucketQuery = queryChartConfig.mock.calls[1] as unknown as [ + { config: { granularity: string } }, + ]; + expect(bucketQuery[0].config.granularity).toBe('1 minute'); + if (result.status !== 'heatmap') throw new Error(result.status); + // 40 buckets over 0..100 are 2.5 wide; the underflow row starts one below 0. + expect(result.grid?.yAxis).toMatchObject({ + type: 'numeric', + scale: 'linear', + }); + const edges = + result.grid?.yAxis.type === 'numeric' ? result.grid.yAxis.edges : []; + expect(edges.slice(0, 3)).toEqual([-2.5, 0, 2.5]); + expect(result.heatmap).toMatchObject({ + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'linear', + }); + }); + + it('refuses distribution heatmaps on non-trace sources', async () => { + const queryChartConfig = jest.fn(); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile(), + source: { ...traceSource, kind: 'log' } as unknown as SourceResponse, + dateRange, + }); + expect(result).toMatchObject({ status: 'unsupported' }); + expect(queryChartConfig).not.toHaveBeenCalled(); + }); }); diff --git a/packages/cli/src/__tests__/tileRender.test.ts b/packages/cli/src/__tests__/tileRender.test.ts index d2c55574c6..f348cac839 100644 --- a/packages/cli/src/__tests__/tileRender.test.ts +++ b/packages/cli/src/__tests__/tileRender.test.ts @@ -120,4 +120,31 @@ describe('renderTileContent', () => { ), ).toContain('Not enough data points'); }); + + it('formats a saved Duration heatmap as durations without a numberFormat', () => { + const result: TileQueryResult = { + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + dateRange, + select: [{ aggFn: 'count', valueExpression: 'Duration' }], + } as unknown as ChartConfigWithDateRange, + grid: { + times: [dateRange[0].getTime()], + stepMs: 0, + yAxis: { type: 'numeric', scale: 'log', edges: [1e6, 1e7, 1e8, 1e9] }, + cells: [3, 5, 1], + }, + }; + const traceSource = { + kind: 'trace', + durationExpression: 'Duration', + durationPrecision: 9, + } as never; + const output = stripAnsi( + renderTileContent({ result, source: traceSource, width: 60, height: 12 }), + ); + expect(output).toMatch(/\d+(ms|s)\b/); + expect(output).not.toContain('1.0B'); + }); }); diff --git a/packages/cli/src/cli.tsx b/packages/cli/src/cli.tsx index 31744ffeb7..46bc08c3eb 100644 --- a/packages/cli/src/cli.tsx +++ b/packages/cli/src/cli.tsx @@ -1372,6 +1372,9 @@ Examples: ...(result.status === 'ok' ? { data: result.data.data, meta: result.data.meta } : {}), + ...(result.status === 'heatmap' + ? { heatmap: result.heatmap ?? null } + : {}), ...(result.status === 'unsupported' ? { message: result.message } : {}), diff --git a/packages/cli/src/shared/formatNumber.ts b/packages/cli/src/shared/formatNumber.ts index 9d51af73a8..7deaeb3901 100644 --- a/packages/cli/src/shared/formatNumber.ts +++ b/packages/cli/src/shared/formatNumber.ts @@ -576,3 +576,21 @@ export function axisTickFormatter( return ''; }; } + +/** + * A heatmap's axis format. Saved heatmap tiles carry aggFn count, which hides + * a duration value expression from the series check, so check it directly. + */ +export function heatmapNumberFormat( + config: ChartConfigWithOptDateRange, + source: SourceResponse | undefined, +): NumberFormat | undefined { + const valueExpression = + isBuilderChartConfig(config) && Array.isArray(config.select) + ? config.select[0]?.valueExpression + : undefined; + return ( + resolveChartNumberFormats(config, source).chartFormat ?? + getTraceDurationNumberFormat(source, { valueExpression }) + ); +} diff --git a/packages/cli/src/shared/tileQuery.ts b/packages/cli/src/shared/tileQuery.ts index 6271241307..e61423a155 100644 --- a/packages/cli/src/shared/tileQuery.ts +++ b/packages/cli/src/shared/tileQuery.ts @@ -8,17 +8,24 @@ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { getHeatmapMode, type HeatmapGrid, + type HeatmapResult, + heatmapRowEdges, + toHeatmapResult, } from '@hyperdx/common-utils/dist/core/heatmap'; import { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + isBuilderChartConfig, + isHeatmapCompatibleSource, +} from '@hyperdx/common-utils/dist/guards'; import type { ChartConfigWithDateRange, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; import type { ProxyClickhouseClient, SourceResponse } from '@/api/client'; +import { heatmapNumberFormat } from '@/shared/formatNumber'; import { CLI_SUPPORTED_DISPLAY_TYPES, convertTileConfigForQuery, @@ -38,6 +45,8 @@ export type TileQueryResult = queriedConfig: ChartConfigWithDateRange; /** Undefined when the range has too few distinct values to bucket */ grid: HeatmapGrid | undefined; + /** The same grid in the shape the MCP server returns, for --json */ + heatmap?: HeatmapResult; } | { status: 'markdown'; @@ -117,20 +126,55 @@ export async function fetchTileData({ message: 'Series heatmap tiles are not supported in the CLI yet.', }; } + // The CLI's source kind is a string; map it onto SourceKind for the shared guard. + const sourceKind = Object.values(SourceKind).find(k => k === source?.kind); + if ( + source && + !(sourceKind && isHeatmapCompatibleSource({ kind: sourceKind })) + ) { + return { + status: 'unsupported', + message: `Distribution heatmaps need a trace source; "${source.name}" is a ${source.kind} source.`, + }; + } const result = await queryDistributionHeatmap({ client: clickhouseClient, metadata, config: heatmapConfig, - granularity: heatmapConfig.granularity, + // resolveTileConfig swaps in the command's granularity, which is unset + // by default; fall back to the tile's own. + granularity: + granularity ?? + ('granularity' in config ? config.granularity : undefined), targetTimeBuckets: maxTimeBuckets, nBuckets: HEATMAP_VALUE_BUCKETS, querySettings: source?.querySettings, opts: { abort_signal: signal }, }); + const { heatmap } = result; + const firstSelect = Array.isArray(heatmapConfig.select) + ? heatmapConfig.select[0] + : undefined; return { status: 'heatmap', queriedConfig: heatmapConfig, - grid: result.heatmap?.grid, + // Label rows with the bucket bounds widthBucket counted, not the + // web's centred display edges. + grid: heatmap && { + ...heatmap.grid, + yAxis: { + type: 'numeric', + scale: heatmap.scaleType, + edges: heatmapRowEdges(heatmap.bucketEdges, heatmap.scaleType), + }, + }, + heatmap: + heatmap && + toHeatmapResult(heatmap, { + valueExpression: firstSelect?.valueExpression ?? '', + granularity: result.granularity, + numberFormat: heatmapNumberFormat(heatmapConfig, source), + }), }; } diff --git a/packages/cli/src/shared/tileRender.ts b/packages/cli/src/shared/tileRender.ts index a50a3ab340..556850e005 100644 --- a/packages/cli/src/shared/tileRender.ts +++ b/packages/cli/src/shared/tileRender.ts @@ -24,6 +24,7 @@ import { import { axisTickFormatter, formatNumber, + heatmapNumberFormat, resolveChartNumberFormats, resolveSingleSeriesNumberFormat, } from '@/shared/formatNumber'; @@ -67,15 +68,12 @@ export function renderTileContent({ if (result.status === 'heatmap') { const data = result.grid && formatHeatmapGrid(result.grid); if (!data) return chalk.dim('Not enough data points to draw a heatmap.'); - const { chartFormat } = resolveChartNumberFormats( - result.queriedConfig, - source, - ); + const numberFormat = heatmapNumberFormat(result.queriedConfig, source); return renderHeatmapChart({ data, width, height, - formatTick: axisTickFormatter(chartFormat), + formatTick: axisTickFormatter(numberFormat), }); } diff --git a/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts index 6ad277853c..a2bf4bd377 100644 --- a/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts +++ b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts @@ -1,7 +1,10 @@ import { queryDistributionHeatmap } from '@/core/distributionHeatmap'; import { heatmapBucketEdges, + HeatmapResultSchema, + heatmapRowEdges, resolveDistributionGranularity, + toHeatmapResult, } from '@/core/heatmap'; import { timeBucketByGranularity } from '@/core/utils'; import { DisplayType } from '@/types'; @@ -120,3 +123,35 @@ describe('timeBucketByGranularity', () => { ).toEqual([]); }); }); + +describe('heatmapRowEdges', () => { + it('adds an under- and overflow row one bucket tall on each side', () => { + expect(heatmapRowEdges([0, 25, 50], 'linear')).toEqual([ + -25, 0, 25, 50, 75, + ]); + expect(heatmapRowEdges([1, 10, 100], 'log')).toEqual([ + 0.1, 1, 10, 100, 1000, + ]); + }); +}); + +describe('toHeatmapResult', () => { + it('keeps exact bucket edges and fits the shared schema', () => { + const result = toHeatmapResult( + { + grid: { + times: [Date.parse('2026-10-13T00:00:00Z')], + stepMs: 0, + yAxis: { type: 'numeric', scale: 'linear', edges: [0, 1, 2, 3, 4] }, + cells: [0, 1, 2, 0], + }, + scaleType: 'linear', + bucketEdges: [1000000000, 1000000050, 1000000100], + }, + { valueExpression: 'Duration', granularity: '1 minute' }, + ); + expect(result.valueEdges).toEqual([1000000000, 1000000050, 1000000100]); + expect(result.counts).toEqual([[0, 1, 2, 0]]); + expect(HeatmapResultSchema.safeParse(result).success).toBe(true); + }); +}); diff --git a/packages/common-utils/src/core/heatmap.ts b/packages/common-utils/src/core/heatmap.ts index 551b0a955a..364c950515 100644 --- a/packages/common-utils/src/core/heatmap.ts +++ b/packages/common-utils/src/core/heatmap.ts @@ -1,9 +1,11 @@ +import { z } from 'zod'; + import type { ColumnMetaType } from '@/clickhouse'; import { convertDateRangeToGranularityString, convertGranularityToSeconds, } from '@/core/utils'; -import type { PromqlHeatmapMode, SQLInterval } from '@/types'; +import type { NumberFormat, PromqlHeatmapMode, SQLInterval } from '@/types'; import { SQLIntervalSchema } from '@/types'; /** A heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ @@ -105,6 +107,63 @@ export function heatmapBucketEdges( return Array.from({ length: nBuckets + 1 }, (_, j) => bucketValue(j)); } +/** + * Row edges for drawing the `nBuckets + 2` rows: the bucket bounds, with the + * under- and overflow rows each given one bucket's height. + */ +export function heatmapRowEdges( + bucketEdges: number[], + scaleType: HeatmapScaleType, +): number[] { + const [first, second] = [bucketEdges[0]!, bucketEdges[1] ?? bucketEdges[0]!]; + const last = bucketEdges.at(-1)!; + const beforeLast = bucketEdges.at(-2) ?? last; + return scaleType === 'log' && first > 0 + ? [(first * first) / second, ...bucketEdges, (last * last) / beforeLast] + : [first - (second - first), ...bucketEdges, last + (last - beforeLast)]; +} + +/** + * A distribution heatmap as returned to agents and scripts: `counts[t][r]`, + * where row 0 counts values below `valueEdges[0]`, row r counts + * [valueEdges[r - 1], valueEdges[r]) and the last row values at or above the + * last edge. + */ +export const HeatmapResultSchema = z.object({ + valueExpression: z.string(), + scale: z.enum(['log', 'linear']), + granularity: z.string(), + // Typed but not re-validated: NumberFormatSchema drags every types.ts schema + // into the Claude plugin's bundle, which then fails to load in Claude Code. + numberFormat: z + .custom(v => typeof v === 'object' && v !== null) + .optional(), + times: z.array(z.string()), + valueEdges: z.array(z.number()), + counts: z.array(z.array(z.number())), +}); +export type HeatmapResult = z.infer; + +export function toHeatmapResult( + heatmap: { + grid: HeatmapGrid; + scaleType: HeatmapScaleType; + bucketEdges: number[]; + }, + context: Pick< + HeatmapResult, + 'valueExpression' | 'granularity' | 'numberFormat' + >, +): HeatmapResult { + return { + ...context, + scale: heatmap.scaleType, + times: heatmap.grid.times.map(t => new Date(t).toISOString()), + valueEdges: heatmap.bucketEdges, + counts: heatmapCountColumns(heatmap.grid), + }; +} + /** Cells as `[timeIdx][rowIdx]`. */ export function heatmapCountColumns(grid: HeatmapGrid): number[][] { const rows = heatmapRowCount(grid.yAxis); From 29ff90d6d8af306402c34523f32b8f8b4d41d353 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Tue, 20 Oct 2026 21:32:27 +1100 Subject: [PATCH 3/3] fix(mcp): draw saved heatmap tiles at their saved granularity query_tile and query_tiles read tiles in the external format, which has no granularity, so saved distribution heatmaps fell back to auto buckets. Also declare the heatmap tool's annotations with READ_ONLY_TOOL, which main now requires. --- .../src/mcp/__tests__/heatmapTool.int.test.ts | 43 +++++++++++++++++++ .../api/src/mcp/tools/dashboards/queryTile.ts | 11 ++++- .../src/mcp/tools/dashboards/queryTiles.ts | 10 ++++- packages/api/src/mcp/tools/query/heatmap.ts | 3 +- packages/api/src/mcp/tools/query/helpers.ts | 13 +++++- 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts index 16a580f625..aaf03116f7 100644 --- a/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts @@ -13,6 +13,7 @@ import { } from '@/fixtures'; import { McpContext } from '@/mcp/tools/types'; import Connection from '@/models/connection'; +import Dashboard from '@/models/dashboard'; import { Source } from '@/models/source'; import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; @@ -20,6 +21,7 @@ import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; type HeatmapBody = { heatmap: { scale: string; + granularity: string; numberFormat?: { output: string; factor: number }; times: string[]; valueEdges: number[]; @@ -205,6 +207,47 @@ describe('MCP Heatmap Tool', () => { expect(body.heatmap!.counts.flat().reduce((a, b) => a + b, 0)).toBe(55); }); + it('uses the granularity saved on the tile', async () => { + const saved = await callTool(client, 'clickstack_save_dashboard', { + name: 'Latency', + tiles: [ + { + name: 'Span latency', + config: { + displayType: 'heatmap', + sourceId: traceSourceId, + select: [{ valueExpression: 'Duration' }], + where, + }, + }, + ], + }); + const dashboard = JSON.parse(getFirstText(saved)); + // The external tile format has no granularity; the web editor stores one. + await Dashboard.updateOne( + { _id: dashboard.id }, + { $set: { 'tiles.0.config.granularity': '1 minute' } }, + ); + + const range = { + dashboardId: dashboard.id, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + }; + const single = await callTool(client, 'clickstack_query_tile', { + ...range, + tileId: dashboard.tiles[0].id, + }); + const body: HeatmapBody = JSON.parse(getFirstText(single)); + expect(body.heatmap!.granularity).toBe('1 minute'); + + // A heatmap summary's rowCount is its number of time buckets. + const batch = await callTool(client, 'clickstack_query_tiles', range); + const [summary] = JSON.parse(getFirstText(batch)).tiles; + expect(summary.rowCount).toBe(body.heatmap!.times.length); + expect(summary.rowCount).toBeLessThanOrEqual(11); + }); + it('rejects a non-trace source', async () => { const result = await callTool(client, 'clickstack_heatmap', { sourceId: logSourceId, diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index c05153c190..dc40302d18 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -1,7 +1,11 @@ import { z } from 'zod'; import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; -import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers'; +import { + parseTimeRange, + runConfigTile, + savedTileGranularity, +} from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; import Dashboard from '@/models/dashboard'; @@ -96,7 +100,10 @@ export function registerQueryTile({ tile, startDate, endDate, - { variables: resolvedVariables.variables }, + { + variables: resolvedVariables.variables, + savedGranularity: savedTileGranularity(dashboard.tiles, tileId), + }, ); // Surface non-blocking missing macro and variable warnings alongside diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index d18a6b74ab..12d46bfa1a 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -2,7 +2,11 @@ import PQueue from '@esm2cjs/p-queue'; import { z } from 'zod'; import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; -import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers'; +import { + parseTimeRange, + runConfigTile, + savedTileGranularity, +} from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; import { runWithTimeout } from '@/mcp/utils/timeout'; @@ -323,6 +327,10 @@ export function registerQueryTiles({ runConfigTile(teamId.toString(), tile, startDate, endDate, { abortSignal: signal, variables, + savedGranularity: savedTileGranularity( + dashboard.tiles, + tile.id, + ), }), deadlineAt, ); diff --git a/packages/api/src/mcp/tools/query/heatmap.ts b/packages/api/src/mcp/tools/query/heatmap.ts index 4978694bba..40af927589 100644 --- a/packages/api/src/mcp/tools/query/heatmap.ts +++ b/packages/api/src/mcp/tools/query/heatmap.ts @@ -1,6 +1,7 @@ import { SQLIntervalSchema } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; +import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; @@ -53,7 +54,7 @@ export function registerHeatmap({ context, registerTool }: ToolRegistrar) { 'clickstack_heatmap', { title: 'Distribution Heatmap', - annotations: { readOnlyHint: true }, + annotations: READ_ONLY_TOOL, description: 'Show how a numeric value is distributed over time: counts of events per ' + 'time bucket and value range, as on a latency heatmap. Use this to see ' + diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 0c7b7b8815..7ede761fa5 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -21,6 +21,7 @@ import type { ChartConfigWithDateRange, ChartVariable, MetricTable, + Tile, } from '@hyperdx/common-utils/dist/types'; import { DisplayType, @@ -415,6 +416,14 @@ function toPlainMetricTables( : metricTables; } +export function savedTileGranularity( + tiles: Tile[], + tileId: string, +): string | undefined { + const config = tiles.find(t => t.id === tileId)?.config; + return config && 'granularity' in config ? config.granularity : undefined; +} + export async function runConfigTile( teamId: string, tile: ExternalDashboardTileWithId, @@ -423,6 +432,8 @@ export async function runConfigTile( options?: { maxResults?: number; granularity?: string; + /** The stored tile's granularity, which the external tile format drops. Distribution heatmaps only. */ + savedGranularity?: string; abortSignal?: AbortSignal; /** Dashboard variables and their selected values. Omitted when not in a dashboard context. */ variables?: ChartVariable[]; @@ -623,7 +634,7 @@ export async function runConfigTile( client: clickhouseClient, metadata: getMetadata(clickhouseClient), config: chartConfig, - granularity: options?.granularity, + granularity: options?.granularity ?? options?.savedGranularity, targetTimeBuckets: MCP_HEATMAP_TIME_BUCKETS, nBuckets: MCP_HEATMAP_VALUE_BUCKETS, querySettings: source.querySettings,