diff --git a/.changeset/claude-charts-plugin.md b/.changeset/claude-charts-plugin.md index e0744e9e6a..a4d28a09ab 100644 --- a/.changeset/claude-charts-plugin.md +++ b/.changeset/claude-charts-plugin.md @@ -3,6 +3,6 @@ --- feat: add a Claude Code plugin that connects the ClickStack MCP server and draws -`clickstack_timeseries` and `clickstack_table` results as terminal charts, using -the CLI's renderers. Install with +`clickstack_timeseries`, `clickstack_table` and `clickstack_heatmap` results as +terminal charts, using the CLI's renderers. Install with `/plugin install hyperdx-charts --marketplace hyperdxio/hyperdx`. diff --git a/.changeset/cli-heatmap-tiles.md b/.changeset/cli-heatmap-tiles.md new file mode 100644 index 0000000000..f2d83ec9c3 --- /dev/null +++ b/.changeset/cli-heatmap-tiles.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/cli': patch +--- + +feat: `hdx` draws distribution heatmap tiles as shaded terminal charts. diff --git a/.changeset/mcp-distribution-heatmap.md b/.changeset/mcp-distribution-heatmap.md new file mode 100644 index 0000000000..a1ad6130d0 --- /dev/null +++ b/.changeset/mcp-distribution-heatmap.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': patch +'@hyperdx/common-utils': patch +--- + +feat: add a `clickstack_heatmap` MCP tool that returns how a value such as span +latency is distributed over time, as counts per time bucket and value range. +`clickstack_query_tile` now returns the same grid for distribution heatmap +tiles instead of failing. The distribution heatmap queries the web app runs +moved to common-utils so the API and CLI share them. diff --git a/MCP.md b/MCP.md index 2065874d08..1c9f1aa18f 100644 --- a/MCP.md +++ b/MCP.md @@ -136,6 +136,7 @@ with: | `clickstack_describe_metric` | Per-metric drill-down: kind(s), unit, description, attribute keys per map column, and sampled values | | `clickstack_timeseries` | Plot metrics over time as a line or stacked bar chart (works on log, trace, and metric sources) | | `clickstack_table` | Compute aggregated metrics as a table, single number, or pie chart (works on log, trace, and metric sources) | +| `clickstack_heatmap` | Show how a value such as span latency is distributed over time (trace sources) | | `clickstack_search` | Browse individual log, event, or trace rows | | `clickstack_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | | `clickstack_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts index 5708f699f9..0140dcdaaa 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTilesSummarizeRows.test.ts @@ -103,4 +103,19 @@ describe('withDeadline', () => { await expect(withDeadline(work, Date.now() + 1000)).resolves.toBe('done'); expect(captured?.aborted).toBe(true); }); + + it('summarizes a distribution heatmap by its time buckets', () => { + const counts = [ + [0, 2, 0], + [0, 0, 0], + ]; + expect(summarizeRows(JSON.stringify({ heatmap: { counts } }))).toEqual({ + hasData: true, + rowCount: 2, + }); + expect(summarizeRows(JSON.stringify({ heatmap: null }))).toEqual({ + hasData: false, + rowCount: 0, + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts new file mode 100644 index 0000000000..aaf03116f7 --- /dev/null +++ b/packages/api/src/mcp/__tests__/heatmapTool.int.test.ts @@ -0,0 +1,259 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { ObjectId } from 'mongodb'; + +import * as config from '@/config'; +import { + bulkInsertData, + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import { McpContext } from '@/mcp/tools/types'; +import Connection from '@/models/connection'; +import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; + +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +type HeatmapBody = { + heatmap: { + scale: string; + granularity: string; + numberFormat?: { output: string; factor: number }; + times: string[]; + valueEdges: number[]; + counts: number[][]; + } | null; + hints: string[]; +}; + +describe('MCP Heatmap Tool', () => { + const server = getServer(); + let client: Client; + let traceSourceId: string; + let logSourceId: string; + // otel_traces is not truncated between tests, so each test filters to its own spans. + let where: string; + const now = Date.now(); + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const { team, user } = await getLoggedInAgent(server); + const connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_TRACES_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + durationExpression: 'Duration', + durationPrecision: 9, + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + }); + const logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + }); + traceSourceId = traceSource._id.toString(); + logSourceId = logSource._id.toString(); + + const service = `heatmap-${new ObjectId().toString()}`; + where = `ServiceName:"${service}"`; + + // A fast mode (~1ms) and a slow tail (~1s), so the heatmap is bimodal. + const durations = [ + ...Array.from({ length: 50 }, (_, i) => 1e6 + i * 1e3), + ...Array.from({ length: 5 }, (_, i) => 1e9 + i * 1e6), + ]; + await bulkInsertData( + `${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE}`, + durations.map((Duration, i) => ({ + Timestamp: new Date(now - (i % 5) * 60_000 - 30_000), + TraceId: `trace-${i}`, + SpanId: `span-${i}`, + ParentSpanId: '', + SpanName: 'GET /api', + SpanKind: 'Server', + ServiceName: service, + Duration, + StatusCode: 'Ok', + StatusMessage: '', + SpanAttributes: {}, + ResourceAttributes: {}, + })), + ); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + it('returns counts per time bucket and value range', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + where, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + granularity: '1 minute', + }); + + expect(result.isError).toBeFalsy(); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + const heatmap = body.heatmap!; + expect(heatmap.scale).toBe('log'); + expect(heatmap.numberFormat).toEqual({ output: 'duration', factor: 1e-9 }); + expect(heatmap.counts).toHaveLength(heatmap.times.length); + // One count per bucket plus an under- and overflow count. + for (const column of heatmap.counts) { + expect(column).toHaveLength(heatmap.valueEdges.length + 1); + } + expect(heatmap.counts.flat().reduce((a, b) => a + b, 0)).toBe(55); + + // The slow tail lands in a higher value range than the fast mode. + const rowTotals = heatmap.counts.reduce((totals, column) => + totals.map((total, r) => total + (column.at(r) ?? 0)), + ); + const populated = rowTotals.flatMap((n, r) => (n > 0 ? [r] : [])); + // Row r counts [valueEdges[r - 1], valueEdges[r]). + expect(heatmap.valueEdges[populated.at(-1)! - 1]).toBeGreaterThan(1e8); + expect(heatmap.valueEdges[populated[0]]).toBeLessThan(1e7); + }); + + it('rejects a granularity the time buckets cannot step by', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + granularity: '1 week', + }); + expect(result.isError).toBe(true); + }); + + it('coarsens a granularity that would flood the grid, and says so', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: traceSourceId, + valueExpression: 'Duration', + where, + startTime: new Date(now - 24 * 60 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + granularity: '1 second', + }); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + expect(body.heatmap!.times.length).toBeLessThanOrEqual(48 * 4); + expect(body.hints.join(' ')).toContain('Granularity "1 second" would make'); + }); + + it('answers clickstack_query_tile for a saved distribution heatmap tile', async () => { + const saved = await callTool(client, 'clickstack_save_dashboard', { + name: 'Latency', + tiles: [ + { + name: 'Span latency', + config: { + displayType: 'heatmap', + sourceId: traceSourceId, + select: [{ valueExpression: 'Duration' }], + where, + }, + }, + ], + }); + expect(saved.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(saved)); + + const result = await callTool(client, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + }); + expect(result.isError).toBeFalsy(); + const body: HeatmapBody = JSON.parse(getFirstText(result)); + expect(body.heatmap!.counts.flat().reduce((a, b) => a + b, 0)).toBe(55); + }); + + it('uses the granularity saved on the tile', async () => { + const saved = await callTool(client, 'clickstack_save_dashboard', { + name: 'Latency', + tiles: [ + { + name: 'Span latency', + config: { + displayType: 'heatmap', + sourceId: traceSourceId, + select: [{ valueExpression: 'Duration' }], + where, + }, + }, + ], + }); + const dashboard = JSON.parse(getFirstText(saved)); + // The external tile format has no granularity; the web editor stores one. + await Dashboard.updateOne( + { _id: dashboard.id }, + { $set: { 'tiles.0.config.granularity': '1 minute' } }, + ); + + const range = { + dashboardId: dashboard.id, + startTime: new Date(now - 10 * 60_000).toISOString(), + endTime: new Date(now).toISOString(), + }; + const single = await callTool(client, 'clickstack_query_tile', { + ...range, + tileId: dashboard.tiles[0].id, + }); + const body: HeatmapBody = JSON.parse(getFirstText(single)); + expect(body.heatmap!.granularity).toBe('1 minute'); + + // A heatmap summary's rowCount is its number of time buckets. + const batch = await callTool(client, 'clickstack_query_tiles', range); + const [summary] = JSON.parse(getFirstText(batch)).tiles; + expect(summary.rowCount).toBe(body.heatmap!.times.length); + expect(summary.rowCount).toBeLessThanOrEqual(11); + }); + + it('rejects a non-trace source', async () => { + const result = await callTool(client, 'clickstack_heatmap', { + sourceId: logSourceId, + valueExpression: 'Duration', + }); + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('need a trace source'); + }); +}); diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index c05153c190..dc40302d18 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -1,7 +1,11 @@ import { z } from 'zod'; import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; -import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers'; +import { + parseTimeRange, + runConfigTile, + savedTileGranularity, +} from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; import Dashboard from '@/models/dashboard'; @@ -96,7 +100,10 @@ export function registerQueryTile({ tile, startDate, endDate, - { variables: resolvedVariables.variables }, + { + variables: resolvedVariables.variables, + savedGranularity: savedTileGranularity(dashboard.tiles, tileId), + }, ); // Surface non-blocking missing macro and variable warnings alongside diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 443076db76..12d46bfa1a 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -2,7 +2,11 @@ import PQueue from '@esm2cjs/p-queue'; import { z } from 'zod'; import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; -import { parseTimeRange, runConfigTile } from '@/mcp/tools/query/helpers'; +import { + parseTimeRange, + runConfigTile, + savedTileGranularity, +} from '@/mcp/tools/query/helpers'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; import { runWithTimeout } from '@/mcp/utils/timeout'; @@ -127,6 +131,23 @@ export function summarizeRows(text: string): { } catch { return {}; } + if (parsed != null && typeof parsed === 'object' && 'heatmap' in parsed) { + // A distribution heatmap: one row per time bucket, null when empty. + const { heatmap } = parsed; + const counts: unknown[] = + heatmap != null && + typeof heatmap === 'object' && + 'counts' in heatmap && + Array.isArray(heatmap.counts) + ? heatmap.counts + : []; + return { + hasData: counts.some( + column => Array.isArray(column) && column.some(n => n > 0), + ), + rowCount: counts.length, + }; + } if (parsed == null || typeof parsed !== 'object' || !('result' in parsed)) { return {}; } @@ -306,6 +327,10 @@ export function registerQueryTiles({ runConfigTile(teamId.toString(), tile, startDate, endDate, { abortSignal: signal, variables, + savedGranularity: savedTileGranularity( + dashboard.tiles, + tile.id, + ), }), deadlineAt, ); diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts index c4b1a77a91..6988b9d0b3 100644 --- a/packages/api/src/mcp/tools/query/builderCatalog.ts +++ b/packages/api/src/mcp/tools/query/builderCatalog.ts @@ -8,6 +8,7 @@ export const BUILDER_TOOLS_LIST = [ ' • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns', ' • clickstack_timeseries — trends / metrics over time', + ' • clickstack_heatmap — how a value (e.g. latency) is distributed over time', ' • clickstack_search — browsing individual log/trace rows', ' • clickstack_event_patterns — recurring log/event pattern discovery', ' • clickstack_event_deltas — attributes that differ between two row groups', diff --git a/packages/api/src/mcp/tools/query/heatmap.ts b/packages/api/src/mcp/tools/query/heatmap.ts new file mode 100644 index 0000000000..40af927589 --- /dev/null +++ b/packages/api/src/mcp/tools/query/heatmap.ts @@ -0,0 +1,96 @@ +import { SQLIntervalSchema } from '@hyperdx/common-utils/dist/types'; +import { z } from 'zod'; + +import { READ_ONLY_TOOL } from '@/mcp/tools/annotations'; +import type { ToolRegistrar } from '@/mcp/tools/types'; +import { mcpUserError } from '@/mcp/utils/errors'; + +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; +import { + endTimeSchema, + sourceIdSchema, + startTimeSchema, + whereLanguageSchema, + whereSchema, +} from './schemas'; + +const heatmapSchema = z.object({ + sourceId: sourceIdSchema.describe( + 'Trace source ID. Distribution heatmaps need a trace source; call clickstack_list_sources to find one.', + ), + valueExpression: z + .string() + .min(1) + .describe( + 'Numeric column or expression to bucket, e.g. "Duration" for span latency. ' + + 'Top-level columns are PascalCase; map attributes use bracket syntax.', + ), + countExpression: z + .string() + .optional() + .describe('What each cell counts. Default: count() (one per span).'), + scale: z + .enum(['log', 'linear']) + .optional() + .default('log') + .describe( + 'Value axis scale. "log" (default) suits latency, which spans orders of magnitude.', + ), + where: whereSchema.describe('Row filter, e.g. restrict to one service.'), + whereLanguage: whereLanguageSchema, + granularity: SQLIntervalSchema.optional().describe( + 'Time bucket size, " " where unit is second, minute, hour or day ' + + '(e.g. "5 minute"). Omit to pick one for ~48 buckets; a size that would make ' + + 'far more buckets is coarsened, with a hint.', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +export function registerHeatmap({ context, registerTool }: ToolRegistrar) { + const { teamId } = context; + + registerTool( + 'clickstack_heatmap', + { + title: 'Distribution Heatmap', + annotations: READ_ONLY_TOOL, + description: + 'Show how a numeric value is distributed over time: counts of events per ' + + 'time bucket and value range, as on a latency heatmap. Use this to see ' + + 'bimodal latency, a slow tail appearing, or a shift in the whole ' + + 'distribution, which a single percentile line hides. ' + + 'Requires a trace source. Returns counts[t][r] with the time buckets and ' + + 'value-range edges they belong to.', + inputSchema: heatmapSchema, + }, + async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return mcpUserError(timeRange.error); + } + + const tile = buildTile('MCP Heatmap', 12, 6, { + displayType: 'heatmap', + sourceId: input.sourceId, + select: [ + { + valueExpression: input.valueExpression, + countExpression: input.countExpression, + heatmapScaleType: input.scale, + }, + ], + where: input.where, + whereLanguage: input.whereLanguage, + }); + + return runConfigTile( + teamId.toString(), + tile, + timeRange.startDate, + timeRange.endDate, + { granularity: input.granularity }, + ); + }, + ); +} diff --git a/packages/api/src/mcp/tools/query/heatmapResult.ts b/packages/api/src/mcp/tools/query/heatmapResult.ts new file mode 100644 index 0000000000..35447cee0c --- /dev/null +++ b/packages/api/src/mcp/tools/query/heatmapResult.ts @@ -0,0 +1,81 @@ +import type { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; +import { + type HeatmapScaleType, + toHeatmapResult, +} from '@hyperdx/common-utils/dist/core/heatmap'; +import type { + BuilderSavedChartConfig, + NumberFormat, + TSource, +} from '@hyperdx/common-utils/dist/types'; + +import type { ToolResult } from '@/mcp/tools/types'; + +// Sized for an agent's context rather than a screen: ~48 x 22 counts. +export const MCP_HEATMAP_TIME_BUCKETS = 48; +export const MCP_HEATMAP_VALUE_BUCKETS = 20; + +function readingHint(scale: HeatmapScaleType) { + return ( + 'valueEdges are in the units of valueExpression (when numberFormat.factor is ' + + 'present, value x factor is seconds). counts[t] has one entry per range: ' + + 'counts[t][0] counts values below valueEdges[0], counts[t][r] counts values in ' + + '[valueEdges[r - 1], valueEdges[r]), and the last entry counts values at or ' + + 'above the last edge.' + + (scale === 'log' + ? ' On this log scale values below valueEdges[0] are counted in counts[t][1].' + : '') + ); +} + +/** A trace duration column formats as a duration unless the tile says otherwise. */ +export function heatmapNumberFormat( + config: Pick, + source: TSource, + valueExpression: string, +): NumberFormat | undefined { + if (config.numberFormat) return config.numberFormat; + if ( + 'durationExpression' in source && + valueExpression === source.durationExpression + ) { + return { + output: 'duration', + factor: Math.pow(10, -(source.durationPrecision ?? 9)), + }; + } + return undefined; +} + +/** A distribution heatmap as compact JSON an agent can read range by range. */ +export function formatHeatmapResult( + { + heatmap, + granularity, + granularityNote, + }: Awaited>, + { + valueExpression, + numberFormat, + }: { valueExpression: string; numberFormat?: NumberFormat }, +): ToolResult { + const notes = granularityNote ? [granularityNote] : []; + const body = heatmap + ? { + heatmap: toHeatmapResult(heatmap, { + valueExpression, + granularity, + numberFormat, + }), + hints: [readingHint(heatmap.scaleType), ...notes], + } + : { + heatmap: null, + hints: [ + `Not enough distinct values of ${valueExpression} in the time range to bucket. ` + + 'Widen startTime/endTime or check the value expression.', + ...notes, + ], + }; + return { content: [{ type: 'text', text: JSON.stringify(body) }] }; +} diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 028164edbe..7ede761fa5 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -2,6 +2,7 @@ import { ClickHouseError, type ClickHouseSettings, } from '@clickhouse/client-common'; +import { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { @@ -12,6 +13,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderSavedChartConfig, + isHeatmapCompatibleSource, isPromqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { UnknownVariableError } from '@hyperdx/common-utils/dist/macroErrors'; @@ -19,6 +21,7 @@ import type { ChartConfigWithDateRange, ChartVariable, MetricTable, + Tile, } from '@hyperdx/common-utils/dist/types'; import { DisplayType, @@ -47,6 +50,12 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +import { + formatHeatmapResult, + heatmapNumberFormat, + MCP_HEATMAP_TIME_BUCKETS, + MCP_HEATMAP_VALUE_BUCKETS, +} from './heatmapResult'; import { runEventPatterns } from './runEventPatterns'; // ─── Source body expression helpers ────────────────────────────────────────── @@ -407,6 +416,14 @@ function toPlainMetricTables( : metricTables; } +export function savedTileGranularity( + tiles: Tile[], + tileId: string, +): string | undefined { + const config = tiles.find(t => t.id === tileId)?.config; + return config && 'granularity' in config ? config.granularity : undefined; +} + export async function runConfigTile( teamId: string, tile: ExternalDashboardTileWithId, @@ -415,6 +432,8 @@ export async function runConfigTile( options?: { maxResults?: number; granularity?: string; + /** The stored tile's granularity, which the external tile format drops. Distribution heatmaps only. */ + savedGranularity?: string; abortSignal?: AbortSignal; /** Dashboard variables and their selected values. Omitted when not in a dashboard context. */ variables?: ChartVariable[]; @@ -596,6 +615,47 @@ export async function runConfigTile( variables: options?.variables, } satisfies ChartConfigWithDateRange; + // Distribution heatmaps bucket in two queries (value bounds, then + // widthBucket counts) that the generic renderer can't express. + if ( + builderConfig.displayType === DisplayType.Heatmap && + getHeatmapMode(builderConfig) === 'distribution' + ) { + if (!isHeatmapCompatibleSource(source)) { + return mcpUserError( + `Distribution heatmaps need a trace source; "${source.name}" is a ${source.kind} source.`, + ); + } + const valueExpression = Array.isArray(builderConfig.select) + ? (builderConfig.select[0]?.valueExpression ?? '') + : ''; + try { + const result = await queryDistributionHeatmap({ + client: clickhouseClient, + metadata: getMetadata(clickhouseClient), + config: chartConfig, + granularity: options?.granularity ?? options?.savedGranularity, + targetTimeBuckets: MCP_HEATMAP_TIME_BUCKETS, + nBuckets: MCP_HEATMAP_VALUE_BUCKETS, + querySettings: source.querySettings, + opts: { + clickhouse_settings: MCP_CLICKHOUSE_SETTINGS, + abort_signal: options?.abortSignal, + }, + }); + return formatHeatmapResult(result, { + valueExpression, + numberFormat: heatmapNumberFormat( + builderConfig, + source, + valueExpression, + ), + }); + } catch (e) { + return clickHouseErrorResult(e); + } + } + // Apply seriesLimit as LIMIT to categorical charts (pie/bar) const isCategorical = builderConfig.displayType === DisplayType.Pie || diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index 700a8a24ca..6af785b4d0 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -3,6 +3,7 @@ import type { ToolDefinition, ToolRegistrar } from '@/mcp/tools/types'; import { registerEmergingSignals } from './emergingSignals'; import { registerEventDeltas } from './eventDeltas'; import { registerEventPatterns } from './eventPatterns'; +import { registerHeatmap } from './heatmap'; import { registerSearch } from './search'; import { registerSql } from './sql'; import { registerTable } from './table'; @@ -11,6 +12,7 @@ import { registerTimeseries } from './timeseries'; const queryTools: ToolDefinition = (registrar: ToolRegistrar) => { registerTimeseries(registrar); registerTable(registrar); + registerHeatmap(registrar); registerSearch(registrar); registerEventPatterns(registrar); registerEventDeltas(registrar); diff --git a/packages/app/package.json b/packages/app/package.json index c6fc18f131..f9928aea73 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -14,7 +14,7 @@ "build:clickhouse": "NEXT_PUBLIC_THEME=clickstack NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_PUBLIC_CLICKHOUSE_BUILD=true next build --webpack && node scripts/prepare-clickhouse-build-export.js", "run:clickhouse": "test -d out && npx rimraf tmp && mkdir tmp && cp -r out tmp/clickstack && echo 'visit http://localhost:3000/clickstack to start' && npx serve tmp -l 3000 || echo 'run build:clickhouse first'", "start": "next start", - "lint": "npx eslint . --ext .ts,.tsx --max-warnings 568", + "lint": "npx eslint . --ext .ts,.tsx --max-warnings 567", "lint:fix": "npx eslint . --ext .ts,.tsx --fix", "lint:styles": "stylelint **/*/*.{css,scss}", "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:styles --quiet", diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index c2b72b654b..5b8fdd9e06 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -26,6 +26,7 @@ import { getPercentileAggFnLevel, Granularity, hasPositiveSeriesLimit, + isAggregateFunction, TIME_SERIES_DISPLAY_TYPE_BY_NAME, } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; @@ -391,114 +392,7 @@ export function timeBucketByGranularity( return buckets; } -export const isAggregateFunction = (value: string) => { - const fns = [ - // Basic aggregates - 'count', - 'countIf', - 'countDistinct', - 'sum', - 'sumIf', - 'avg', - 'avgIf', - 'min', - 'max', - 'any', - 'anyLast', - 'groupArray', - 'groupArrayInsertAt', - 'groupArrayMovingAvg', - 'groupArraySample', - 'groupUniqArray', - 'groupUniqArrayIf', - 'groupArrayIntersect', - 'groupArrayIntersectIf', - 'groupArrayReduce', - 'groupBitmap', - 'groupBitmapIf', - 'groupBitmapOr', - 'groupBitmapXor', - - // Quantiles - 'quantile', - 'quantileIf', - 'quantileExact', - 'quantileExactWeighted', - 'quantileTiming', - 'quantileTimingWeighted', - 'quantileTDigest', - 'quantileTDigestWeighted', - 'quantileBFloat16', - 'quantileBFloat16Weighted', - 'quantiles', - 'median', - 'medianExact', - 'medianTDigest', - 'medianBFloat16', - - // Statistical functions - 'stddevPop', - 'stddevPopIf', - 'stddevSamp', - 'stddevSampIf', - 'varPop', - 'varPopIf', - 'varSamp', - 'varSampIf', - 'covarPop', - 'covarSamp', - 'corr', - - // Combinators - 'uniq', - 'uniqExact', - 'uniqCombined', - 'uniqCombined64', - 'uniqHLL12', - 'uniqTheta', - - // Bit operations - 'groupBitAnd', - 'groupBitOr', - 'groupBitXor', - - // Map and tuple - 'groupArrayMap', - 'groupArrayTuple', - 'groupArraySorted', - 'topK', - 'topKIf', - 'topKWeighted', - - // Aggregate combinators - 'argMin', - 'argMax', - 'minMap', - 'maxMap', - - // Specialized aggregates - 'runningDifference', - 'retention', - 'sequenceCount', - 'sequenceMatch', - 'histogram', - 'simpleLinearRegression', - 'stochasticLinearRegression', - 'categoricalInformationValue', - 'sumMap', - 'sumMapFiltered', - 'sumWithOverflow', - 'entropy', - 'skewPop', - 'skewSamp', - 'kurtPop', - 'kurtSamp', - ]; - - // Make case-insensitive since ClickHouse function names are case-insensitive - const lowerValue = value.toLowerCase(); - return fns.some(fn => lowerValue.includes(fn.toLowerCase() + '(')); -}; +export { isAggregateFunction }; export const INTEGER_NUMBER_FORMAT: NumberFormat = { factor: 1, diff --git a/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts b/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts index 5d1db49594..cab9695854 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapBounds.ts @@ -1,27 +1,6 @@ -import type { HeatmapScaleType } from './heatmapGrid'; +export { + computeEffectiveMin, + heatmapLowQuantile, +} from '@hyperdx/common-utils/dist/core/heatmap'; export const HEATMAP_N_BUCKETS = 80; - -/** - * Quantile used for the y-axis lower bound. A quantile (rather than min) - * keeps near-zero outliers from stretching the log axis. The upper bound - * uses the actual max so that rare latency spikes stay visible; log scale - * already compresses wide ranges. - */ -export function heatmapLowQuantile(scaleType: HeatmapScaleType) { - return scaleType === 'log' ? 0.01 : 0.001; -} - -/** - * Lower bound actually used for bucketing. Log scale needs a positive - * minimum (log(0) is undefined), and the range is capped to ~4 orders of - * magnitude so the axis isn't dominated by a long empty tail of near-zero - * outliers. - */ -export function computeEffectiveMin( - min: number, - max: number, - scaleType: HeatmapScaleType, -) { - return scaleType === 'log' ? Math.max(min, max * 1e-4 || 1e-4) : min; -} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts index c1079471bd..571ac4ca0b 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts @@ -1,31 +1,16 @@ -import { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; - -export type HeatmapScaleType = 'log' | 'linear'; - -export type HeatmapYAxis = - | { - type: 'numeric'; - scale: HeatmapScaleType; - /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ - edges: number[]; - } - /** - * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per - * row; the top bound may be `Infinity` (a `+Inf` bucket). - */ - | { type: 'buckets'; bounds: number[] } - | { type: 'series'; labels: string[] }; - -/** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ -export type HeatmapGrid = { - /** Column centers, ms ascending. */ - times: number[]; - /** Column width, ms. */ - stepMs: number; - yAxis: HeatmapYAxis; - /** Row-major: `cells[timeIdx * rowCount + rowIdx]`. */ - cells: number[]; -}; +import { + type HeatmapGrid, + heatmapRowCount, + type HeatmapYAxis, +} from '@hyperdx/common-utils/dist/core/heatmap'; + +export { + gridFromBucketRows, + type HeatmapGrid, + heatmapRowCount, + type HeatmapScaleType, + type HeatmapYAxis, +} from '@hyperdx/common-utils/dist/core/heatmap'; /** * Numeric- and bucket-axis cells are counts, empty at 0. Series-axis cells @@ -48,17 +33,6 @@ export const EMPTY_HEATMAP_GRID: HeatmapGrid = { cells: [], }; -export function heatmapRowCount(yAxis: HeatmapYAxis) { - switch (yAxis.type) { - case 'series': - return yAxis.labels.length; - case 'buckets': - return yAxis.bounds.length; - case 'numeric': - return Math.max(0, yAxis.edges.length - 1); - } -} - /** * Lower and upper bound of each row in plot space. Series and histogram * bucket rows are unit-height bands. The uPlot y scale is linear, so log-scale @@ -121,80 +95,6 @@ export function gridToPlotData(grid: HeatmapGrid): HeatmapPlotData { return [xs, ys, grid.cells, x0s, x1s, y0s, y1s]; } -/** - * Grid for the rows of the server-side `widthBucket` query. widthBucket - * returns buckets 0 (underflow) through nBuckets + 1 (overflow), so each time - * bucket has nBuckets + 2 rows. - */ -export function gridFromBucketRows({ - data, - timestampColumn, - generatedTsBuckets, - scaleType, - effectiveMin, - max, - nBuckets, -}: { - data: Record[]; - timestampColumn: ColumnMetaType; - generatedTsBuckets: Date[]; - scaleType: HeatmapScaleType; - effectiveMin: number; - max: number; - nBuckets: number; -}): HeatmapGrid { - const isLog = scaleType === 'log' && effectiveMin > 0 && max > effectiveMin; - const bucketValue = (j: number) => - isLog - ? effectiveMin * Math.pow(max / effectiveMin, j / nBuckets) - : effectiveMin + j * ((max - effectiveMin) / nBuckets); - - // Row j is centered on bucketValue(j), which is the upper bound of - // widthBucket's bucket j, so the row boundaries sit half a bucket either - // side of it. - const rows = nBuckets + 2; - const edges = Array.from({ length: rows + 1 }, (_, k) => - bucketValue(k - 0.5), - ); - - const times = generatedTsBuckets.map(d => d.getTime()); - const cells: number[] = []; - - let dataIndex = 0; - for (const generatedTs of times) { - for (let j = 0; j < rows; j++) { - const row = data[dataIndex]; - - if ( - row != null && - new Date(row[timestampColumn.name]).getTime() == generatedTs && - row['x_bucket'] == j - ) { - cells.push(Number.parseInt(row['count'], 10)); // UInt64 returns as string - - // Skip duplicate buckets (from unmerged distributed table results) - while ( - dataIndex < data.length && - new Date(data[dataIndex][timestampColumn.name]).getTime() == - generatedTs && - data[dataIndex]['x_bucket'] == j - ) { - dataIndex++; - } - } else { - cells.push(0); - } - } - } - - return { - times, - stepMs: times.length > 1 ? times[1] - times[0] : 0, - yAxis: { type: 'numeric', scale: isLog ? 'log' : 'linear', edges }, - cells, - }; -} - /** * Cumulative share (0-100) of events at or below each row, keyed by row * index and pooled across all time buckets. diff --git a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts index 948df6920f..21ea61c90c 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts @@ -1,10 +1,17 @@ +import type { HeatmapScaleType } from '@hyperdx/common-utils/dist/core/heatmap'; import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; +import { + type HeatmapChartConfig, + toDistributionHeatmapConfig, +} from '@hyperdx/common-utils/dist/core/heatmapQuery'; import { getAlignedDateRange } from '@hyperdx/common-utils/dist/core/utils'; -import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + isBuilderChartConfig, + isPromqlChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, - DisplayType, PromqlConfigWithDateRange, SQLInterval, } from '@hyperdx/common-utils/dist/types'; @@ -12,31 +19,13 @@ import { import { convertToTimeChartConfig, getTimeChartGranularity, - isAggregateFunction, } from '@/ChartUtils'; -import { heatmapLowQuantile } from './heatmapBounds'; -import type { HeatmapScaleType } from './heatmapGrid'; - -export type HeatmapChartConfig = { - displayType: DisplayType.Heatmap; - select: [ - { - aggFn: 'heatmap'; - valueExpression: string; - countExpression?: string; - }, - ]; - from: BuilderChartConfigWithDateRange['from']; - where: BuilderChartConfigWithDateRange['where']; - dateRange: BuilderChartConfigWithDateRange['dateRange']; - granularity: BuilderChartConfigWithDateRange['granularity']; - timestampValueExpression: BuilderChartConfigWithDateRange['timestampValueExpression']; - numberFormat?: BuilderChartConfigWithDateRange['numberFormat']; - filters?: BuilderChartConfigWithDateRange['filters']; - connection: string; - with?: BuilderChartConfigWithDateRange['with']; -}; +export { + buildHeatmapBoundsConfig, + buildHeatmapBucketConfig, + type HeatmapChartConfig, +} from '@hyperdx/common-utils/dist/core/heatmapQuery'; /** The configs a series-mode heatmap can query. */ export type HeatmapSeriesChartConfig = @@ -112,26 +101,7 @@ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { }; } - const firstSelect = Array.isArray(config.select) - ? config.select[0] - : undefined; - return { - mode: 'distribution', - config: { - ...config, - displayType: DisplayType.Heatmap, - select: [ - { - aggFn: 'heatmap' as const, - valueExpression: firstSelect?.valueExpression ?? '', - countExpression: firstSelect?.countExpression, - }, - ], - granularity: config.granularity, - numberFormat: config.numberFormat, - }, - scaleType: firstSelect?.heatmapScaleType ?? 'log', - }; + return { mode: 'distribution', ...toDistributionHeatmapConfig(config) }; } /** @@ -168,175 +138,3 @@ export function buildHeatmapSeriesConfig( heatmap: undefined, }); } - -/** - * Build the bounds (min/max) ChartConfig that runs first. Result feeds - * `effectiveMin`/`max` into `buildHeatmapBucketConfig`. - */ -export function buildHeatmapBoundsConfig({ - config, - scaleType, - granularity, -}: { - config: HeatmapChartConfig; - scaleType: HeatmapScaleType; - granularity: SQLInterval; -}): BuilderChartConfigWithDateRange { - const valueExpression = config.select[0].valueExpression; - const isAggregateExpression = isAggregateFunction(valueExpression); - const qLo = heatmapLowQuantile(scaleType); - - return isAggregateExpression - ? { - ...config, - where: '', - orderBy: undefined, - granularity: undefined, - select: [ - { - aggFn: 'quantile' as const, - level: qLo, - aggCondition: `value_calc >= 0`, - aggConditionLanguage: 'sql', - valueExpression: 'value_calc', - alias: 'min', - }, - { - aggFn: 'max' as const, - valueExpression: 'value_calc', - alias: 'max', - }, - ], - with: [ - { - name: 'min_max_calc', - chartConfig: { - ...config, - select: [{ valueExpression, alias: 'value_calc' }], - orderBy: undefined, - // Emits the __hdx_time_bucket column the outer query filters on. - granularity, - }, - }, - ], - timestampValueExpression: '__hdx_time_bucket', - // The first bucket is labelled before an unaligned start, so align - // this filter on bucket labels. The inner query keeps the original - // range so no events outside it are counted. - dateRange: getAlignedDateRange(config.dateRange, granularity), - from: { databaseName: '', tableName: 'min_max_calc' }, - } - : { - ...config, - orderBy: undefined, - granularity: undefined, - select: [ - { - aggFn: 'quantile' as const, - level: qLo, - valueExpression, - aggCondition: `${valueExpression} >= 0`, - aggConditionLanguage: 'sql', - alias: 'min', - }, - { - aggFn: 'max' as const, - valueExpression, - alias: 'max', - }, - ], - }; -} - -/** - * Build the bucketed-counts ChartConfig that runs second. `effectiveMin`/`max` - * are usually numbers (resolved from the bounds query), but accept strings so - * callers — like the editor's SQL preview — can pass placeholder tokens - * (e.g. `'{min}'`) before the bounds are known. - */ -export function buildHeatmapBucketConfig({ - config, - scaleType, - effectiveMin, - max, - granularity, - nBuckets, -}: { - config: HeatmapChartConfig; - scaleType: HeatmapScaleType; - effectiveMin: string | number; - max: string | number; - granularity: SQLInterval; - nBuckets: number; -}): BuilderChartConfigWithDateRange { - const valueExpression = config.select[0].valueExpression; - // The chart editor saves a cleared Count input as ''. - const countExpression = config.select[0].countExpression?.trim() || 'count()'; - const isAggregateExpression = isAggregateFunction(valueExpression); - - const bucketExprAgg = - scaleType === 'log' - ? `widthBucket(log(greatest(toFloat64(value_calc), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` - : `widthBucket(value_calc, ${effectiveMin}, ${max}, ${nBuckets})`; - const bucketExprDirect = - scaleType === 'log' - ? `widthBucket(log(greatest(toFloat64(${valueExpression}), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` - : `widthBucket(${valueExpression}, ${effectiveMin}, ${max}, ${nBuckets})`; - - return isAggregateExpression - ? { - ...config, - where: '', - select: [ - { - valueExpression: 'sum(value_count)', - alias: 'count', - }, - ], - groupBy: [ - { - valueExpression: bucketExprAgg, - alias: 'x_bucket', - }, - ], - with: [ - { - name: 'bucket_calc', - chartConfig: { - ...config, - select: [ - { valueExpression, alias: 'value_calc' }, - { - valueExpression: countExpression, - alias: 'value_count', - }, - ], - granularity, - orderBy: undefined, - }, - }, - ], - timestampValueExpression: '__hdx_time_bucket', - dateRange: getAlignedDateRange(config.dateRange, granularity), - from: { databaseName: '', tableName: 'bucket_calc' }, - orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], - granularity, - } - : { - ...config, - select: [ - { - valueExpression: countExpression, - alias: 'count', - }, - ], - groupBy: [ - { - valueExpression: bucketExprDirect, - alias: 'x_bucket', - }, - ], - orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], - granularity, - }; -} diff --git a/packages/claude-plugin/README.md b/packages/claude-plugin/README.md index da84395e5c..c76c51d04c 100644 --- a/packages/claude-plugin/README.md +++ b/packages/claude-plugin/README.md @@ -1,9 +1,11 @@ # @hyperdx/claude-plugin A Claude Code plugin that draws ClickStack MCP query results as charts in the -terminal. When Claude runs `clickstack_timeseries` or `clickstack_table`, the -tool row shows a line, stacked bar, bar, pie, number or table chart in place of -the raw JSON. Claude still reads the full result. +terminal. When Claude runs `clickstack_timeseries`, `clickstack_table` or +`clickstack_heatmap`, the tool row shows a line, stacked bar, bar, pie, number, +table or heatmap chart in place of the raw JSON. Claude still reads the full +result. `clickstack_query_tile` results are drawn when the tile is a +distribution heatmap. Charts are drawn by the same renderers as the `hdx` CLI, so a query looks the same in both. diff --git a/packages/claude-plugin/hooks/chart.d.ts b/packages/claude-plugin/hooks/chart.d.ts index 94e6b16a98..b4ae0de7fe 100644 --- a/packages/claude-plugin/hooks/chart.d.ts +++ b/packages/claude-plugin/hooks/chart.d.ts @@ -10,6 +10,11 @@ export type Segment = { }; export function chartedTool(toolName: string): string | undefined; +export function isChartable( + toolName: string, + input: unknown, + output: unknown, +): boolean; export function renderChart( call: ChartCall, columns: number, diff --git a/packages/claude-plugin/hooks/register.tsx b/packages/claude-plugin/hooks/register.tsx index 40e0cb7d53..2e94e70294 100644 --- a/packages/claude-plugin/hooks/register.tsx +++ b/packages/claude-plugin/hooks/register.tsx @@ -8,7 +8,12 @@ import type { import type { ChartCall } from '../types'; -import { chartedTool, renderChart, stripControlChars } from './chart.js'; +import { + chartedTool, + isChartable, + renderChart, + stripControlChars, +} from './chart.js'; const KEEP = 20; const MAX_COLUMNS = 140; @@ -82,21 +87,23 @@ export const register: Register = on => { // The server fixes its default window before the query runs. const calledAt = Date.now(); const ran = await next(e); - if (!chartedTool(e.tool) || ran.deny !== undefined || ran.isError) - return ran; + if (ran.deny !== undefined || ran.isError) return ran; const call: ChartCall = { tool: e.tool, input: e, output: ran.result, calledAt, }; - // Chart bookkeeping must never fail the call it watched. + // Chart bookkeeping must never fail the call it watched. Only results that + // draw are kept, so query_tile calls on other tiles don't evict charts. try { - await update($, calls, all => - Object.fromEntries( - [...Object.entries(all), [e.tool_use_id, call]].slice(-KEEP), - ), - ); + if (isChartable(e.tool, e, ran.result)) { + await update($, calls, all => + Object.fromEntries( + [...Object.entries(all), [e.tool_use_id, call]].slice(-KEEP), + ), + ); + } } catch { // The row keeps the engine's own rendering. } diff --git a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts index e3cd5665de..91e52fef40 100644 --- a/packages/claude-plugin/src/__tests__/toTileResult.spec.ts +++ b/packages/claude-plugin/src/__tests__/toTileResult.spec.ts @@ -3,6 +3,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { chartedTool, + isChartable, isTrimmed, parseResponse, toTileResult, @@ -169,4 +170,106 @@ describe('toTileResult', () => { ), ).toBeUndefined(); }); + + it('turns a heatmap result into a grid, for the heatmap and query_tile tools', () => { + const heatmap = { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + numberFormat: { output: 'duration', factor: 1e-9 }, + times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], + valueEdges: [1, 10, 100], + counts: [ + [0, 3, 0, 0], + [0, 1, 2, 0], + ], + }; + const out = [{ type: 'text', text: JSON.stringify({ heatmap }) }]; + for (const tool of ['clickstack_heatmap', 'clickstack_query_tile']) { + const result = toTileResult( + `mcp__clickstack__${tool}`, + {}, + out, + calledAt, + ); + expect(result).toMatchObject({ + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + numberFormat: heatmap.numberFormat, + }, + grid: { + stepMs: 60_000, + // Under- and overflow rows get one bucket's height on either side. + yAxis: { + type: 'numeric', + scale: 'log', + edges: [0.1, 1, 10, 100, 1000], + }, + cells: [0, 3, 0, 0, 0, 1, 2, 0], + }, + }); + } + }); + + it('leaves a query_tile result alone unless it is a heatmap', () => { + expect( + toTileResult( + 'mcp__clickstack__clickstack_query_tile', + {}, + output, + calledAt, + ), + ).toBeUndefined(); + }); + + it('rejects a heatmap body that does not match the shared schema', () => { + const bad = (heatmap: object) => [ + { type: 'text', text: JSON.stringify({ heatmap }) }, + ]; + const body = { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + times: ['2026-10-13T00:00:00.000Z'], + valueEdges: [1, 10], + counts: [[0, 1, 0]], + }; + const tool = 'mcp__clickstack__clickstack_heatmap'; + expect(toTileResult(tool, {}, bad(body), calledAt)).toBeDefined(); + const { granularity: _omit, ...noGranularity } = body; + expect( + toTileResult(tool, {}, bad(noGranularity), calledAt), + ).toBeUndefined(); + expect( + toTileResult(tool, {}, bad({ ...body, counts: 'x' }), calledAt), + ).toBeUndefined(); + }); + + it('keeps only results it can draw for caching', () => { + const heatmapBody = [ + { + type: 'text', + text: JSON.stringify({ + heatmap: { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + times: ['2026-10-13T00:00:00.000Z'], + valueEdges: [1, 10], + counts: [[0, 1, 0]], + }, + }), + }, + ]; + expect( + isChartable('mcp__clickstack__clickstack_query_tile', {}, heatmapBody), + ).toBe(true); + expect( + isChartable('mcp__clickstack__clickstack_query_tile', {}, output), + ).toBe(false); + expect(isChartable('mcp__clickstack__clickstack_search', {}, output)).toBe( + false, + ); + }); }); diff --git a/packages/claude-plugin/src/index.ts b/packages/claude-plugin/src/index.ts index d627d028bb..43563c173b 100644 --- a/packages/claude-plugin/src/index.ts +++ b/packages/claude-plugin/src/index.ts @@ -1,3 +1,3 @@ export { stripControlChars } from './ansiLines'; export { renderChart } from './renderChart'; -export { chartedTool } from './toTileResult'; +export { chartedTool, isChartable } from './toTileResult'; diff --git a/packages/claude-plugin/src/renderChart.ts b/packages/claude-plugin/src/renderChart.ts index c1f991f268..4e4f75ff51 100644 --- a/packages/claude-plugin/src/renderChart.ts +++ b/packages/claude-plugin/src/renderChart.ts @@ -9,6 +9,7 @@ import { isTrimmed, toTileResult } from './toTileResult'; const TIME_CHART_ROWS = 14; const MAX_TABLE_ROWS = 20; +const HEATMAP_ROWS = 18; const TRIMMED_NOTE = 'The server trimmed this result to fit its size limit; the chart shows only the rows it returned.'; @@ -29,9 +30,14 @@ export function renderChart( const height = displayType === DisplayType.Number ? 3 - : isTimeSeriesDisplayType(displayType) - ? TIME_CHART_ROWS - : Math.min(result.data.data.length + 2, MAX_TABLE_ROWS); + : displayType === DisplayType.Heatmap + ? HEATMAP_ROWS + : isTimeSeriesDisplayType(displayType) + ? TIME_CHART_ROWS + : Math.min( + (result.status === 'ok' ? result.data.data.length : 0) + 2, + MAX_TABLE_ROWS, + ); try { const lines = ansiLines( diff --git a/packages/claude-plugin/src/toTileResult.ts b/packages/claude-plugin/src/toTileResult.ts index b68a77fbb4..f16a1fd1c1 100644 --- a/packages/claude-plugin/src/toTileResult.ts +++ b/packages/claude-plugin/src/toTileResult.ts @@ -1,6 +1,12 @@ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; -import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { + type HeatmapGrid, + type HeatmapResult, + HeatmapResultSchema, + heatmapRowEdges, +} from '@hyperdx/common-utils/dist/core/heatmap'; import { TIME_SERIES_DISPLAY_TYPE_BY_NAME } from '@hyperdx/common-utils/dist/core/utils'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { DisplayType } from '@hyperdx/common-utils/dist/types'; import type { TileQueryResult } from '@hyperdx/cli/src/shared/tileQuery'; @@ -19,6 +25,10 @@ const DISPLAY_TYPES: Record> = { }, }; +// Tools answering with a `heatmap` grid. A query_tile result for any other +// tile carries no display type, so it is left alone. +const HEATMAP_TOOLS = ['clickstack_heatmap', 'clickstack_query_tile']; + type ToolInput = { shape?: string; select?: unknown[]; @@ -28,12 +38,14 @@ type ToolInput = { endTime?: string; }; -type ChartResult = Extract; +type ChartResult = Extract; /** Matches by MCP tool name suffix, so it works whatever the user named the server. */ export function chartedTool(toolName: string): string | undefined { const suffix = toolName.split('__').pop() ?? ''; - return suffix in DISPLAY_TYPES ? suffix : undefined; + return suffix in DISPLAY_TYPES || HEATMAP_TOOLS.includes(suffix) + ? suffix + : undefined; } /** MCP output arrives as text blocks holding one JSON document. */ @@ -51,6 +63,15 @@ function parseBody(output: unknown): Record | undefined { } } +/** Whether a call's result can be drawn, so only those are kept for drawing. */ +export function isChartable( + toolName: string, + input: unknown, + output: unknown, +): boolean { + return toTileResult(toolName, input, output, Date.now()) !== undefined; +} + /** The server trims results over its size limit and says so in `note`. */ export function isTrimmed(output: unknown): boolean { return typeof parseBody(output)?.note === 'string'; @@ -84,6 +105,21 @@ export function toTileResult( ? new Date(args.startTime) : new Date(end.getTime() - DEFAULT_WINDOW_MS); + if (HEATMAP_TOOLS.includes(tool)) { + const parsed = HeatmapResultSchema.safeParse(parseBody(output)?.heatmap); + if (!parsed.success || parsed.data.counts.length === 0) return undefined; + const heatmap = parsed.data; + return { + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + dateRange: [start, end], + numberFormat: heatmap.numberFormat, + } as unknown as ChartConfigWithDateRange, + grid: toGrid(heatmap), + }; + } + const data = parseResponse(output); if (!data || data.data.length === 0) return undefined; const shapes = DISPLAY_TYPES[tool]!; @@ -110,3 +146,21 @@ export function toTileResult( return { status: 'ok', queriedConfig, data }; } +function toGrid({ + times, + valueEdges, + counts, + scale, +}: HeatmapResult): HeatmapGrid { + const ms = times.map(t => Date.parse(t)); + return { + times: ms, + stepMs: ms.length > 1 ? ms[1]! - ms[0]! : 0, + yAxis: { + type: 'numeric', + scale, + edges: heatmapRowEdges(valueEdges, scale), + }, + cells: counts.flat(), + }; +} diff --git a/packages/claude-plugin/test/render.test.tsx b/packages/claude-plugin/test/render.test.tsx index 3d4fe99779..ea890b7872 100644 --- a/packages/claude-plugin/test/render.test.tsx +++ b/packages/claude-plugin/test/render.test.tsx @@ -130,6 +130,55 @@ test( }, ); +test('draws a heatmap result as shaded cells', { options }, async ($, on) => { + const heatmap = { + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'log', + times: ['2026-10-13T00:00:00.000Z', '2026-10-13T00:01:00.000Z'], + valueEdges: [1, 10, 100, 1000], + counts: [ + [0, 9, 1, 0, 0], + [0, 0, 2, 30, 0], + ], + }; + let id = ''; + on('tool.call', (_, e) => { + id = e.tool_use_id; + return { result: [{ type: 'text', text: JSON.stringify({ heatmap }) }] }; + }); + on('ui.render', ($, e) => { + const { Text } = $.ui.resolve(e); + return engine row; + }); + const tool = 'mcp__clickstack__clickstack_heatmap'; + await $.tool.call({ tool, sourceId: 's', valueExpression: 'Duration' }); + const ui = await $.ui.mount({ + plugin: 'hyperdx-charts', + surface: 'terminal', + component: 'ToolGroup', + props: { + calls: [ + { + tool_use_id: id, + tool, + input: {}, + isRunning: false, + isErrored: false, + isInterrupted: false, + }, + ], + isActive: false, + isExpanded: false, + }, + viewport: { columns: 100, rows: 40 }, + }); + expect( + await ui.find({ type: 'Text', text: /events per cell, max 30/ }), + ).toBeDefined(); + expect(await ui.find({ type: 'Text', text: /[░▒▓█]/ })).toBeDefined(); +}); + test( 'draws a timeseries call drawn as its own tool row', { options }, diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index d74077b978..8a22c1d84a 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -161,8 +161,10 @@ dashboard**: module (only dependency: chalk) — number formatting is injected via callbacks, and it must never import from the rest of the CLI. -Supported display types: line, stacked*bar, number, table, bar, pie, markdown. -Heatmap / search / event patterns / PromQL show a placeholder. The web's +Supported display types: line, stacked*bar, number, table, bar, pie, markdown, +and distribution heatmaps (queried through common-utils' +`queryDistributionHeatmap`, the same two queries the web runs). Series +heatmaps / search / event patterns / PromQL show a placeholder. The web's previous-period comparison overlay is not rendered; tiles with `compareToPreviousPeriod` set show a dim callout instead. Time-series charts stretch to the full terminal width via peak-preserving resampling @@ -182,7 +184,8 @@ same pipeline in three modes: dashboard tiles (`-d`), ad-hoc builder charts `shared/adhocChart.ts`), and ad-hoc raw SQL (`--sql` with `$__timeFilter` / `$__timeInterval` macros). ANSI colors are stripped automatically when stdout is not a TTY (`--color auto|always|never`), and `--json` emits raw rows + column -metadata. +metadata (distribution heatmap tiles emit a `heatmap` object in the same shape +as the MCP `clickstack_heatmap` result). Do NOT change resolution/shaping rules without checking the web components first (see the alignment table below). diff --git a/packages/cli/src/__tests__/termchart.test.ts b/packages/cli/src/__tests__/termchart.test.ts index 60c051aaac..38a2f5d42a 100644 --- a/packages/cli/src/__tests__/termchart.test.ts +++ b/packages/cli/src/__tests__/termchart.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from '@jest/globals'; import { renderLegend } from '@/termchart/ansi'; import { niceTicks, + renderHeatmapChart, renderLineChart, renderStackedBarChart, resampleSeries, @@ -386,6 +387,66 @@ describe('renderStackedBarChart', () => { }); }); +describe('renderHeatmapChart', () => { + // Two time buckets, four value rows (edges 1..16 on a log axis); the second + // bucket has a dense band in the top row. + const data = { + times: [1_700_000_000, 1_700_000_060], + rowEdges: [1, 2, 4, 8, 16], + counts: [ + [0, 5, 0, 0], + [0, 1, 0, 50], + ], + }; + + it('shades denser cells darker and labels the value axis', () => { + const output = stripAnsi( + renderHeatmapChart({ data, width: 40, height: 12 }), + ); + const lines = output.split('\n'); + expect(lines[0]).toContain('16'); + expect(lines[0]).toContain('█'); + expect(output).toContain('max 50'); + const bottomRow = lines.at(-4)!; + expect(bottomRow).toMatch(/[░▒]/); + expect(bottomRow).not.toContain('█'); + }); + + it('trims empty value rows so the populated range fills the plot', () => { + const output = stripAnsi( + renderHeatmapChart({ + data: { + ...data, + counts: [ + [0, 5, 5, 0], + [0, 5, 5, 0], + ], + }, + width: 40, + height: 12, + }), + ); + expect(output.split('\n')[0]).toContain('8'); + }); + + it('says so when every cell is empty', () => { + const output = stripAnsi( + renderHeatmapChart({ + data: { + ...data, + counts: [ + [0, 0, 0, 0], + [0, 0, 0, 0], + ], + }, + width: 40, + height: 12, + }), + ); + expect(output).toContain('No data'); + }); +}); + describe('renderLegend', () => { const series = Array.from({ length: 8 }, (_, i) => ({ displayName: `service-${i}-${'x'.repeat(12)}`, diff --git a/packages/cli/src/__tests__/tileQuery.test.ts b/packages/cli/src/__tests__/tileQuery.test.ts new file mode 100644 index 0000000000..393791b2e8 --- /dev/null +++ b/packages/cli/src/__tests__/tileQuery.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, jest } from '@jest/globals'; + +import type { SavedChartConfig } from '@hyperdx/common-utils/dist/types'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import type { ProxyClickhouseClient, SourceResponse } from '@/api/client'; +import { fetchTileData } from '@/shared/tileQuery'; + +const traceSource = { + id: 'src-2', + kind: 'trace', + name: 'Traces', + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, +} as unknown as SourceResponse; + +const dateRange: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-13T00:02:00Z'), +]; + +function heatmapTile(extra: Record = {}) { + return { + displayType: DisplayType.Heatmap, + source: 'src-2', + select: [{ aggFn: 'count', valueExpression: 'Duration' }], + where: '', + ...extra, + } as unknown as SavedChartConfig; +} + +describe('fetchTileData heatmap tiles', () => { + it('runs a distribution heatmap through the bounds and bucket queries', async () => { + const queryChartConfig = jest + .fn<() => Promise>() + .mockResolvedValueOnce({ meta: [], data: [{ min: '1', max: '100' }] }) + .mockResolvedValueOnce({ + meta: [{ name: '__hdx_time_bucket', type: 'DateTime' }], + data: [ + { + __hdx_time_bucket: '2026-10-13T00:00:00Z', + x_bucket: 2, + count: '4', + }, + ], + }); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile(), + source: traceSource, + dateRange, + granularity: '1 minute', + }); + + expect(queryChartConfig).toHaveBeenCalledTimes(2); + expect(result.status).toBe('heatmap'); + if (result.status !== 'heatmap') return; + expect(result.grid?.times).toHaveLength(2); + expect(result.grid?.cells.reduce((a, b) => a + b, 0)).toBe(4); + }); + + it('reports series heatmaps as unsupported without querying', async () => { + const queryChartConfig = jest.fn(); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile({ heatmap: { mode: 'series' } }), + source: traceSource, + dateRange, + }); + expect(result.status).toBe('unsupported'); + expect(queryChartConfig).not.toHaveBeenCalled(); + }); + + it('uses the saved granularity and labels rows with widthBucket bounds', async () => { + const queryChartConfig = jest + .fn<() => Promise>() + .mockResolvedValueOnce({ meta: [], data: [{ min: '0', max: '100' }] }) + .mockResolvedValueOnce({ + meta: [{ name: '__hdx_time_bucket', type: 'DateTime' }], + data: [], + }); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile({ + granularity: '1 minute', + select: [ + { + aggFn: 'count', + valueExpression: 'Duration', + heatmapScaleType: 'linear', + }, + ], + }), + source: traceSource, + dateRange, + }); + + const bucketQuery = queryChartConfig.mock.calls[1] as unknown as [ + { config: { granularity: string } }, + ]; + expect(bucketQuery[0].config.granularity).toBe('1 minute'); + if (result.status !== 'heatmap') throw new Error(result.status); + // 40 buckets over 0..100 are 2.5 wide; the underflow row starts one below 0. + expect(result.grid?.yAxis).toMatchObject({ + type: 'numeric', + scale: 'linear', + }); + const edges = + result.grid?.yAxis.type === 'numeric' ? result.grid.yAxis.edges : []; + expect(edges.slice(0, 3)).toEqual([-2.5, 0, 2.5]); + expect(result.heatmap).toMatchObject({ + valueExpression: 'Duration', + granularity: '1 minute', + scale: 'linear', + }); + }); + + it('refuses distribution heatmaps on non-trace sources', async () => { + const queryChartConfig = jest.fn(); + const result = await fetchTileData({ + clickhouseClient: { + queryChartConfig, + } as unknown as ProxyClickhouseClient, + metadata: {} as never, + config: heatmapTile(), + source: { ...traceSource, kind: 'log' } as unknown as SourceResponse, + dateRange, + }); + expect(result).toMatchObject({ status: 'unsupported' }); + expect(queryChartConfig).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/__tests__/tileRender.test.ts b/packages/cli/src/__tests__/tileRender.test.ts index 5a015d7cd9..f348cac839 100644 --- a/packages/cli/src/__tests__/tileRender.test.ts +++ b/packages/cli/src/__tests__/tileRender.test.ts @@ -86,4 +86,65 @@ describe('renderTileContent', () => { ); expect(render(DisplayType.StackedLine)).not.toBe(render(DisplayType.Line)); }); + + it('draws a distribution heatmap with its duration axis', () => { + const heatmapConfig = { + displayType: DisplayType.Heatmap, + dateRange, + numberFormat: { output: 'duration', factor: 0.001 }, + } as unknown as ChartConfigWithDateRange; + const result: TileQueryResult = { + status: 'heatmap', + queriedConfig: heatmapConfig, + grid: { + times: [dateRange[0].getTime(), dateRange[0].getTime() + 60_000], + stepMs: 60_000, + yAxis: { type: 'numeric', scale: 'log', edges: [1, 10, 100, 1000] }, + cells: [9, 1, 0, 0, 2, 30], + }, + }; + const output = stripAnsi( + renderTileContent({ result, source: undefined, width: 60, height: 12 }), + ); + expect(output).toContain('events per cell, max 30'); + expect(output).toMatch(/[░▒▓█]/); + + expect( + stripAnsi( + renderTileContent({ + result: { ...result, grid: undefined }, + source: undefined, + width: 60, + height: 12, + }), + ), + ).toContain('Not enough data points'); + }); + + it('formats a saved Duration heatmap as durations without a numberFormat', () => { + const result: TileQueryResult = { + status: 'heatmap', + queriedConfig: { + displayType: DisplayType.Heatmap, + dateRange, + select: [{ aggFn: 'count', valueExpression: 'Duration' }], + } as unknown as ChartConfigWithDateRange, + grid: { + times: [dateRange[0].getTime()], + stepMs: 0, + yAxis: { type: 'numeric', scale: 'log', edges: [1e6, 1e7, 1e8, 1e9] }, + cells: [3, 5, 1], + }, + }; + const traceSource = { + kind: 'trace', + durationExpression: 'Duration', + durationPrecision: 9, + } as never; + const output = stripAnsi( + renderTileContent({ result, source: traceSource, width: 60, height: 12 }), + ); + expect(output).toMatch(/\d+(ms|s)\b/); + expect(output).not.toContain('1.0B'); + }); }); diff --git a/packages/cli/src/cli.tsx b/packages/cli/src/cli.tsx index 31744ffeb7..46bc08c3eb 100644 --- a/packages/cli/src/cli.tsx +++ b/packages/cli/src/cli.tsx @@ -1372,6 +1372,9 @@ Examples: ...(result.status === 'ok' ? { data: result.data.data, meta: result.data.meta } : {}), + ...(result.status === 'heatmap' + ? { heatmap: result.heatmap ?? null } + : {}), ...(result.status === 'unsupported' ? { message: result.message } : {}), diff --git a/packages/cli/src/shared/chartData.ts b/packages/cli/src/shared/chartData.ts index 3dca9f9d94..70f06dc359 100644 --- a/packages/cli/src/shared/chartData.ts +++ b/packages/cli/src/shared/chartData.ts @@ -20,6 +20,10 @@ import { type ColumnMetaType, type ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; +import { + heatmapCountColumns, + type HeatmapGrid, +} from '@hyperdx/common-utils/dist/core/heatmap'; import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; import type { ChartConfigWithOptDateRange, @@ -29,6 +33,7 @@ import type { import type { SourceResponse } from '@/api/client'; import type { CategoricalEntry, + HeatmapData, TableColumn, TimeChartSeries, } from '@/termchart'; @@ -551,3 +556,13 @@ export function deriveTableColumns({ isGroupColumn: groupByKeys.includes(key), })); } + +/** A heatmap grid as termchart draws it: per-column counts, seconds timestamps. */ +export function formatHeatmapGrid(grid: HeatmapGrid): HeatmapData | undefined { + if (grid.yAxis.type !== 'numeric') return undefined; + return { + times: grid.times.map(t => t / 1000), + rowEdges: grid.yAxis.edges, + counts: heatmapCountColumns(grid), + }; +} diff --git a/packages/cli/src/shared/formatNumber.ts b/packages/cli/src/shared/formatNumber.ts index 9d51af73a8..7deaeb3901 100644 --- a/packages/cli/src/shared/formatNumber.ts +++ b/packages/cli/src/shared/formatNumber.ts @@ -576,3 +576,21 @@ export function axisTickFormatter( return ''; }; } + +/** + * A heatmap's axis format. Saved heatmap tiles carry aggFn count, which hides + * a duration value expression from the series check, so check it directly. + */ +export function heatmapNumberFormat( + config: ChartConfigWithOptDateRange, + source: SourceResponse | undefined, +): NumberFormat | undefined { + const valueExpression = + isBuilderChartConfig(config) && Array.isArray(config.select) + ? config.select[0]?.valueExpression + : undefined; + return ( + resolveChartNumberFormats(config, source).chartFormat ?? + getTraceDurationNumberFormat(source, { valueExpression }) + ); +} diff --git a/packages/cli/src/shared/tileConfig.ts b/packages/cli/src/shared/tileConfig.ts index b8b1fea940..9c02ce0481 100644 --- a/packages/cli/src/shared/tileConfig.ts +++ b/packages/cli/src/shared/tileConfig.ts @@ -401,5 +401,6 @@ export const CLI_SUPPORTED_DISPLAY_TYPES: ReadonlySet = new Set([ DisplayType.Table, DisplayType.Pie, DisplayType.Bar, + DisplayType.Heatmap, DisplayType.Markdown, ]); diff --git a/packages/cli/src/shared/tileQuery.ts b/packages/cli/src/shared/tileQuery.ts index 80b90de471..e61423a155 100644 --- a/packages/cli/src/shared/tileQuery.ts +++ b/packages/cli/src/shared/tileQuery.ts @@ -5,14 +5,27 @@ */ import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { + getHeatmapMode, + type HeatmapGrid, + type HeatmapResult, + heatmapRowEdges, + toHeatmapResult, +} from '@hyperdx/common-utils/dist/core/heatmap'; +import { queryDistributionHeatmap } from '@hyperdx/common-utils/dist/core/distributionHeatmap'; import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + isBuilderChartConfig, + isHeatmapCompatibleSource, +} from '@hyperdx/common-utils/dist/guards'; import type { ChartConfigWithDateRange, SavedChartConfig, } from '@hyperdx/common-utils/dist/types'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; import type { ProxyClickhouseClient, SourceResponse } from '@/api/client'; +import { heatmapNumberFormat } from '@/shared/formatNumber'; import { CLI_SUPPORTED_DISPLAY_TYPES, convertTileConfigForQuery, @@ -27,6 +40,14 @@ export type TileQueryResult = queriedConfig: ChartConfigWithDateRange; data: ResponseJSON>; } + | { + status: 'heatmap'; + queriedConfig: ChartConfigWithDateRange; + /** Undefined when the range has too few distinct values to bucket */ + grid: HeatmapGrid | undefined; + /** The same grid in the shape the MCP server returns, for --json */ + heatmap?: HeatmapResult; + } | { status: 'markdown'; /** Markdown tiles render statically without querying */ @@ -38,6 +59,9 @@ export type TileQueryResult = resolution: Extract; }; +// Value buckets per heatmap column; the renderer merges them to fit its rows. +const HEATMAP_VALUE_BUCKETS = 40; + export interface FetchTileDataParams { clickhouseClient: ProxyClickhouseClient; metadata: Metadata; @@ -91,6 +115,69 @@ export async function fetchTileData({ return { status: 'unresolved', resolution }; } + if (displayType === DisplayType.Heatmap) { + const heatmapConfig = resolution.config; + if ( + !isBuilderChartConfig(heatmapConfig) || + getHeatmapMode(heatmapConfig) === 'series' + ) { + return { + status: 'unsupported', + message: 'Series heatmap tiles are not supported in the CLI yet.', + }; + } + // The CLI's source kind is a string; map it onto SourceKind for the shared guard. + const sourceKind = Object.values(SourceKind).find(k => k === source?.kind); + if ( + source && + !(sourceKind && isHeatmapCompatibleSource({ kind: sourceKind })) + ) { + return { + status: 'unsupported', + message: `Distribution heatmaps need a trace source; "${source.name}" is a ${source.kind} source.`, + }; + } + const result = await queryDistributionHeatmap({ + client: clickhouseClient, + metadata, + config: heatmapConfig, + // resolveTileConfig swaps in the command's granularity, which is unset + // by default; fall back to the tile's own. + granularity: + granularity ?? + ('granularity' in config ? config.granularity : undefined), + targetTimeBuckets: maxTimeBuckets, + nBuckets: HEATMAP_VALUE_BUCKETS, + querySettings: source?.querySettings, + opts: { abort_signal: signal }, + }); + const { heatmap } = result; + const firstSelect = Array.isArray(heatmapConfig.select) + ? heatmapConfig.select[0] + : undefined; + return { + status: 'heatmap', + queriedConfig: heatmapConfig, + // Label rows with the bucket bounds widthBucket counted, not the + // web's centred display edges. + grid: heatmap && { + ...heatmap.grid, + yAxis: { + type: 'numeric', + scale: heatmap.scaleType, + edges: heatmapRowEdges(heatmap.bucketEdges, heatmap.scaleType), + }, + }, + heatmap: + heatmap && + toHeatmapResult(heatmap, { + valueExpression: firstSelect?.valueExpression ?? '', + granularity: result.granularity, + numberFormat: heatmapNumberFormat(heatmapConfig, source), + }), + }; + } + const queriedConfig = convertTileConfigForQuery(resolution.config, { maxTimeBuckets, }); diff --git a/packages/cli/src/shared/tileRender.ts b/packages/cli/src/shared/tileRender.ts index 4b62c77ab7..556850e005 100644 --- a/packages/cli/src/shared/tileRender.ts +++ b/packages/cli/src/shared/tileRender.ts @@ -15,6 +15,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import type { SourceResponse } from '@/api/client'; import { deriveTableColumns, + formatHeatmapGrid, formatResponseForCategoricalChart, formatResponseForTimeChart, getNumberChartValue, @@ -23,6 +24,7 @@ import { import { axisTickFormatter, formatNumber, + heatmapNumberFormat, resolveChartNumberFormats, resolveSingleSeriesNumberFormat, } from '@/shared/formatNumber'; @@ -30,6 +32,7 @@ import type { TileQueryResult } from '@/shared/tileQuery'; import { defaultFormatValue, renderCategoricalChart, + renderHeatmapChart, renderLineChart, renderMarkdown, renderNumberChart, @@ -62,6 +65,17 @@ export function renderTileContent({ if (result.status === 'unresolved') { return chalk.yellow(result.resolution.message); } + if (result.status === 'heatmap') { + const data = result.grid && formatHeatmapGrid(result.grid); + if (!data) return chalk.dim('Not enough data points to draw a heatmap.'); + const numberFormat = heatmapNumberFormat(result.queriedConfig, source); + return renderHeatmapChart({ + data, + width, + height, + formatTick: axisTickFormatter(numberFormat), + }); + } const { queriedConfig, data } = result; const displayType = queriedConfig.displayType ?? DisplayType.Line; diff --git a/packages/cli/src/termchart/heatmap.ts b/packages/cli/src/termchart/heatmap.ts new file mode 100644 index 0000000000..bc1d91e099 --- /dev/null +++ b/packages/cli/src/termchart/heatmap.ts @@ -0,0 +1,106 @@ +/** + * Heatmap renderer: time across, value buckets down, each cell shaded by + * how many events landed in it. + */ + +import chalk from 'chalk'; + +import { + chalkColor, + defaultFormatValue, + NO_DATA_TEXT, + renderTimeAxis, + truncate, +} from '@/termchart/ansi'; +import { resampleLinear } from '@/termchart/scale'; +import type { HeatmapData, ValueFormatter } from '@/termchart/types'; + +const SHADES = ['░', '▒', '▓', '█']; +const SHADE_COLORS = ['blue', 'cyan', 'yellow', 'red']; +const LABEL_WIDTH = 10; + +/** The source index range each of `length` output slots covers, summed when shrinking. */ +function slotRange(slot: number, length: number, from: number, count: number) { + const start = from + Math.floor((slot * count) / length); + const end = Math.max( + start + 1, + from + Math.floor(((slot + 1) * count) / length), + ); + return [start, end] as const; +} + +export function renderHeatmapChart({ + data, + width, + height, + formatTick = defaultFormatValue, +}: { + data: HeatmapData; + width: number; + height: number; + formatTick?: ValueFormatter; +}): string { + const { times, rowEdges, counts } = data; + const rowCount = rowEdges.length - 1; + const populated = Array.from({ length: rowCount }, (_, r) => + counts.some(column => (column[r] ?? 0) > 0), + ); + const first = populated.indexOf(true); + const last = populated.lastIndexOf(true); + if (first === -1 || times.length === 0) return chalk.dim(NO_DATA_TEXT); + + const gutterWidth = LABEL_WIDTH + 2; + const plotWidth = Math.max(10, width - gutterWidth); + const plotHeight = Math.max(2, height - 3); + const visibleRows = last - first + 1; + + const cells = Array.from({ length: plotHeight }, (_, row) => { + const [r0, r1] = slotRange(row, plotHeight, first, visibleRows); + return Array.from({ length: plotWidth }, (_, col) => { + const [t0, t1] = slotRange(col, plotWidth, 0, times.length); + let sum = 0; + for (let t = t0; t < t1; t++) + for (let r = r0; r < r1; r++) sum += counts[t]?.[r] ?? 0; + return sum; + }); + }); + const max = Math.max(...cells.flat()); + + // Event counts are heavy-tailed, so shade on a log scale or one dense + // bucket washes out everything else. + const shade = (v: number) => { + if (v <= 0) return ' '; + const level = Math.min( + 3, + Math.floor((Math.log1p(v) / Math.log1p(max)) * 4 - 1e-9), + ); + return chalkColor(SHADE_COLORS[level])(SHADES[level]); + }; + + const labels = new Map([ + [plotHeight - 1, rowEdges[last + 1]], + [Math.floor(plotHeight / 2), rowEdges[first + Math.round(visibleRows / 2)]], + [0, rowEdges[first]], + ]); + const lines = cells + .map((columns, row) => { + const value = labels.get(row); + const label = + value === undefined + ? ' '.repeat(LABEL_WIDTH) + : truncate(formatTick(value), LABEL_WIDTH).padStart(LABEL_WIDTH); + return `${chalk.dim(label)} ${chalk.dim('┤')}${columns.map(shade).join('')}`; + }) + .reverse(); + + const axis = renderTimeAxis({ + timestamps: resampleLinear(times, plotWidth).map(Math.round), + gutterWidth, + plotWidth, + }); + const legend = `${SHADES.map((s, i) => chalkColor(SHADE_COLORS[i])(s)).join('')} ${chalk.dim( + `events per cell, max ${defaultFormatValue(max)}`, + )}`; + + return [...lines, axis, '', legend].join('\n'); +} diff --git a/packages/cli/src/termchart/index.ts b/packages/cli/src/termchart/index.ts index a84c69697e..597c3b2055 100644 --- a/packages/cli/src/termchart/index.ts +++ b/packages/cli/src/termchart/index.ts @@ -15,11 +15,13 @@ export { renderNumberChart, renderTableChart, } from '@/termchart/charts'; +export { renderHeatmapChart } from '@/termchart/heatmap'; export { niceTicks, resampleSeries } from '@/termchart/scale'; export { renderStackedBarChart } from '@/termchart/stackedBar'; export { renderLineChart } from '@/termchart/timeseries'; export type { CategoricalEntry, + HeatmapData, TableColumn, TimeChartData, TimeChartSeries, diff --git a/packages/cli/src/termchart/types.ts b/packages/cli/src/termchart/types.ts index 0badcb0303..1bf71d0e6a 100644 --- a/packages/cli/src/termchart/types.ts +++ b/packages/cli/src/termchart/types.ts @@ -51,3 +51,13 @@ export interface TableColumn { /** Group-by columns render left-aligned and skip numeric formatting */ isGroupColumn: boolean; } + +/** A heatmap: one column per time bucket, one row per value range. */ +export interface HeatmapData { + /** Column timestamps (unix seconds), ascending. */ + times: number[]; + /** `rows + 1` ascending row boundaries in data space. */ + rowEdges: number[]; + /** `counts[timeIdx][rowIdx]`: events in that time bucket and value range. */ + counts: number[][]; +} diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 1a2aee6743..eb63dcb67f 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -50,7 +50,7 @@ "dev:build": "tsup && tsc --emitDeclarationOnly --declaration && tsc-alias", "build": "tsup", "ci:build": "tsup", - "lint": "npx eslint . --ext .ts --max-warnings 80", + "lint": "npx eslint . --ext .ts --max-warnings 83", "lint:fix": "npx eslint . --ext .ts --fix", "ci:lint": "yarn lint && yarn tsc --noEmit", "ci:unit": "jest --ci --coverage", diff --git a/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts new file mode 100644 index 0000000000..a2bf4bd377 --- /dev/null +++ b/packages/common-utils/src/core/__tests__/distributionHeatmap.test.ts @@ -0,0 +1,157 @@ +import { queryDistributionHeatmap } from '@/core/distributionHeatmap'; +import { + heatmapBucketEdges, + HeatmapResultSchema, + heatmapRowEdges, + resolveDistributionGranularity, + toHeatmapResult, +} from '@/core/heatmap'; +import { timeBucketByGranularity } from '@/core/utils'; +import { DisplayType } from '@/types'; + +const dateRange: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-13T00:02:00Z'), +]; +const config = { + displayType: DisplayType.Heatmap, + select: [{ aggFn: 'count' as const, valueExpression: 'Duration' }], + from: { databaseName: 'default', tableName: 'otel_traces' }, + where: '', + timestampValueExpression: 'Timestamp', + connection: 'c1', + dateRange, +}; + +function fakeClient( + bounds: Record, + buckets: Record[], +) { + const queryChartConfig = jest + .fn() + .mockResolvedValueOnce({ meta: [], data: [bounds] }) + .mockResolvedValueOnce({ + meta: [ + { name: '__hdx_time_bucket', type: 'DateTime' }, + { name: 'x_bucket', type: 'UInt32' }, + { name: 'count', type: 'UInt64' }, + ], + data: buckets, + }); + return { queryChartConfig }; +} + +const run = (client: ReturnType, granularity = '1 minute') => + queryDistributionHeatmap({ + client, + metadata: {} as never, + config, + granularity, + targetTimeBuckets: 48, + nBuckets: 4, + querySettings: undefined, + }); + +describe('queryDistributionHeatmap', () => { + it('buckets between the bounds query result and shapes a grid', async () => { + const client = fakeClient({ min: '10', max: '1000' }, [ + { __hdx_time_bucket: '2026-10-13T00:00:00Z', x_bucket: '1', count: '7' }, + { __hdx_time_bucket: '2026-10-13T00:01:00Z', x_bucket: '4', count: '3' }, + ]); + const { heatmap } = await run(client); + + const bucketQuery = client.queryChartConfig.mock.calls[1][0].config; + expect(JSON.stringify(bucketQuery)).toContain('widthBucket('); + expect(heatmap?.scaleType).toBe('log'); + expect(heatmap?.grid.times).toHaveLength(2); + // nBuckets + under/overflow rows per time bucket + expect(heatmap?.grid.cells).toEqual([0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0]); + expect(heatmap?.bucketEdges[0]).toBeCloseTo(10); + expect(heatmap?.bucketEdges.at(-1)).toBeCloseTo(1000); + }); + + it('skips the bucket query when the range has no spread', async () => { + const client = fakeClient({ min: '5', max: '5' }, []); + const result = await run(client); + expect(result.heatmap).toBeUndefined(); + expect(client.queryChartConfig).toHaveBeenCalledTimes(1); + }); + + it('replaces an unsupported granularity instead of looping on it', async () => { + const client = fakeClient({ min: '5', max: '5' }, []); + const result = await run(client, '1 week'); + expect(result.granularity).not.toBe('1 week'); + expect(result.granularityNote).toContain('not supported'); + }); +}); + +describe('resolveDistributionGranularity', () => { + const day: [Date, Date] = [ + new Date('2026-10-13T00:00:00Z'), + new Date('2026-10-14T00:00:00Z'), + ]; + + it('keeps a supported granularity that fits the bucket budget', () => { + expect(resolveDistributionGranularity('1 hour', day, 48)).toEqual({ + granularity: '1 hour', + }); + }); + + it('coarsens a granularity that would flood the grid', () => { + const { granularity, note } = resolveDistributionGranularity( + '1 second', + day, + 48, + ); + expect(granularity).not.toBe('1 second'); + expect(note).toContain('86400 time buckets'); + }); +}); + +describe('heatmapBucketEdges', () => { + it('returns the widthBucket bounds, not the half-bucket display edges', () => { + expect(heatmapBucketEdges('linear', 0, 100, 4)).toEqual([ + 0, 25, 50, 75, 100, + ]); + }); +}); + +describe('timeBucketByGranularity', () => { + it('returns no buckets for a unit it cannot step by', () => { + expect( + timeBucketByGranularity(dateRange[0], dateRange[1], '1 week' as never), + ).toEqual([]); + }); +}); + +describe('heatmapRowEdges', () => { + it('adds an under- and overflow row one bucket tall on each side', () => { + expect(heatmapRowEdges([0, 25, 50], 'linear')).toEqual([ + -25, 0, 25, 50, 75, + ]); + expect(heatmapRowEdges([1, 10, 100], 'log')).toEqual([ + 0.1, 1, 10, 100, 1000, + ]); + }); +}); + +describe('toHeatmapResult', () => { + it('keeps exact bucket edges and fits the shared schema', () => { + const result = toHeatmapResult( + { + grid: { + times: [Date.parse('2026-10-13T00:00:00Z')], + stepMs: 0, + yAxis: { type: 'numeric', scale: 'linear', edges: [0, 1, 2, 3, 4] }, + cells: [0, 1, 2, 0], + }, + scaleType: 'linear', + bucketEdges: [1000000000, 1000000050, 1000000100], + }, + { valueExpression: 'Duration', granularity: '1 minute' }, + ); + expect(result.valueEdges).toEqual([1000000000, 1000000050, 1000000100]); + expect(result.counts).toEqual([[0, 1, 2, 0]]); + expect(HeatmapResultSchema.safeParse(result).success).toBe(true); + }); +}); diff --git a/packages/common-utils/src/core/distributionHeatmap.ts b/packages/common-utils/src/core/distributionHeatmap.ts new file mode 100644 index 0000000000..71eb7c526f --- /dev/null +++ b/packages/common-utils/src/core/distributionHeatmap.ts @@ -0,0 +1,115 @@ +import type { BaseClickhouseClient } from '@/clickhouse'; +import { inferTimestampColumn } from '@/clickhouse'; +import type { HeatmapGrid, HeatmapScaleType } from '@/core/heatmap'; +import { + computeEffectiveMin, + gridFromBucketRows, + heatmapBucketEdges, + resolveDistributionGranularity, +} from '@/core/heatmap'; +import { + buildHeatmapBoundsConfig, + buildHeatmapBucketConfig, + toDistributionHeatmapConfig, +} from '@/core/heatmapQuery'; +import type { Metadata } from '@/core/metadata'; +import { timeBucketByGranularity } from '@/core/utils'; +import type { + BuilderChartConfigWithDateRange, + QuerySettings, + SQLInterval, +} from '@/types'; + +/** + * Runs a distribution heatmap outside the browser: the bounds query, then the + * `widthBucket` query, shaped into a grid. `heatmap` is absent when the range + * holds too few distinct values to bucket. + */ +export async function queryDistributionHeatmap({ + client, + metadata, + config, + granularity: requestedGranularity, + targetTimeBuckets, + nBuckets, + querySettings, + opts, +}: { + client: Pick; + metadata: Metadata; + config: BuilderChartConfigWithDateRange; + granularity: string | undefined; + targetTimeBuckets: number; + nBuckets: number; + querySettings: QuerySettings | undefined; + opts?: Parameters[0]['opts']; +}): Promise<{ + granularity: SQLInterval; + granularityNote?: string; + heatmap?: { + grid: HeatmapGrid; + scaleType: HeatmapScaleType; + bucketEdges: number[]; + }; +}> { + const { granularity, note: granularityNote } = resolveDistributionGranularity( + requestedGranularity, + config.dateRange, + targetTimeBuckets, + ); + const { config: heatmapConfig, scaleType } = + toDistributionHeatmapConfig(config); + + const bounds = await client.queryChartConfig({ + config: buildHeatmapBoundsConfig({ + config: heatmapConfig, + scaleType, + granularity, + }), + metadata, + querySettings, + opts, + }); + // quantile returns floats; UInt64 come back as strings + const min = Number.parseFloat(String(bounds.data?.[0]?.['min'] ?? '0')); + const max = Number.parseFloat(String(bounds.data?.[0]?.['max'] ?? '0')); + const effectiveMin = computeEffectiveMin(min, max, scaleType); + if (!(max > effectiveMin)) return { granularity, granularityNote }; + + const buckets = await client.queryChartConfig({ + config: buildHeatmapBucketConfig({ + config: heatmapConfig, + scaleType, + effectiveMin, + max, + granularity, + nBuckets, + }), + metadata, + querySettings, + opts, + }); + const timestampColumn = inferTimestampColumn(buckets.meta ?? []); + if (timestampColumn == null) { + throw new Error('Heatmap bucket query returned no time bucket column'); + } + + const [start, end] = config.dateRange; + return { + granularity, + granularityNote, + heatmap: { + grid: gridFromBucketRows({ + data: buckets.data, + timestampColumn, + generatedTsBuckets: timeBucketByGranularity(start, end, granularity), + scaleType, + effectiveMin, + max, + nBuckets, + }), + scaleType, + bucketEdges: heatmapBucketEdges(scaleType, effectiveMin, max, nBuckets), + }, + }; +} diff --git a/packages/common-utils/src/core/heatmap.ts b/packages/common-utils/src/core/heatmap.ts index 2018299a00..364c950515 100644 --- a/packages/common-utils/src/core/heatmap.ts +++ b/packages/common-utils/src/core/heatmap.ts @@ -1,4 +1,12 @@ -import type { PromqlHeatmapMode } from '@/types'; +import { z } from 'zod'; + +import type { ColumnMetaType } from '@/clickhouse'; +import { + convertDateRangeToGranularityString, + convertGranularityToSeconds, +} from '@/core/utils'; +import type { NumberFormat, PromqlHeatmapMode, SQLInterval } from '@/types'; +import { SQLIntervalSchema } from '@/types'; /** A heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ export function getHeatmapMode(config: { @@ -6,3 +14,259 @@ export function getHeatmapMode(config: { }): M | 'distribution' { return config.heatmap?.mode ?? 'distribution'; } + +export type HeatmapScaleType = 'log' | 'linear'; + +export type HeatmapYAxis = + | { + type: 'numeric'; + scale: HeatmapScaleType; + /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ + edges: number[]; + } + /** + * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per + * row; the top bound may be `Infinity` (a `+Inf` bucket). + */ + | { type: 'buckets'; bounds: number[] } + | { type: 'series'; labels: string[] }; + +/** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ +export type HeatmapGrid = { + /** Column centers, ms ascending. */ + times: number[]; + /** Column width, ms. */ + stepMs: number; + yAxis: HeatmapYAxis; + /** Row-major: `cells[timeIdx * rowCount + rowIdx]`. */ + cells: number[]; +}; + +export function heatmapRowCount(yAxis: HeatmapYAxis) { + switch (yAxis.type) { + case 'series': + return yAxis.labels.length; + case 'buckets': + return yAxis.bounds.length; + case 'numeric': + return Math.max(0, yAxis.edges.length - 1); + } +} + +/** + * Quantile used for the y-axis lower bound. A quantile (rather than min) + * keeps near-zero outliers from stretching the log axis. The upper bound + * uses the actual max so that rare latency spikes stay visible; log scale + * already compresses wide ranges. + */ +export function heatmapLowQuantile(scaleType: HeatmapScaleType) { + return scaleType === 'log' ? 0.01 : 0.001; +} + +/** + * Lower bound actually used for bucketing. Log scale needs a positive + * minimum (log(0) is undefined), and the range is capped to ~4 orders of + * magnitude so the axis isn't dominated by a long empty tail of near-zero + * outliers. + */ +export function computeEffectiveMin( + min: number, + max: number, + scaleType: HeatmapScaleType, +) { + return scaleType === 'log' ? Math.max(min, max * 1e-4 || 1e-4) : min; +} + +function isLogBuckets(scaleType: HeatmapScaleType, min: number, max: number) { + return scaleType === 'log' && min > 0 && max > min; +} + +/** Upper bound of `widthBucket` bucket j (bucket 0 is the underflow). */ +function bucketValueFn( + scaleType: HeatmapScaleType, + effectiveMin: number, + max: number, + nBuckets: number, +) { + return isLogBuckets(scaleType, effectiveMin, max) + ? (j: number) => effectiveMin * Math.pow(max / effectiveMin, j / nBuckets) + : (j: number) => effectiveMin + j * ((max - effectiveMin) / nBuckets); +} + +/** + * The `nBuckets + 1` bounds `widthBucket` counted between: bucket j covers + * [edges[j - 1], edges[j]). The grid's edges are display edges, half a bucket off. + */ +export function heatmapBucketEdges( + scaleType: HeatmapScaleType, + effectiveMin: number, + max: number, + nBuckets: number, +): number[] { + const bucketValue = bucketValueFn(scaleType, effectiveMin, max, nBuckets); + return Array.from({ length: nBuckets + 1 }, (_, j) => bucketValue(j)); +} + +/** + * Row edges for drawing the `nBuckets + 2` rows: the bucket bounds, with the + * under- and overflow rows each given one bucket's height. + */ +export function heatmapRowEdges( + bucketEdges: number[], + scaleType: HeatmapScaleType, +): number[] { + const [first, second] = [bucketEdges[0]!, bucketEdges[1] ?? bucketEdges[0]!]; + const last = bucketEdges.at(-1)!; + const beforeLast = bucketEdges.at(-2) ?? last; + return scaleType === 'log' && first > 0 + ? [(first * first) / second, ...bucketEdges, (last * last) / beforeLast] + : [first - (second - first), ...bucketEdges, last + (last - beforeLast)]; +} + +/** + * A distribution heatmap as returned to agents and scripts: `counts[t][r]`, + * where row 0 counts values below `valueEdges[0]`, row r counts + * [valueEdges[r - 1], valueEdges[r]) and the last row values at or above the + * last edge. + */ +export const HeatmapResultSchema = z.object({ + valueExpression: z.string(), + scale: z.enum(['log', 'linear']), + granularity: z.string(), + // Typed but not re-validated: NumberFormatSchema drags every types.ts schema + // into the Claude plugin's bundle, which then fails to load in Claude Code. + numberFormat: z + .custom(v => typeof v === 'object' && v !== null) + .optional(), + times: z.array(z.string()), + valueEdges: z.array(z.number()), + counts: z.array(z.array(z.number())), +}); +export type HeatmapResult = z.infer; + +export function toHeatmapResult( + heatmap: { + grid: HeatmapGrid; + scaleType: HeatmapScaleType; + bucketEdges: number[]; + }, + context: Pick< + HeatmapResult, + 'valueExpression' | 'granularity' | 'numberFormat' + >, +): HeatmapResult { + return { + ...context, + scale: heatmap.scaleType, + times: heatmap.grid.times.map(t => new Date(t).toISOString()), + valueEdges: heatmap.bucketEdges, + counts: heatmapCountColumns(heatmap.grid), + }; +} + +/** Cells as `[timeIdx][rowIdx]`. */ +export function heatmapCountColumns(grid: HeatmapGrid): number[][] { + const rows = heatmapRowCount(grid.yAxis); + return grid.times.map((_, t) => grid.cells.slice(t * rows, (t + 1) * rows)); +} + +/** + * A time bucket size for a heatmap outside the browser: the requested one when + * it is supported and not so fine it floods the grid, else one sized for + * `targetBuckets`, with a note saying why. + */ +export function resolveDistributionGranularity( + granularity: string | undefined, + dateRange: [Date, Date], + targetBuckets: number, +): { granularity: SQLInterval; note?: string } { + const auto = convertDateRangeToGranularityString(dateRange, targetBuckets); + if (!granularity || granularity === 'auto') return { granularity: auto }; + if (!SQLIntervalSchema.safeParse(granularity).success) { + return { + granularity: auto, + note: `Granularity "${granularity}" is not supported (use " second|minute|hour|day"); used "${auto}".`, + }; + } + const seconds = convertGranularityToSeconds(granularity as SQLInterval); + const buckets = + (dateRange[1].getTime() - dateRange[0].getTime()) / 1000 / seconds; + if (!(seconds > 0) || buckets > targetBuckets * 4) { + return { + granularity: auto, + note: `Granularity "${granularity}" would make ${Math.round(buckets)} time buckets; used "${auto}".`, + }; + } + return { granularity: granularity as SQLInterval }; +} + +/** + * Grid for the rows of the server-side `widthBucket` query. widthBucket + * returns buckets 0 (underflow) through nBuckets + 1 (overflow), so each time + * bucket has nBuckets + 2 rows. + */ +export function gridFromBucketRows({ + data, + timestampColumn, + generatedTsBuckets, + scaleType, + effectiveMin, + max, + nBuckets, +}: { + data: Record[]; + timestampColumn: ColumnMetaType; + generatedTsBuckets: Date[]; + scaleType: HeatmapScaleType; + effectiveMin: number; + max: number; + nBuckets: number; +}): HeatmapGrid { + const isLog = isLogBuckets(scaleType, effectiveMin, max); + const bucketValue = bucketValueFn(scaleType, effectiveMin, max, nBuckets); + + // Row j is centered on bucketValue(j), which is the upper bound of + // widthBucket's bucket j, so the row boundaries sit half a bucket either + // side of it. + const rows = nBuckets + 2; + const edges = Array.from({ length: rows + 1 }, (_, k) => + bucketValue(k - 0.5), + ); + + const times = generatedTsBuckets.map(d => d.getTime()); + const cells: number[] = []; + + let dataIndex = 0; + for (const generatedTs of times) { + for (let j = 0; j < rows; j++) { + const row = data[dataIndex]; + + if ( + row != null && + new Date(row[timestampColumn.name]).getTime() == generatedTs && + row['x_bucket'] == j + ) { + cells.push(Number.parseInt(row['count'], 10)); // UInt64 returns as string + + // Skip duplicate buckets (from unmerged distributed table results) + while ( + dataIndex < data.length && + new Date(data[dataIndex][timestampColumn.name]).getTime() == + generatedTs && + data[dataIndex]['x_bucket'] == j + ) { + dataIndex++; + } + } else { + cells.push(0); + } + } + } + + return { + times, + stepMs: times.length > 1 ? times[1] - times[0] : 0, + yAxis: { type: 'numeric', scale: isLog ? 'log' : 'linear', edges }, + cells, + }; +} diff --git a/packages/common-utils/src/core/heatmapQuery.ts b/packages/common-utils/src/core/heatmapQuery.ts new file mode 100644 index 0000000000..d1e924abfc --- /dev/null +++ b/packages/common-utils/src/core/heatmapQuery.ts @@ -0,0 +1,225 @@ +import type { HeatmapScaleType } from '@/core/heatmap'; +import { heatmapLowQuantile } from '@/core/heatmap'; +import { getAlignedDateRange, isAggregateFunction } from '@/core/utils'; +import type { BuilderChartConfigWithDateRange, SQLInterval } from '@/types'; +import { DisplayType } from '@/types'; + +export type HeatmapChartConfig = { + displayType: DisplayType.Heatmap; + select: [ + { + aggFn: 'heatmap'; + valueExpression: string; + countExpression?: string; + }, + ]; + from: BuilderChartConfigWithDateRange['from']; + where: BuilderChartConfigWithDateRange['where']; + dateRange: BuilderChartConfigWithDateRange['dateRange']; + granularity: BuilderChartConfigWithDateRange['granularity']; + timestampValueExpression: BuilderChartConfigWithDateRange['timestampValueExpression']; + numberFormat?: BuilderChartConfigWithDateRange['numberFormat']; + filters?: BuilderChartConfigWithDateRange['filters']; + connection: string; + with?: BuilderChartConfigWithDateRange['with']; +}; + +/** + * The query a distribution heatmap runs, from a builder config whose first + * select item names the value to bucket. + */ +export function toDistributionHeatmapConfig( + config: BuilderChartConfigWithDateRange, +): { config: HeatmapChartConfig; scaleType: HeatmapScaleType } { + const firstSelect = Array.isArray(config.select) + ? config.select[0] + : undefined; + return { + config: { + ...config, + displayType: DisplayType.Heatmap, + select: [ + { + aggFn: 'heatmap' as const, + valueExpression: firstSelect?.valueExpression ?? '', + countExpression: firstSelect?.countExpression, + }, + ], + granularity: config.granularity, + numberFormat: config.numberFormat, + }, + scaleType: firstSelect?.heatmapScaleType ?? 'log', + }; +} + +/** + * Build the bounds (min/max) ChartConfig that runs first. Result feeds + * `effectiveMin`/`max` into `buildHeatmapBucketConfig`. + */ +export function buildHeatmapBoundsConfig({ + config, + scaleType, + granularity, +}: { + config: HeatmapChartConfig; + scaleType: HeatmapScaleType; + granularity: SQLInterval; +}): BuilderChartConfigWithDateRange { + const valueExpression = config.select[0].valueExpression; + const isAggregateExpression = isAggregateFunction(valueExpression); + const qLo = heatmapLowQuantile(scaleType); + + return isAggregateExpression + ? { + ...config, + where: '', + orderBy: undefined, + granularity: undefined, + select: [ + { + aggFn: 'quantile' as const, + level: qLo, + aggCondition: `value_calc >= 0`, + aggConditionLanguage: 'sql', + valueExpression: 'value_calc', + alias: 'min', + }, + { + aggFn: 'max' as const, + valueExpression: 'value_calc', + alias: 'max', + }, + ], + with: [ + { + name: 'min_max_calc', + chartConfig: { + ...config, + select: [{ valueExpression, alias: 'value_calc' }], + orderBy: undefined, + // Emits the __hdx_time_bucket column the outer query filters on. + granularity, + }, + }, + ], + timestampValueExpression: '__hdx_time_bucket', + // The first bucket is labelled before an unaligned start, so align + // this filter on bucket labels. The inner query keeps the original + // range so no events outside it are counted. + dateRange: getAlignedDateRange(config.dateRange, granularity), + from: { databaseName: '', tableName: 'min_max_calc' }, + } + : { + ...config, + orderBy: undefined, + granularity: undefined, + select: [ + { + aggFn: 'quantile' as const, + level: qLo, + valueExpression, + aggCondition: `${valueExpression} >= 0`, + aggConditionLanguage: 'sql', + alias: 'min', + }, + { + aggFn: 'max' as const, + valueExpression, + alias: 'max', + }, + ], + }; +} + +/** + * Build the bucketed-counts ChartConfig that runs second. `effectiveMin`/`max` + * are usually numbers (resolved from the bounds query), but accept strings so + * callers — like the editor's SQL preview — can pass placeholder tokens + * (e.g. `'{min}'`) before the bounds are known. + */ +export function buildHeatmapBucketConfig({ + config, + scaleType, + effectiveMin, + max, + granularity, + nBuckets, +}: { + config: HeatmapChartConfig; + scaleType: HeatmapScaleType; + effectiveMin: string | number; + max: string | number; + granularity: SQLInterval; + nBuckets: number; +}): BuilderChartConfigWithDateRange { + const valueExpression = config.select[0].valueExpression; + // The chart editor saves a cleared Count input as ''. + const countExpression = config.select[0].countExpression?.trim() || 'count()'; + const isAggregateExpression = isAggregateFunction(valueExpression); + + const bucketExprAgg = + scaleType === 'log' + ? `widthBucket(log(greatest(toFloat64(value_calc), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` + : `widthBucket(value_calc, ${effectiveMin}, ${max}, ${nBuckets})`; + const bucketExprDirect = + scaleType === 'log' + ? `widthBucket(log(greatest(toFloat64(${valueExpression}), ${effectiveMin})), log(${effectiveMin}), log(${max}), ${nBuckets})` + : `widthBucket(${valueExpression}, ${effectiveMin}, ${max}, ${nBuckets})`; + + return isAggregateExpression + ? { + ...config, + where: '', + select: [ + { + valueExpression: 'sum(value_count)', + alias: 'count', + }, + ], + groupBy: [ + { + valueExpression: bucketExprAgg, + alias: 'x_bucket', + }, + ], + with: [ + { + name: 'bucket_calc', + chartConfig: { + ...config, + select: [ + { valueExpression, alias: 'value_calc' }, + { + valueExpression: countExpression, + alias: 'value_count', + }, + ], + granularity, + orderBy: undefined, + }, + }, + ], + timestampValueExpression: '__hdx_time_bucket', + dateRange: getAlignedDateRange(config.dateRange, granularity), + from: { databaseName: '', tableName: 'bucket_calc' }, + orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], + granularity, + } + : { + ...config, + select: [ + { + valueExpression: countExpression, + alias: 'count', + }, + ], + groupBy: [ + { + valueExpression: bucketExprDirect, + alias: 'x_bucket', + }, + ], + orderBy: [{ valueExpression: 'x_bucket', ordering: 'ASC' }], + granularity, + }; +} diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index e249239c9c..e634aad6e3 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -709,6 +709,8 @@ export function timeBucketByGranularity( let current = toStartOfInterval(start, granularity); const granularitySeconds = convertGranularityToSeconds(granularity); + // Units beyond day convert to 0 seconds, which would never advance. + if (!(granularitySeconds > 0)) return buckets; while (current < end) { buckets.push(current); current = fnsAdd(current, { @@ -1877,3 +1879,113 @@ export type MetadataMVQueryOptions = Map< string, Map<'NativeColumn' | string, string[]> // map from column name to keys >; + +/** Whether a SQL expression calls an aggregate function. */ +export const isAggregateFunction = (value: string) => { + const fns = [ + // Basic aggregates + 'count', + 'countIf', + 'countDistinct', + 'sum', + 'sumIf', + 'avg', + 'avgIf', + 'min', + 'max', + 'any', + 'anyLast', + 'groupArray', + 'groupArrayInsertAt', + 'groupArrayMovingAvg', + 'groupArraySample', + 'groupUniqArray', + 'groupUniqArrayIf', + 'groupArrayIntersect', + 'groupArrayIntersectIf', + 'groupArrayReduce', + 'groupBitmap', + 'groupBitmapIf', + 'groupBitmapOr', + 'groupBitmapXor', + + // Quantiles + 'quantile', + 'quantileIf', + 'quantileExact', + 'quantileExactWeighted', + 'quantileTiming', + 'quantileTimingWeighted', + 'quantileTDigest', + 'quantileTDigestWeighted', + 'quantileBFloat16', + 'quantileBFloat16Weighted', + 'quantiles', + 'median', + 'medianExact', + 'medianTDigest', + 'medianBFloat16', + + // Statistical functions + 'stddevPop', + 'stddevPopIf', + 'stddevSamp', + 'stddevSampIf', + 'varPop', + 'varPopIf', + 'varSamp', + 'varSampIf', + 'covarPop', + 'covarSamp', + 'corr', + + // Combinators + 'uniq', + 'uniqExact', + 'uniqCombined', + 'uniqCombined64', + 'uniqHLL12', + 'uniqTheta', + + // Bit operations + 'groupBitAnd', + 'groupBitOr', + 'groupBitXor', + + // Map and tuple + 'groupArrayMap', + 'groupArrayTuple', + 'groupArraySorted', + 'topK', + 'topKIf', + 'topKWeighted', + + // Aggregate combinators + 'argMin', + 'argMax', + 'minMap', + 'maxMap', + + // Specialized aggregates + 'runningDifference', + 'retention', + 'sequenceCount', + 'sequenceMatch', + 'histogram', + 'simpleLinearRegression', + 'stochasticLinearRegression', + 'categoricalInformationValue', + 'sumMap', + 'sumMapFiltered', + 'sumWithOverflow', + 'entropy', + 'skewPop', + 'skewSamp', + 'kurtPop', + 'kurtSamp', + ]; + + // Make case-insensitive since ClickHouse function names are case-insensitive + const lowerValue = value.toLowerCase(); + return fns.some(fn => lowerValue.includes(fn.toLowerCase() + '(')); +};