diff --git a/.changeset/nice-cows-hug.md b/.changeset/nice-cows-hug.md new file mode 100644 index 0000000000..74f51a6c2f --- /dev/null +++ b/.changeset/nice-cows-hug.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Support distribution heatmaps calculated from promql data diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 6be88f101e..d739f6d71c 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -65,7 +65,8 @@ export type ChartConfigDisplaySettings = Pick< // PromQL-only: Handlebars template over each series' Prometheus label set // that renders the legend/tooltip name. legendTemplate?: string; - // Distribution heatmaps only; stored on the heatmap's select[0]. + // Distribution heatmaps only; stored on a builder heatmap's select[0] and + // on a PromQL heatmap's `heatmap.scaleType`. heatmapScaleType?: HeatmapScaleType; }; @@ -90,6 +91,8 @@ interface ChartDisplaySettingsDrawerProps { promqlUsesRange?: boolean; /** A builder heatmap's mode; only distribution heatmaps have a numeric y axis to scale. */ heatmapMode?: HeatmapMode; + /** The heatmap scale used when the tile has none set. */ + defaultHeatmapScaleType?: HeatmapScaleType; previousDateRange?: [Date, Date]; onChange: (settings: ChartConfigDisplaySettings, isDirty: boolean) => void; onClose: () => void; @@ -98,7 +101,8 @@ interface ChartDisplaySettingsDrawerProps { function applyDefaultSettings( settings: ChartConfigDisplaySettings, - fallbackNumberFormat?: NumberFormat, + fallbackNumberFormat: NumberFormat | undefined, + defaultHeatmapScaleType: HeatmapScaleType, ): DrawerFormValues { return { numberFormat: @@ -121,7 +125,7 @@ function applyDefaultSettings( ? attachLocalIds(settings.colorRules) : undefined, backgroundChart: settings.backgroundChart, - heatmapScaleType: settings.heatmapScaleType ?? 'log', + heatmapScaleType: settings.heatmapScaleType ?? defaultHeatmapScaleType, }; } @@ -132,6 +136,7 @@ export default function ChartDisplaySettingsDrawer({ configType, promqlUsesRange = false, heatmapMode, + defaultHeatmapScaleType = 'log', defaultNumberFormat, onChange, onClose, @@ -139,8 +144,13 @@ export default function ChartDisplaySettingsDrawer({ isPerSeriesNumberFormatAllowed = false, }: ChartDisplaySettingsDrawerProps) { const appliedDefaults = useMemo( - () => applyDefaultSettings(settings, defaultNumberFormat), - [settings, defaultNumberFormat], + () => + applyDefaultSettings( + settings, + defaultNumberFormat, + defaultHeatmapScaleType, + ), + [settings, defaultNumberFormat, defaultHeatmapScaleType], ); const { @@ -197,9 +207,10 @@ export default function ChartDisplaySettingsDrawer({ applyDefaultSettings( {} as ChartConfigDisplaySettings, defaultNumberFormat, + defaultHeatmapScaleType, ), ); - }, [reset, defaultNumberFormat]); + }, [reset, defaultNumberFormat, defaultHeatmapScaleType]); const isTimeChart = isTimeSeriesDisplayType(displayType); @@ -212,12 +223,17 @@ export default function ChartDisplaySettingsDrawer({ const isClientSideSeriesLimit = showSeriesLimit && (configType === 'sql' || configType === 'promql'); + const isDistributionHeatmap = + displayType === DisplayType.Heatmap && heatmapMode === 'distribution'; + // Every PromQL display that surfaces a series name. A number tile shows one - // value and a table gives each label its own column, so neither has a legend. + // value, a table gives each label its own column, and a distribution heatmap + // merges every series into one, so none has a legend. const showLegendTemplate = configType === 'promql' && displayType !== DisplayType.Number && - displayType !== DisplayType.Table; + displayType !== DisplayType.Table && + !isDistributionHeatmap; // On pie/bar builder charts, seriesLimit becomes a plain SQL LIMIT on the // number of slices/bars; on PromQL it trims the reduced series client-side @@ -254,8 +270,7 @@ export default function ChartDisplaySettingsDrawer({ : 'Available on query-builder number tiles.'; // Log scale only means something on a numeric (distribution mode) y axis - const showHeatmapScale = - displayType === DisplayType.Heatmap && heatmapMode === 'distribution'; + const showHeatmapScale = isDistributionHeatmap; return ( ( )} diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 07313da232..58204d60e4 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -11,12 +11,15 @@ import { } from '@hyperdx/common-utils/dist/core/promql'; import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { + DisplayType, + HeatmapMode, MAX_PROMQL_EXPRESSIONS, SourceKind, } from '@hyperdx/common-utils/dist/types'; -import { Button, Divider, Flex, Group, Text } from '@mantine/core'; +import { Box, Button, Divider, Flex, Group, Text } from '@mantine/core'; import { IconCirclePlus } from '@tabler/icons-react'; +import { HeatmapModeControl } from '@/components/DBEditTimeChartForm/HeatmapModeControl'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; import { useSource } from '@/source'; @@ -30,12 +33,16 @@ export default function PromqlChartEditor({ allowedSourceKinds, onSubmit, onOpenDisplaySettings, + heatmapMode, + onHeatmapModeChange, }: { control: Control; getValues: UseFormGetValues; allowedSourceKinds: SourceKind[]; onSubmit: (suppressErrorNotification?: boolean) => void; onOpenDisplaySettings: () => void; + heatmapMode: HeatmapMode; + onHeatmapModeChange: (mode: HeatmapMode) => void; }) { const { fields: expressions, @@ -106,6 +113,14 @@ export default function PromqlChartEditor({ allowedSourceKinds={allowedSourceKinds} /> + {displayType === DisplayType.Heatmap && ( + + + + )} {expressions.map((field, index) => ( { series: [], }); - it('saves a heatmap with only its first expression queried and no heatmap mode', () => { + it('saves a heatmap with only its first expression queried and its heatmap settings', () => { const result = convertFormStateToSavedChartConfig( { ...promqlForm([{ expression: 'up' }], DisplayType.Heatmap), legendTemplate: '{{pod}}', - heatmap: { mode: 'distribution' }, + heatmap: { mode: 'distribution', scaleType: 'log' }, granularity: '5 minute', }, undefined, @@ -684,8 +684,37 @@ describe('PromQL expressions', () => { promqlExpression: [{ expression: 'up' }], legendTemplate: '{{pod}}', granularity: '5 minute', + heatmap: { mode: 'distribution', scaleType: 'log' }, }); - expect(result).not.toHaveProperty('heatmap'); + }); + + it('queries a heatmap with its heatmap settings', () => { + const result = convertFormStateToChartConfig( + { + ...promqlForm([{ expression: 'up' }], DisplayType.Heatmap), + heatmap: { mode: 'series' }, + }, + [new Date(0), new Date(1000)], + undefined, + ); + expect(result).toMatchObject({ heatmap: { mode: 'series' } }); + }); + + it('drops heatmap settings from PromQL tiles that are not heatmaps', () => { + const form = { + ...promqlForm([{ expression: 'up' }], DisplayType.Line), + heatmap: { mode: 'series' as const }, + }; + expect( + convertFormStateToSavedChartConfig(form, undefined), + ).not.toHaveProperty('heatmap', expect.anything()); + expect( + convertFormStateToChartConfig( + form, + [new Date(0), new Date(1000)], + undefined, + ), + ).not.toHaveProperty('heatmap', expect.anything()); }); it('saves the form rows as the expression list', () => { diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 0709cf6974..2c2e4784a4 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -280,6 +280,7 @@ export function convertFormStateToSavedChartConfig( connection: form.connection ?? '', source: form.source || undefined, legendTemplate: form.legendTemplate?.trim() || undefined, + heatmap: formPromqlHeatmap(form), }; return promqlConfig; @@ -380,6 +381,7 @@ export function convertFormStateToChartConfig( source: form.source || undefined, from: source?.from, legendTemplate: form.legendTemplate?.trim() || undefined, + heatmap: formPromqlHeatmap(form), }; return { ...promqlConfig, dateRange }; @@ -552,6 +554,13 @@ function formPromqlExpressions(form: ChartEditorFormState): PromqlSeries[] { })); } +/** Heatmap settings are dropped from PromQL tiles that are not heatmaps. */ +function formPromqlHeatmap( + form: ChartEditorFormState, +): PromqlSavedChartConfig['heatmap'] { + return form.displayType === DisplayType.Heatmap ? form.heatmap : undefined; +} + export const validateChartForm = ( form: ChartEditorFormState, source: TSource | undefined, diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx index d6ead0b592..eedb420f8b 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx @@ -106,6 +106,11 @@ function HeatmapSQLPreview({ }; const query = toHeatmapQuery(configWithTimestamp); + // Only PromQL heatmaps are calculated, and they have no SQL to preview + if (query.mode === 'calculated') { + return null; + } + if (query.mode === 'series') { const seriesConfig = { ...query.config, dateRange }; return ( diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 55d7f2d62c..9ccab64d5d 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -18,7 +18,6 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { displayTypeRequiresSource, - getHeatmapSourceKinds, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -76,7 +75,10 @@ import { isStringSelectDisplayType, validateChartForm, } from '@/components/ChartEditor/utils'; -import { HEATMAP_DURATION_NUMBER_FORMAT } from '@/components/DBHeatmapChart'; +import { + HEATMAP_DURATION_NUMBER_FORMAT, + PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, +} from '@/components/DBHeatmapChart'; import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { InputControlled } from '@/components/InputControlled'; import SaveToDashboardModal from '@/components/SaveToDashboardModal'; @@ -161,7 +163,8 @@ const ALERT_IGNORES_DASHBOARD_FILTERS = * Populate form state with the standard series for a heatmap mode. * Distribution heatmaps bucket the trace duration (or `fallbackValueExpression` * on a trace source without one) with a duration number format; series - * heatmaps start from a plain count. + * heatmaps start from a plain count. PromQL distribution heatmaps bucket + * arbitrary samples, so they get no duration format. */ function applyHeatmapDefaults( setValue: UseFormSetValue, @@ -170,10 +173,12 @@ function applyHeatmapDefaults( mode, tableSource, fallbackValueExpression = '', + isPromql = false, }: { mode: HeatmapMode; tableSource: TSource | undefined; fallbackValueExpression?: string; + isPromql?: boolean; }, ) { const isTraceDuration = @@ -194,13 +199,13 @@ function applyHeatmapDefaults( setValue('select', heatmapSeries); setValue('series', heatmapSeries); - if (mode === 'distribution') { + if (mode === 'distribution' && !isPromql) { setValue('series.0.countExpression', 'count()'); setValue('numberFormat', { ...HEATMAP_DURATION_NUMBER_FORMAT }); } else if ( isEqual(getValues('numberFormat'), HEATMAP_DURATION_NUMBER_FORMAT) ) { - // Unselect the default-applied duration format for series mode + // Unselect the default-applied duration format for builder series-mode setValue('numberFormat', undefined); } } @@ -310,7 +315,7 @@ export default function EditTimeChartForm({ const configType = useWatch({ control, name: 'configType' }); const connection = useWatch({ control, name: 'connection' }); const promqlExpressions = useWatch({ control, name: 'promqlExpressions' }); - const formHeatmapMode = getHeatmapMode({ + const heatmapMode = getHeatmapMode({ heatmap: useWatch({ control, name: 'heatmap' }), }); @@ -319,8 +324,6 @@ export default function EditTimeChartForm({ configType === 'sql' && isRawSqlDisplayType(displayType); const isPromqlInput = configType === 'promql' && isPromqlDisplayType(displayType); - // PromQL heatmaps only support series mode - const heatmapMode: HeatmapMode = isPromqlInput ? 'series' : formHeatmapMode; const { data: sources } = useSources(); @@ -382,7 +385,8 @@ export default function EditTimeChartForm({ colorRules, backgroundChart, legendTemplate, - heatmapScaleType, + builderHeatmapScaleType, + promqlHeatmapScaleType, ] = useWatch({ control, name: [ @@ -399,8 +403,12 @@ export default function EditTimeChartForm({ 'backgroundChart', 'legendTemplate', 'series.0.heatmapScaleType', + 'heatmap.scaleType', ], }); + const heatmapScaleType = isPromqlInput + ? promqlHeatmapScaleType + : builderHeatmapScaleType; // Format auto-detected purely from the datasource (e.g. duration for a trace // Duration column), used as the drawer's fallback when no explicit @@ -825,6 +833,7 @@ export default function EditTimeChartForm({ fallbackValueExpression: Array.isArray(select) ? (select[0]?.valueExpression ?? '') : '', + isPromql: isPromqlInput, }); } else if (!Array.isArray(select)) { const defaultSeries: SavedChartConfigWithSelectArray['select'] = [ @@ -856,21 +865,38 @@ export default function EditTimeChartForm({ configType, tableSource, heatmapMode, + isPromqlInput, ]); // Switching heatmap mode may trigger a source swap, and should apply new defaults. const onHeatmapModeChange = useCallback( (mode: HeatmapMode) => { setValue('heatmap.mode', mode); - applyHeatmapDefaults(setValue, getValues, { mode, tableSource }); + applyHeatmapDefaults(setValue, getValues, { + mode, + tableSource, + isPromql: isPromqlInput, + }); if ( tableSource != null && - getHeatmapSourceKinds(mode).includes(tableSource.kind) + getAllowedSourceKinds({ + configType, + displayType, + heatmapMode: mode, + }).includes(tableSource.kind) ) { onSubmit(true); } }, - [setValue, getValues, tableSource, onSubmit], + [ + setValue, + getValues, + tableSource, + onSubmit, + configType, + displayType, + isPromqlInput, + ], ); // Handle auto-submitting and form state updates when the source changes. @@ -995,7 +1021,11 @@ export default function EditTimeChartForm({ displayType === DisplayType.Heatmap && heatmapMode === 'distribution' ) { - setValue('series.0.heatmapScaleType', heatmapScaleType); + if (isPromqlInput) { + setValue('heatmap.scaleType', heatmapScaleType); + } else { + setValue('series.0.heatmapScaleType', heatmapScaleType); + } } // Display settings live in a separate drawer form, so RHF can't track // them. Latch dirty state only when the drawer reports actual changes. @@ -1005,7 +1035,15 @@ export default function EditTimeChartForm({ } onSubmit(); }, - [setValue, onDirtyChange, onSubmit, configType, displayType, heatmapMode], + [ + setValue, + onDirtyChange, + onSubmit, + configType, + displayType, + heatmapMode, + isPromqlInput, + ], ); const tableConnection = useMemo( @@ -1159,6 +1197,8 @@ export default function EditTimeChartForm({ allowedSourceKinds={allowedSourceKinds} onSubmit={onSubmit} onOpenDisplaySettings={openDisplaySettings} + heatmapMode={heatmapMode} + onHeatmapModeChange={onHeatmapModeChange} /> ) : isRawSqlInput ? ( void; +}) { + return ( + { + if (value === 'distribution' || value === 'series') { + onModeChange(value); + } + }} + data={[ + { label: 'Distribution', value: 'distribution' }, + { label: 'Series', value: 'series' }, + ]} + data-testid="heatmap-mode-control" + /> + ); +} diff --git a/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx index dde2aab494..dd3439b1f6 100644 --- a/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx @@ -8,15 +8,7 @@ import { } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { HeatmapMode, TSource } from '@hyperdx/common-utils/dist/types'; -import { - Button, - Divider, - Flex, - SegmentedControl, - Stack, - Text, - Tooltip, -} from '@mantine/core'; +import { Button, Divider, Flex, Stack, Text, Tooltip } from '@mantine/core'; import { IconHelpCircle } from '@tabler/icons-react'; import { ChartEditorFormState } from '@/components/ChartEditor/types'; @@ -25,6 +17,7 @@ import SearchWhereInput from '@/components/SearchInput/SearchWhereInput'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { ChartSeriesEditor } from './ChartSeriesEditor'; +import { HeatmapModeControl } from './HeatmapModeControl'; type HeatmapSeriesEditorProps = { control: Control; @@ -70,21 +63,7 @@ export function HeatmapSeriesEditor({ return ( - { - if (value === 'distribution' || value === 'series') { - onModeChange(value); - } - }} - data={[ - { label: 'Distribution', value: 'distribution' }, - { label: 'Series', value: 'series' }, - ]} - data-testid="heatmap-mode-control" - /> + {mode === 'series' ? ( ; +type CalculatedQuery = Extract; type SeriesQuery = Extract; type DistributionOnlyProps = { @@ -90,6 +92,27 @@ function SeriesHeatmapChart({ ); } +function CalculatedHeatmapChart({ + query, + enabled, + ...viewProps +}: HeatmapChartViewProps & { query: CalculatedQuery; enabled: boolean }) { + const data = useCalculatedHeatmapData({ + config: query.config, + scaleType: query.scaleType, + enabled, + }); + + return ( + + ); +} + function DBHeatmapChart({ query, enabled = true, @@ -98,9 +121,17 @@ function DBHeatmapChart({ selectionBounds, ...viewProps }: DBHeatmapChartProps) { - return query.mode === 'series' ? ( - - ) : ( + if (query.mode === 'series') { + return ( + + ); + } + if (query.mode === 'calculated') { + return ( + + ); + } + return ( { const splits = logScaleSplits(Math.log(2), Math.log(40)); expect(splits.map(Math.exp).map(v => Math.round(v))).toEqual([3, 10, 30]); }); + + it('uses every mantissa within a range narrower than a decade', () => { + const splits = logScaleSplits(Math.log(0.3), Math.log(0.66)); + expect(splits.map(v => +Math.exp(v).toFixed(2))).toEqual([ + 0.3, 0.4, 0.5, 0.6, + ]); + }); + + it('spaces nice values evenly within a range narrower than a mantissa step', () => { + const splits = logScaleSplits(Math.log(0.3), Math.log(0.33)); + expect(splits.length).toBeGreaterThanOrEqual(3); + expect(splits.map(v => +Math.exp(v).toFixed(3))).toEqual([ + 0.3, 0.31, 0.32, 0.33, + ]); + }); }); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts new file mode 100644 index 0000000000..e507595f8c --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts @@ -0,0 +1,315 @@ +import { + CALCULATED_MAX_BUCKETS, + CALCULATED_MIN_BUCKETS, + gridFromSamples, +} from '@/components/DBHeatmapChart/heatmapCalculate'; +import { EMPTY_HEATMAP_GRID } from '@/components/DBHeatmapChart/heatmapGrid'; + +const STEP = 60_000; +const times = [0, STEP, 2 * STEP]; + +const series = (name: string, points: [number, number][]) => ({ + name, + points: points.map(([t, v]) => ({ t, v })), +}); + +const linearEdges = (values: number[]) => { + const { grid } = gridFromSamples({ + series: [ + series( + 'a', + values.map(v => [0, v]), + ), + ], + times, + scaleType: 'linear', + }); + return grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; +}; + +describe('gridFromSamples', () => { + it.each([ + [95, 10], + [9, 1], + [1, 0.1], + [0.45, 0.05], + [1000, 100], + [130, 10], + [150, 15], + ])('splits [0, %p] into about 10 linear buckets of %p', (max, bucketSize) => { + const edges = linearEdges([0, max]); + expect(edges[0]).toBe(0); + expect(edges[1]).toBe(bucketSize); + }); + + it('counts the samples of every series per time column and linear bucket', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [STEP, 50], + ]), + series('b', [ + [0, 5], + [STEP, 55], + [2 * STEP, 95], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis).toEqual({ + type: 'numeric', + scale: 'linear', + edges: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], + }); + expect(effectiveMin).toBe(0); + const rows = 10; + const column = (ti: number) => + grid.cells.slice(ti * rows, ti * rows + rows); + expect(column(0)).toEqual([2, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(column(1)).toEqual([0, 0, 0, 0, 0, 2, 0, 0, 0, 0]); + expect(column(2)).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + }); + + it('puts a value on a bucket edge in the bucket it starts', () => { + const { grid } = gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [0, 0.3], + [0, 0.9], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis.type === 'numeric' && grid.yAxis.edges).toEqual([ + 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, + ]); + expect(grid.cells.slice(0, 10)).toEqual([1, 0, 0, 1, 0, 0, 0, 0, 0, 1]); + }); + + it.each([ + ['linear', 100, CALCULATED_MIN_BUCKETS], + ['linear', 1600, 40], + ['linear', 100_000, CALCULATED_MAX_BUCKETS], + ['log', 100, CALCULATED_MIN_BUCKETS], + ['log', 1600, 40], + ['log', 100_000, CALCULATED_MAX_BUCKETS], + ] as const)( + 'splits %s samples into rows by the square root of the busiest column (%p samples → %p rows)', + (scaleType, samples, rows) => { + const { grid } = gridFromSamples({ + series: [ + series( + 'a', + Array.from({ length: samples }, (_, i): [number, number] => [ + 0, + 1 + (99 * i) / (samples - 1), + ]), + ), + series('b', [[STEP, 50]]), + ], + times, + scaleType, + }); + const edges = grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; + if (scaleType === 'log') { + expect(edges).toHaveLength(rows + 1); + } else { + expect(edges.length - 1).toBeGreaterThanOrEqual(rows - 1); + expect(edges.length - 1).toBeLessThanOrEqual(rows * 1.5 + 1); + } + }, + ); + + it('starts the buckets at a multiple of the bucket size', () => { + const edges = linearEdges([-12, 75]); + expect(edges[0]).toBe(-18); + expect(edges.at(-1)).toBe(81); + }); + + it('draws a constant value as one bucket sized by its magnitude', () => { + const { grid } = gridFromSamples({ + series: [series('a', [[0, 1]]), series('b', [[0, 1]])], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis).toEqual({ + type: 'numeric', + scale: 'linear', + edges: [1, 1.1], + }); + expect(grid.cells).toEqual([2, 0, 0]); + }); + + it('keeps the edges of buckets smaller than 1e-6', () => { + const edges = linearEdges([1e-7, 5e-7, 9e-7]); + expect(edges[0]).toBe(8.5e-8); + expect(edges[1]).toBe(1.7e-7); + expect(edges.at(-1)).toBe(9.35e-7); + expect(linearEdges([3e-6, 3e-6])).toEqual([3e-6, 3.3e-6]); + }); + + it('buckets a range wider than 1e17', () => { + const edges = linearEdges([0, 9.2e18]); + expect(edges[0]).toBe(0); + expect(edges[1]).toBe(1e18); + expect(edges.at(-1)).toBe(1e19); + }); + + it('buckets a range near the largest finite magnitude', () => { + const edges = linearEdges([0, 1e300]); + expect(edges[1]).toBe(1e299); + expect(edges).toHaveLength(12); + }); + + it('places samples between column starts in the column containing them', () => { + const { grid } = gridFromSamples({ + series: [series('a', [[STEP + 1000, 1]])], + times, + scaleType: 'linear', + }); + + expect(grid.cells).toEqual([0, 1, 0]); + }); + + it('skips non-finite values and samples outside the time range', () => { + const { grid } = gridFromSamples({ + series: [ + series('a', [ + [0, NaN], + [0, Infinity], + [10 * STEP, 5], + [0, 1], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.cells).toEqual([1, 0, 0]); + }); + + it('is empty without samples', () => { + expect( + gridFromSamples({ series: [], times, scaleType: 'linear' }).grid, + ).toBe(EMPTY_HEATMAP_GRID); + }); + + describe('log scale', () => { + it('spaces buckets geometrically from the effective min to the max', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 1], + [0, 10], + [STEP, 100], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(1); + const edges = grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; + expect(grid.yAxis.type === 'numeric' && grid.yAxis.scale).toBe('log'); + expect(edges).toHaveLength(CALCULATED_MIN_BUCKETS + 1); + expect(edges[0]).toBe(1); + expect(edges[CALCULATED_MIN_BUCKETS / 2]).toBeCloseTo(10); + expect(edges[CALCULATED_MIN_BUCKETS]).toBeCloseTo(100); + + const rows = CALCULATED_MIN_BUCKETS; + expect(grid.cells[0]).toBe(1); + expect(grid.cells[rows / 2]).toBe(1); + expect(grid.cells[rows + rows - 1]).toBe(1); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(3); + }); + + it('clamps small values into the bottom bucket and drops non-positive ones', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, -5], + [0, 0], + [0, 1e-9], + [0, 1000], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBeCloseTo(0.1); + expect(grid.cells[0]).toBe(1); + expect(grid.cells[CALCULATED_MIN_BUCKETS - 1]).toBe(1); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(2); + }); + + it('bounds the buckets by the positive samples only', () => { + const { effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + ...Array.from({ length: 50 }, (): [number, number] => [0, 0]), + [0, 10], + [0, 1000], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(10); + }); + + it('draws a constant value in the top bucket', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [series('a', [[0, 1]]), series('b', [[0, 1]])], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBeCloseTo(1e-4); + const edges = grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; + expect(edges.at(-1)).toBeCloseTo(1); + expect(grid.cells[CALCULATED_MIN_BUCKETS - 1]).toBe(2); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(2); + }); + + it('starts at the smallest sample when the low quantile reaches the max', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 0.5], + ...Array.from({ length: 100 }, (): [number, number] => [0, 1]), + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(0.5); + expect(grid.cells[0]).toBe(1); + expect(grid.cells[CALCULATED_MIN_BUCKETS - 1]).toBe(100); + }); + + it('is empty when there is no positive range to bucket', () => { + expect( + gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [0, -1], + ]), + ], + times, + scaleType: 'log', + }).grid, + ).toBe(EMPTY_HEATMAP_GRID); + }); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts index 42e0903e9e..753aec8778 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts @@ -76,8 +76,35 @@ describe('PromQL heatmaps', () => { legendTemplate: '{{job}}', }; - it('are always series heatmaps', () => { - expect(toHeatmapQuery(config)).toEqual({ mode: 'series', config }); + it('are calculated distribution heatmaps by default, on a linear scale', () => { + expect(toHeatmapQuery(config)).toEqual({ + mode: 'calculated', + config, + scaleType: 'linear', + }); + }); + + it('use the configured distribution scale', () => { + const logConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'distribution', scaleType: 'log' }, + }; + expect(toHeatmapQuery(logConfig)).toEqual({ + mode: 'calculated', + config: logConfig, + scaleType: 'log', + }); + }); + + it('are series heatmaps in series mode', () => { + const seriesModeConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'series' }, + }; + expect(toHeatmapQuery(seriesModeConfig)).toEqual({ + mode: 'series', + config: seriesModeConfig, + }); }); it('query at the heatmap granularity over a bucket-aligned range', () => { diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts new file mode 100644 index 0000000000..d5147ddbff --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts @@ -0,0 +1,123 @@ +import { + DisplayType, + PromqlConfigWithDateRange, +} from '@hyperdx/common-utils/dist/types'; +import { renderHook } from '@testing-library/react'; + +import { useCalculatedHeatmapData } from '@/components/DBHeatmapChart/useCalculatedHeatmapData'; +import { useSeriesHeatmapData } from '@/components/DBHeatmapChart/useSeriesHeatmapData'; + +const mockUseQueriedChartConfig = jest.fn(); +jest.mock('@/hooks/useChartConfig', () => ({ + useQueriedChartConfig: (...args: unknown[]) => + mockUseQueriedChartConfig(...args), +})); + +const HOUR = 60 * 60 * 1000; +const T0 = new Date('2026-07-06T00:00:00Z').getTime(); +const dateRange: [Date, Date] = [new Date(T0), new Date(T0 + 2 * HOUR)]; + +const config: PromqlConfigWithDateRange = { + configType: 'promql', + displayType: DisplayType.Heatmap, + promqlExpression: [{ expression: 'up' }], + connection: 'test-connection', + dateRange, + granularity: 'auto', +}; + +const meta = [ + { name: '__hdx_time_bucket', type: 'DateTime64(3)' }, + { name: 'value', type: 'Float64' }, + { name: 'series_name', type: 'String' }, +]; + +function mockResponse(data: Record[]) { + mockUseQueriedChartConfig.mockReturnValue({ + data: { meta, data }, + isLoading: false, + isPlaceholderData: false, + error: null, + }); +} + +const row = (t: number, value: number, series_name: string) => ({ + __hdx_time_bucket: new Date(t).toISOString(), + value, + series_name, +}); + +describe('useCalculatedHeatmapData', () => { + beforeEach(() => { + mockUseQueriedChartConfig.mockReset(); + }); + + it('counts the series samples in each time column and value bucket', () => { + mockResponse([row(T0, 0, 'a'), row(T0, 5, 'b'), row(T0 + HOUR, 95, 'a')]); + const { result } = renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + + const { grid, generatedTsBuckets } = result.current.view; + const t1 = generatedTsBuckets.map(d => d.getTime()).indexOf(T0 + HOUR); + expect(grid.yAxis).toMatchObject({ type: 'numeric', scale: 'linear' }); + const rows = 10; + expect(grid.cells.slice(0, rows)).toEqual([2, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(grid.cells.slice(t1 * rows, t1 * rows + rows)).toEqual([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + ]); + }); + + it('counts every series', () => { + mockResponse( + Array.from({ length: 60 }, (_, i) => row(T0, 1, `series-${i}`)), + ); + const { result } = renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + + expect(result.current.view.grid.cells[0]).toBe(60); + expect(result.current.view.hiddenSeriesCount).toBe(0); + }); + + it('re-buckets a scale change without changing the query', () => { + mockResponse([row(T0, 1, 'a'), row(T0, 100, 'b')]); + const { result, rerender } = renderHook( + ({ scaleType }: { scaleType: 'linear' | 'log' }) => + useCalculatedHeatmapData({ + config: { ...config, heatmap: { scaleType } }, + scaleType, + enabled: true, + }), + { initialProps: { scaleType: 'linear' } }, + ); + const [linearQuery] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + rerender({ scaleType: 'log' }); + + const [logQuery] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + expect(logQuery).toEqual(linearQuery); + expect(result.current.view.grid.yAxis).toMatchObject({ + type: 'numeric', + scale: 'log', + }); + }); + + it('shares its query with a series heatmap of the same expression', () => { + mockResponse([]); + renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + const [, calculatedOptions] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + renderHook(() => + useSeriesHeatmapData({ + config: { ...config, heatmap: { mode: 'series' } }, + enabled: true, + }), + ); + const [, seriesOptions] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + expect(seriesOptions.queryKey).toEqual(calculatedOptions.queryKey); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts index d9cd40f319..dd60245a84 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts @@ -41,38 +41,49 @@ export function formatHeatmapTick( }).format(actualValue); } +const MIN_LOG_SPLITS = 3; + +/** Split mantissas within a power of 10, sparsest first. */ +const LOG_SPLIT_MANTISSAS = [ + [1], // 1, 10, 100... + [1, 3], // 1, 3, 10, 30, 100... + [1, 2, 5], // 1, 2, 5, 10, 20, 50, 100... + [1, 2, 3, 5], + [1, 2, 3, 4, 5, 6, 7, 8, 9], +]; + /** - * Log-scale y-axis splits at powers of 10 (0.01, 0.1, 1, 10, 100…) so labels - * are clean round numbers instead of arbitrary positions in log space. - * `yMin`/`yMax` and the returned splits are natural-log values. + * Log-scale y-axis splits at round numbers: powers of 10 (0.01, 0.1, 1, 10…) + * when the range spans enough decades, otherwise denser mantissas within each + * decade, and evenly spaced nice values when the range is narrower than one + * mantissa step. `yMin`/`yMax` and the returned splits are natural-log values. */ export function logScaleSplits(yMin: number, yMax: number): number[] { const realMin = Math.exp(yMin); const realMax = Math.exp(yMax); - const splits: number[] = []; - // Generate powers of 10 within range const startExp = Math.floor(Math.log10(realMin)); const endExp = Math.ceil(Math.log10(realMax)); - for (let e = startExp; e <= endExp; e++) { - const v = Math.pow(10, e); - const logV = Math.log(v); - if (logV >= yMin && logV <= yMax) { - splits.push(logV); - } - } - // If too few splits, add intermediate values (×3) - if (splits.length < 3) { + + // Try increasingly dense sets of mantissas until the minimum + // number of log splits (axis ticks) is reached. + for (const mantissas of LOG_SPLIT_MANTISSAS) { + const splits: number[] = []; for (let e = startExp; e <= endExp; e++) { - for (const mult of [1, 3]) { - const v = mult * Math.pow(10, e); - const logV = Math.log(v); - if (logV >= yMin && logV <= yMax) { - splits.push(logV); - } + for (const mult of mantissas) { + const logV = Math.log(mult * Math.pow(10, e)); + if (logV >= yMin && logV <= yMax) splits.push(logV); } } - // Deduplicate and sort - return [...new Set(splits)].sort((a, b) => a - b); + if (splits.length >= MIN_LOG_SPLITS) return splits; + } + + // Fallback to linear splits when the log-scale splits are insufficient. + const maxIncr = (realMax - realMin) / MIN_LOG_SPLITS; + const mag = Math.pow(10, Math.floor(Math.log10(maxIncr))); + const incr = [5, 2, 1].map(m => m * mag).find(i => i <= maxIncr) ?? mag; + const splits: number[] = []; + for (let k = Math.ceil(realMin / incr); k * incr <= realMax; k++) { + splits.push(Math.log(k * incr)); } return splits; } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts b/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts new file mode 100644 index 0000000000..1575838252 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts @@ -0,0 +1,194 @@ +import { clamp, range, round } from 'lodash'; + +import { computeEffectiveMin, heatmapLowQuantile } from './heatmapBounds'; +import { + EMPTY_HEATMAP_GRID, + HeatmapGrid, + HeatmapScaleType, +} from './heatmapGrid'; +import { HeatmapSeries, makeTimeIndexer } from './heatmapSeriesGrid'; + +export const CALCULATED_MIN_BUCKETS = 10; +export const CALCULATED_MAX_BUCKETS = 100; + +/** Decimal places required to represent multiples of the given nice y step */ +function yStepDecimals(yStep: number) { + return Math.max(0, 1 - Math.floor(Math.log10(yStep))); +} + +const ALL_MULTS = [ + 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5, 9, 9.5, +]; + +/** + * ...0.01, 0.015, 0.02, ..., 0.95, 1, 2, 3, ..., 9, 10, 15, 20, ... + * + * Covers the full range of numbers representable in JavaScript floats. + */ +const NICE_Y_STEPS = range(-16, 308).flatMap(exp => + ALL_MULTS.map(m => round(m * 10 ** exp, yStepDecimals(m * 10 ** exp))).filter( + yStep => exp < 0 || Number.isInteger(yStep), + ), +); + +/** The largest nice y step that splits `[min, max]` into at least `count` buckets. */ +function niceYStep(min: number, max: number, count: number) { + const approx = (max - min) / Math.max(count - 1, 1); + const idx = NICE_Y_STEPS.findIndex(yStep => yStep > approx); + if (idx === -1) return NICE_Y_STEPS[NICE_Y_STEPS.length - 1]; + return NICE_Y_STEPS[Math.max(idx - 1, 0)]; +} + +/** Index of the `yStep`-tall row holding `value`. */ +function yStepIndex(value: number, yStep: number) { + // The epsilon absorbs float error like 0.3 / 0.1 = 2.9999999999999996. + return Math.floor(value / yStep + 1e-9); +} + +/** + * Number of rows to split the values into. Fewer samples get fewer + * rows while many samples keep their shape. + */ +function targetBuckets(maxSamplesPerColumn: number) { + return clamp( + Math.round(Math.sqrt(maxSamplesPerColumn)), + CALCULATED_MIN_BUCKETS, + CALCULATED_MAX_BUCKETS, + ); +} + +/** + * Linear buckets: at least `n` rows of a nice y step, aligned to multiples of + * the step so edges are round numbers. + */ +function linearBuckets(min: number, max: number, n: number) { + // A constant value has no range to split, so size its single row by the + // value's magnitude instead of the smallest step, which would be invisible. + const yStep = + max > min + ? niceYStep(min, max, n) + : niceYStep(0, Math.abs(min) || 1, CALCULATED_MIN_BUCKETS); + const decimals = yStepDecimals(yStep); + const minIdx = yStepIndex(min, yStep); + const rows = yStepIndex(max, yStep) - minIdx + 1; + const edges = Array.from({ length: rows + 1 }, (_, k) => + round((minIdx + k) * yStep, decimals), + ); + + return { edges, rowOf: (v: number) => yStepIndex(v, yStep) - minIdx }; +} + +/** + * Log buckets over positive samples, bounded like a builder distribution + * heatmap's: `n` geometric rows from `effectiveMin` to the max, with smaller + * values in the bottom row. + */ +function logBuckets(values: Float64Array, n: number) { + const q = heatmapLowQuantile('log'); + const min = values[Math.floor(q * (values.length - 1))]; + const max = values[values.length - 1]; + let effectiveMin = computeEffectiveMin(min, max, 'log'); + if (!(max > effectiveMin)) { + // The low quantile reaches the max when nearly every sample has the same + // value; fall back to the smallest sample, then the range cap. + effectiveMin = computeEffectiveMin( + values[0] < max ? values[0] : 0, + max, + 'log', + ); + } + if (!(max > effectiveMin)) return undefined; + + const logRange = Math.log(max / effectiveMin); + const edges = Array.from( + { length: n + 1 }, + (_, k) => effectiveMin * Math.pow(max / effectiveMin, k / n), + ); + + return { + edges, + rowOf: (v: number) => + Math.floor((n * Math.log(v / effectiveMin)) / logRange), + }; +} + +/** + * A distribution grid of every series' samples: each cell counts the samples + * in a time column whose values fall in a row's bucket. + */ +export function gridFromSamples({ + series, + times, + scaleType, +}: { + series: HeatmapSeries[]; + times: number[]; + scaleType: HeatmapScaleType; +}): { grid: HeatmapGrid; effectiveMin: number } { + const stepMs = times.length > 1 ? times[1] - times[0] : 0; + const timeIndex = makeTimeIndexer(times, stepMs); + + // Two passes over the points, one for the bounds and one to count, so no + // per-sample copy is held between them. + const forEachSample = (fn: (ti: number, v: number) => void) => { + for (const s of series) { + for (const { t, v } of s.points) { + const ti = timeIndex(t); + // Non-positive values have no log, so log buckets drop them + if (ti < 0 || !Number.isFinite(v) || (scaleType === 'log' && v <= 0)) { + continue; + } + fn(ti, v); + } + } + }; + + // Log bounds need a quantile, so they keep every value + const values = + scaleType === 'log' + ? new Float64Array(series.reduce((n, s) => n + s.points.length, 0)) + : undefined; + const columnCounts = new Uint32Array(times.length); + let count = 0; + let maxSamplesPerColumn = 0; + let min = Infinity; + let max = -Infinity; + forEachSample((ti, v) => { + if (values != null) values[count] = v; + count++; + maxSamplesPerColumn = Math.max(maxSamplesPerColumn, ++columnCounts[ti]); + min = Math.min(min, v); + max = Math.max(max, v); + }); + if (count === 0) { + return { grid: EMPTY_HEATMAP_GRID, effectiveMin: 0 }; + } + + const n = targetBuckets(maxSamplesPerColumn); + const buckets = + values != null + ? logBuckets(values.subarray(0, count).sort(), n) + : linearBuckets(min, max, n); + if (buckets == null) { + return { grid: EMPTY_HEATMAP_GRID, effectiveMin: 0 }; + } + + const rows = buckets.edges.length - 1; + const cells = new Array(times.length * rows).fill(0); + forEachSample((ti, v) => { + // Log values below the effective min fall in the bottom row, and the max + // computes to one past the top row + const r = Math.min(rows - 1, Math.max(0, buckets.rowOf(v))); + cells[ti * rows + r]++; + }); + + return { + grid: { + times, + stepMs, + yAxis: { type: 'numeric', scale: scaleType, edges: buckets.edges }, + cells, + }, + effectiveMin: buckets.edges[0], + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts index 1a7a114e39..fef15c961b 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts @@ -46,9 +46,13 @@ export type HeatmapSeriesChartConfig = | BuilderChartConfigWithDateRange | PromqlConfigWithDateRange; +/** Linear, since PromQL samples often include zeros, which log scale drops. */ +export const PROMQL_DEFAULT_HEATMAP_SCALE_TYPE: HeatmapScaleType = 'linear'; + /** * What a heatmap queries, by mode. Distribution heatmaps bucket a value - * expression server-side; series heatmaps query one builder series or PromQL + * expression server-side; calculated heatmaps bucket the samples of a PromQL + * range query client-side; series heatmaps query one builder series or PromQL * expression per time bucket and draw a row per series. */ export type HeatmapQuery = @@ -57,6 +61,11 @@ export type HeatmapQuery = config: HeatmapChartConfig; scaleType: HeatmapScaleType; } + | { + mode: 'calculated'; + config: PromqlConfigWithDateRange; + scaleType: HeatmapScaleType; + } | { mode: 'series'; config: HeatmapSeriesChartConfig; @@ -84,12 +93,19 @@ export function resolveHeatmapGranularity({ ); } -/** PromQL heatmaps only support series mode. */ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { - if (isPromqlChartConfig(config) || getHeatmapMode(config) === 'series') { + if (getHeatmapMode(config) === 'series') { return { mode: 'series', config }; } + if (isPromqlChartConfig(config)) { + return { + mode: 'calculated', + config, + scaleType: config.heatmap?.scaleType ?? PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, + }; + } + const firstSelect = Array.isArray(config.select) ? config.select[0] : undefined; @@ -113,9 +129,9 @@ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { } /** - * The time-chart query behind a series-mode heatmap, bucketed at the heatmap's - * granularity: `select[0]` and the group by for a builder config, or the first - * expression's range query for PromQL. + * The time-chart query behind a series-mode or calculated heatmap, bucketed at + * the heatmap's granularity: `select[0]` and the group by for a builder + * config, or the first expression's range query for PromQL. */ export function buildHeatmapSeriesConfig( config: HeatmapSeriesChartConfig, @@ -129,6 +145,9 @@ export function buildHeatmapSeriesConfig( granularity, // Heatmaps have no series limit control, a default is applied automatically seriesLimit: undefined, + // Heatmap settings only change how the response is drawn, so keep them + // out of the query's cache key: switching mode or scale reuses the data. + heatmap: undefined, }); } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts index 5ddc233325..88775ca3b4 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts @@ -10,7 +10,7 @@ const naturalCompare = new Intl.Collator(undefined, { numeric: true }).compare; * normally match `times` exactly; the arithmetic fallback covers timestamps * that land inside a column instead of on its start. */ -function makeTimeIndexer(times: number[], stepMs: number) { +export function makeTimeIndexer(times: number[], stepMs: number) { const indexByTime = new Map(times.map((t, i) => [t, i])); return (t: number) => { const exact = indexByTime.get(t); diff --git a/packages/app/src/components/DBHeatmapChart/index.ts b/packages/app/src/components/DBHeatmapChart/index.ts index f0b63103d7..112380d1b0 100644 --- a/packages/app/src/components/DBHeatmapChart/index.ts +++ b/packages/app/src/components/DBHeatmapChart/index.ts @@ -8,6 +8,7 @@ export { buildHeatmapBoundsConfig, buildHeatmapBucketConfig, buildHeatmapSeriesConfig, + PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, resolveHeatmapGranularity, toHeatmapQuery, } from './heatmapQueries'; diff --git a/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts new file mode 100644 index 0000000000..3867f42bf2 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts @@ -0,0 +1,53 @@ +import { useMemo } from 'react'; +import { PromqlConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; + +import { gridFromSamples } from './heatmapCalculate'; +import { HeatmapScaleType } from './heatmapGrid'; +import { HeatmapData, HeatmapView, useSettledView } from './useHeatmapView'; +import { useHeatmapSeriesPoints } from './useSeriesHeatmapData'; + +/** + * Query a calculated (PromQL distribution) heatmap: the same range query as a + * series heatmap, with every series' samples bucketed client-side into a + * value distribution per time column. + */ +export function useCalculatedHeatmapData({ + config, + scaleType, + enabled, +}: { + config: PromqlConfigWithDateRange; + scaleType: HeatmapScaleType; + enabled: boolean; +}): HeatmapData { + const { series, generatedTsBuckets, isLoading, isPlaceholderData, error } = + useHeatmapSeriesPoints({ + config, + enabled, + // Every series counts towards the distribution, so none are dropped. This is a + // client-side limit, so capping it wouldn't affect how much data is in memory. + maxSeries: Number.POSITIVE_INFINITY, + }); + + const currentView = useMemo(() => { + const { grid, effectiveMin } = gridFromSamples({ + series, + times: generatedTsBuckets.map(d => d.getTime()), + scaleType, + }); + return { + grid, + generatedTsBuckets, + scaleType, + effectiveMin, + hiddenSeriesCount: 0, + }; + }, [series, generatedTsBuckets, scaleType]); + + return { + view: useSettledView(currentView, isPlaceholderData), + isLoading, + isRefreshing: isPlaceholderData, + error, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts index f418f88d26..fdc676bdb3 100644 --- a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts +++ b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts @@ -20,18 +20,19 @@ const HEATMAP_MAX_SERIES = 50; /** * Group time-chart rows into one point list per series, keeping the - * `HEATMAP_MAX_SERIES` series with the largest peaks. + * `maxSeries` series with the largest peaks. */ function responseToHeatmapSeries( response: ResponseJSON>, dateRange: [Date, Date], + maxSeries: number, ): { series: HeatmapSeries[]; hiddenSeriesCount: number } { const { graphResults, timestampColumn, lineData, hiddenSeriesCount } = formatResponseForTimeChart({ currentPeriodResponse: response, dateRange, generateEmptyBuckets: false, - maxSeries: HEATMAP_MAX_SERIES, + maxSeries, }); const series = lineData.map(line => ({ @@ -50,17 +51,19 @@ function responseToHeatmapSeries( } /** - * Query a series-mode heatmap. Queried like a time chart, formatting the - * response into a heatmap grid with series on the y axis, time on the - * x-axis, and value determining the cell value. + * Query a heatmap's series like a time chart, bucketed at the heatmap + * granularity, as one point list per series. Keeps the `maxSeries` series + * with the largest peaks. */ -export function useSeriesHeatmapData({ +export function useHeatmapSeriesPoints({ config, enabled, + maxSeries, }: { config: HeatmapSeriesChartConfig; enabled: boolean; -}): HeatmapData { + maxSeries: number; +}) { const { granularity, generatedTsBuckets, fromMs, toMs } = useHeatmapTimeBuckets(config); @@ -83,7 +86,11 @@ export function useSeriesHeatmapData({ }>(() => { if (data == null) return { series: [], hiddenSeriesCount: 0 }; try { - return responseToHeatmapSeries(data, [new Date(fromMs), new Date(toMs)]); + return responseToHeatmapSeries( + data, + [new Date(fromMs), new Date(toMs)], + maxSeries, + ); } catch (e) { return { series: [], @@ -91,26 +98,61 @@ export function useSeriesHeatmapData({ error: e instanceof Error ? e : new Error(String(e)), }; } - }, [data, fromMs, toMs]); + }, [data, fromMs, toMs, maxSeries]); + + return { + series: parsed.series, + hiddenSeriesCount: parsed.hiddenSeriesCount, + generatedTsBuckets, + isLoading, + isPlaceholderData, + error: error ?? parsed.error, + }; +} + +/** + * Query a series-mode heatmap. Queried like a time chart, formatting the + * response into a heatmap grid with series on the y axis, time on the + * x-axis, and value determining the cell value. + */ +export function useSeriesHeatmapData({ + config, + enabled, +}: { + config: HeatmapSeriesChartConfig; + enabled: boolean; +}): HeatmapData { + const { + series, + hiddenSeriesCount, + generatedTsBuckets, + isLoading, + isPlaceholderData, + error, + } = useHeatmapSeriesPoints({ + config, + enabled, + maxSeries: HEATMAP_MAX_SERIES, + }); const currentView = useMemo( () => ({ grid: gridFromSeries( - parsed.series, + series, generatedTsBuckets.map(d => d.getTime()), ), generatedTsBuckets, scaleType: 'linear', effectiveMin: 0, - hiddenSeriesCount: parsed.hiddenSeriesCount, + hiddenSeriesCount, }), - [parsed, generatedTsBuckets], + [series, hiddenSeriesCount, generatedTsBuckets], ); return { view: useSettledView(currentView, isPlaceholderData), isLoading, isRefreshing: isPlaceholderData, - error: error ?? parsed.error, + error, }; } diff --git a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx index 6bcd400664..6a6b263768 100644 --- a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx +++ b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx @@ -76,6 +76,30 @@ describe('ChartDisplaySettingsDrawer', () => { heatmapScaleType: 'linear', }); }); + + it.each(['linear', 'log'] as const)( + 'selects the %s default scale when the tile has none', + async scale => { + const onChange = jest.fn(); + const user = userEvent.setup(); + renderWithMantine( + , + ); + + await user.click(screen.getByRole('button', { name: /apply/i })); + + expect(onChange.mock.calls[0][0]).toMatchObject({ + heatmapScaleType: scale, + }); + }, + ); }); describe('color picker section', () => { @@ -754,6 +778,26 @@ describe('ChartDisplaySettingsDrawer', () => { ).not.toBeInTheDocument(); }); + it.each([ + ['hidden', 'distribution', false], + ['offered', 'series', true], + ] as const)( + 'is %s on a PromQL %s heatmap', + (_label, heatmapMode, isShown) => { + renderWithMantine( + , + ); + + expect(screen.queryByTestId('legend-template-input') != null).toBe( + isShown, + ); + }, + ); + it('blocks Apply when the template exceeds the persisted length cap', async () => { const onChange = jest.fn(); const onClose = jest.fn(); diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index 9fd6b5ad8b..2c9e53e02d 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -13,6 +13,7 @@ import { import { switchWhereToLucene } from '../utils/lucene-autocomplete'; import { addTagViaPicker, removeTagViaPicker } from '../utils/tags'; +import { HeatmapComponent } from './HeatmapComponent'; import { WebhookAlertModalComponent } from './WebhookAlertModalComponent'; import { WhereInputComponent } from './WhereInputComponent'; @@ -73,7 +74,7 @@ export class ChartEditorComponent { await this.chartTypeInput.getByRole('tab', { name: tabName }).click(); } - /** Pick the builder heatmap's mode in its segmented control. */ + /** Pick the heatmap's mode in its segmented control. */ async setHeatmapMode(mode: 'Distribution' | 'Series') { await this.editorForm() .getByTestId('heatmap-mode-control') @@ -99,6 +100,11 @@ export class ChartEditorComponent { .locator('.cm-content'); } + /** The heatmap the editor previews. */ + get previewHeatmap(): HeatmapComponent { + return new HeatmapComponent(this.page, this.editorForm()); + } + /** A heatmap mode's (visually hidden) radio, for checking the selection. */ heatmapModeOption(mode: 'Distribution' | 'Series'): Locator { return this.editorForm() diff --git a/packages/app/tests/e2e/components/HeatmapComponent.ts b/packages/app/tests/e2e/components/HeatmapComponent.ts index 6d38251270..6eac1f3e36 100644 --- a/packages/app/tests/e2e/components/HeatmapComponent.ts +++ b/packages/app/tests/e2e/components/HeatmapComponent.ts @@ -77,6 +77,15 @@ export class HeatmapComponent { return ((await row.textContent()) ?? '').replace(/^Series:\s*/, '').trim(); } + /** A labelled value of the open cell tooltip, e.g. `Count Value`. */ + async hoveredTooltipValue(label: 'Count Value' | 'Y Value') { + const row = this.container + .getByText(`${label}:`, { exact: true }) + .locator('..'); + await expect(row).toBeVisible(); + return ((await row.textContent()) ?? '').replace(`${label}:`, '').trim(); + } + /** * Hover the top series-axis label, which the canvas draws, so its DOM * tooltip opens. Returns that tooltip. diff --git a/packages/app/tests/e2e/features/promql-heatmap.spec.ts b/packages/app/tests/e2e/features/promql-heatmap.spec.ts index 9d2db47213..365e3ef6ae 100644 --- a/packages/app/tests/e2e/features/promql-heatmap.spec.ts +++ b/packages/app/tests/e2e/features/promql-heatmap.spec.ts @@ -1,6 +1,7 @@ /** - * PromQL heatmaps are series heatmaps: one row per series of a range query, - * named by the legend template. There is no distribution mode to pick. + * PromQL heatmaps draw either a distribution of every series' samples, + * bucketed client-side, or one row per series of a range query, named by the + * legend template. * * The seed gives `e2e_service_up` one series per `SERVICES` entry, labelled * `service`, one sample of 1 a minute. @@ -16,6 +17,34 @@ import { E2E_PROMQL_METRIC_NAME, PROMQL_SOURCE_NAME } from '../utils/constants'; // Ends in `)` rather than inside the selector so the autocomplete popup closes. const EXPRESSION = `last_over_time(${E2E_PROMQL_METRIC_NAME}{service!=""}[$__interval])`; +/** Open a new tile on a new dashboard as a PromQL heatmap of `EXPRESSION`. */ +async function startPromqlHeatmapTile(dashboardPage: DashboardPage) { + const editor = dashboardPage.chartEditor; + await dashboardPage.goto(); + await dashboardPage.createNewDashboard(); + await dashboardPage.addTile(); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setChartType(DisplayType.Heatmap); + await editor.switchToPromqlMode(); + await editor.selectSource(PROMQL_SOURCE_NAME); + await editor.setChartName('PromQL heatmap tile'); + await editor.replacePromqlExpression(EXPRESSION); +} + +/** Save the tile, and check it renders before and after a reload. */ +async function saveAndAssertRenders( + dashboardPage: DashboardPage, + assertRenders: (heatmap: HeatmapComponent) => Promise, +) { + await dashboardPage.chartEditor.save(); + await expect(dashboardPage.getTiles()).toHaveCount(1, { timeout: 10000 }); + await expect(dashboardPage.getTileError()).toHaveCount(0); + await assertRenders(dashboardPage.getTileHeatmap()); + await dashboardPage.reload(); + await assertRenders(dashboardPage.getTileHeatmap()); +} + test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { test('draws a row per series, and persists', async ({ page }) => { test.setTimeout(120000); @@ -35,22 +64,13 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }; await test.step('Create a PromQL heatmap tile', async () => { - await dashboardPage.goto(); - await dashboardPage.createNewDashboard(); - await dashboardPage.addTile(); - await expect(editor.nameInput).toBeVisible(); - await editor.waitForDataToLoad(); - await editor.setChartType(DisplayType.Heatmap); - await editor.switchToPromqlMode(); - await editor.selectSource(PROMQL_SOURCE_NAME); - await editor.setChartName('PromQL heatmap tile'); - await editor.replacePromqlExpression(EXPRESSION); + await startPromqlHeatmapTile(dashboardPage); + await editor.setHeatmapMode('Series'); await editor.setLegendTemplate('{{service}}'); await editor.runQuery(false); }); - await test.step('The editor offers no heatmap mode or instant query', async () => { - await expect(page.getByTestId('heatmap-mode-control')).toHaveCount(0); + await test.step('The editor offers no instant query', async () => { await expect(page.getByTestId('promql-query-type-control-0')).toHaveCount( 0, ); @@ -58,9 +78,7 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }); await test.step('The preview draws a row per series', async () => { - await assertRendersServices( - new HeatmapComponent(page, page.getByTestId('tile-editor-form')), - ); + await assertRendersServices(editor.previewHeatmap); await editor.openGeneratedPromql(); await expect .poll(() => editor.getGeneratedPromqlText()) @@ -68,14 +86,61 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }); await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersServices); + }); + }); + + test('draws a distribution of the samples, and persists', async ({ + page, + }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + // Every service reports 1, so a populated cell counts at most one sample + // per service, in a row around 1. + const assertRendersDistribution = async (heatmap: HeatmapComponent) => { + await expect(heatmap.canvas.first()).toBeVisible(); + await expect(heatmap.notEnoughDataText).toHaveCount(0); + await heatmap.hoverPopulatedCell(); + const count = Number(await heatmap.hoveredTooltipValue('Count Value')); + expect(count).toBeGreaterThan(0); + expect(count).toBeLessThanOrEqual(SERVICES.length); + expect(await heatmap.hoveredTooltipValue('Y Value')).toMatch( + /^1(\.\d+)?\b/, + ); + }; + + await test.step('Create a PromQL heatmap tile', async () => { + await startPromqlHeatmapTile(dashboardPage); + await editor.runQuery(false); + }); + + await test.step('Distribution is the default mode, on a linear scale', async () => { + await expect(editor.heatmapModeOption('Distribution')).toBeChecked(); + await editor.openDisplaySettings(); + await expect(editor.heatmapScaleOption('Linear')).toBeChecked(); + await editor.applyDisplaySettings(); + }); + + await test.step('The preview draws the distribution', async () => { + await assertRendersDistribution(editor.previewHeatmap); + }); + + await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersDistribution); + }); + + await test.step('The chosen scale persists', async () => { + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setHeatmapScale('Log'); await editor.save(); - await expect(dashboardPage.getTiles()).toHaveCount(1, { - timeout: 10000, - }); - await expect(dashboardPage.getTileError()).toHaveCount(0); - await assertRendersServices(dashboardPage.getTileHeatmap()); await dashboardPage.reload(); - await assertRendersServices(dashboardPage.getTileHeatmap()); + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await editor.openDisplaySettings(); + await expect(editor.heatmapScaleOption('Log')).toBeChecked(); }); }); }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 51c8a272e0..07c779cd55 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -397,6 +397,8 @@ export const ColorConditionSchema = z.discriminatedUnion('operator', [ export type ColorCondition = z.infer; +export const HeatmapScaleTypeSchema = z.enum(['log', 'linear']); + export const DerivedColumnSchema = z.intersection( RootValueExpressionSchema, z.object({ @@ -406,7 +408,7 @@ export const DerivedColumnSchema = z.intersection( metricNameSql: z.string().optional(), // Heatmap-specific fields (optional, only used when displayType is Heatmap) countExpression: z.string().optional(), - heatmapScaleType: z.enum(['log', 'linear']).optional(), + heatmapScaleType: HeatmapScaleTypeSchema.optional(), numberFormat: NumberFormatSchema.optional(), // Per-column palette-token color, applied by the renderer to this // column's cells on table tiles only (gated in the series editor by @@ -1706,9 +1708,13 @@ export type RatioMode = z.infer; export const HeatmapModeSchema = z.enum(['distribution', 'series']); export type HeatmapMode = z.infer; +export const HeatmapSettingsSchema = z.object({ + mode: HeatmapModeSchema.optional(), +}); + export const _ChartConfigSchema = SharedChartSettingsSchema.extend({ // Heatmap tiles only; an absent mode is 'distribution' (see getHeatmapMode). - heatmap: z.object({ mode: HeatmapModeSchema.optional() }).optional(), + heatmap: HeatmapSettingsSchema.optional(), timestampValueExpression: z.string(), implicitColumnExpression: z.string().optional(), // Fallback expression for bare-text Lucene search when no implicit column is @@ -1905,6 +1911,9 @@ const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ source: z.string().optional(), step: z.string().optional(), legendTemplate: z.string().max(MAX_LEGEND_TEMPLATE_LENGTH).optional(), + heatmap: HeatmapSettingsSchema.extend({ + scaleType: HeatmapScaleTypeSchema.optional(), + }).optional(), }); /** Schema describing PromQL chart configs with runtime-only fields */