diff --git a/.changeset/promql-histogram-heatmap.md b/.changeset/promql-histogram-heatmap.md new file mode 100644 index 0000000000..a86ac25ecf --- /dev/null +++ b/.changeset/promql-histogram-heatmap.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +feat: Support histogram heatmaps from PromQL bucket metrics diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index d739f6d71c..6aa957e671 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -8,9 +8,9 @@ import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { ChartConfigWithDateRange, DisplayType, - HeatmapMode, MAX_LEGEND_TEMPLATE_LENGTH, NumberFormat, + PromqlHeatmapMode, } from '@hyperdx/common-utils/dist/types'; import { Alert, @@ -89,8 +89,8 @@ interface ChartDisplaySettingsDrawerProps { configType?: 'sql' | 'builder' | 'promql'; /** Whether a PromQL tile's queried expression runs over a range. */ promqlUsesRange?: boolean; - /** A builder heatmap's mode; only distribution heatmaps have a numeric y axis to scale. */ - heatmapMode?: HeatmapMode; + /** A heatmap's mode; only distribution heatmaps have a numeric y axis to scale. */ + heatmapMode?: PromqlHeatmapMode; /** The heatmap scale used when the tile has none set. */ defaultHeatmapScaleType?: HeatmapScaleType; previousDateRange?: [Date, Date]; @@ -227,13 +227,14 @@ export default function ChartDisplaySettingsDrawer({ displayType === DisplayType.Heatmap && heatmapMode === 'distribution'; // Every PromQL display that surfaces a series name. A number tile shows one - // value, a table gives each label its own column, and a distribution heatmap - // merges every series into one, so none has a legend. + // value, a table gives each label its own column, a distribution heatmap + // merges every series into one, and a histogram heatmap labels rows by + // bucket, so none has a legend. const showLegendTemplate = configType === 'promql' && displayType !== DisplayType.Number && displayType !== DisplayType.Table && - !isDistributionHeatmap; + !(displayType === DisplayType.Heatmap && heatmapMode !== 'series'); // On pie/bar builder charts, seriesLimit becomes a plain SQL LIMIT on the // number of slices/bars; on PromQL it trims the reduced series client-side diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 58204d60e4..5dac49f90a 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -12,8 +12,8 @@ import { import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { DisplayType, - HeatmapMode, MAX_PROMQL_EXPRESSIONS, + PromqlHeatmapMode, SourceKind, } from '@hyperdx/common-utils/dist/types'; import { Box, Button, Divider, Flex, Group, Text } from '@mantine/core'; @@ -26,6 +26,7 @@ import { useSource } from '@/source'; import PromqlExpressionEditor from './PromqlExpressionEditor'; import { ChartEditorFormState } from './types'; +import { promqlExpressionPlaceholder } from './utils'; export default function PromqlChartEditor({ control, @@ -41,8 +42,8 @@ export default function PromqlChartEditor({ allowedSourceKinds: SourceKind[]; onSubmit: (suppressErrorNotification?: boolean) => void; onOpenDisplaySettings: () => void; - heatmapMode: HeatmapMode; - onHeatmapModeChange: (mode: HeatmapMode) => void; + heatmapMode: PromqlHeatmapMode; + onHeatmapModeChange: (mode: PromqlHeatmapMode) => void; }) { const { fields: expressions, @@ -118,6 +119,7 @@ export default function PromqlChartEditor({ )} @@ -128,6 +130,7 @@ export default function PromqlChartEditor({ index={index} length={expressions.length} metricNames={metricNames} + placeholder={promqlExpressionPlaceholder(displayType, heatmapMode)} isIgnored={!displayTypeSupportsMultiExpression && index > 0} isInstantQuerySupported={displayTypeSupportsInstantQuery({ displayType, diff --git a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx index c2bea4b95d..61c5d707cd 100644 --- a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx @@ -12,6 +12,7 @@ export default function PromqlExpressionEditor({ index, length, metricNames, + placeholder, isIgnored, isInstantQuerySupported, isReducerSupported, @@ -26,6 +27,7 @@ export default function PromqlExpressionEditor({ index: number; length: number; metricNames: string[] | undefined; + placeholder: string; /** Whether the display type leaves this expression unqueried. */ isIgnored: boolean; /** Whether the display type lets this expression choose between instant and range queries. */ @@ -62,7 +64,7 @@ export default function PromqlExpressionEditor({ value={expressionField.value ?? ''} onChange={expressionField.onChange} onSubmit={() => onSubmit()} - placeholder="rate(http_requests_total{service='api'}[5m])" + placeholder={placeholder} metricNames={metricNames} enableMacros /> diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index f625da3785..a45eb39922 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -19,6 +19,7 @@ import { convertFormStateToSavedChartConfig, convertSavedChartConfigToFormState, getAllowedSourceKinds, + promqlExpressionPlaceholder, validateChartForm, } from '@/components/ChartEditor/utils'; @@ -688,6 +689,21 @@ describe('PromQL expressions', () => { }); }); + it('saves a histogram heatmap', () => { + expect( + convertFormStateToSavedChartConfig( + { + ...promqlForm( + [{ expression: 'sum by (le) (x_bucket)' }], + DisplayType.Heatmap, + ), + heatmap: { mode: 'histogram' }, + }, + undefined, + ), + ).toMatchObject({ heatmap: { mode: 'histogram' } }); + }); + it('queries a heatmap with its heatmap settings', () => { const result = convertFormStateToChartConfig( { @@ -1929,6 +1945,22 @@ describe('validateChartForm', () => { expect.objectContaining({ path: 'series.0.valueExpression' }), ); }); + + it('requires a value expression on a builder heatmap left in histogram mode', () => { + const errors = validateChartForm( + makeForm({ + displayType: DisplayType.Heatmap, + source: 'source-trace', + heatmap: { mode: 'histogram' }, + series: [{ ...seriesItem, valueExpression: '' }], + }), + traceSource, + jest.fn(), + ); + expect(errors).toContainEqual( + expect.objectContaining({ path: 'series.0.valueExpression' }), + ); + }); }); describe('color round-trip (sql/promql Number tile)', () => { @@ -2082,6 +2114,41 @@ describe('heatmap round-trip', () => { }); }); +describe('builder heatmap settings', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Heatmap, + source: 'source-trace', + where: '', + series: [{ ...seriesItem, valueExpression: 'Duration' }], + }; + + it('fall back to distribution mode from PromQL-only histogram mode', () => { + const histogramForm: ChartEditorFormState = { + ...form, + heatmap: { mode: 'histogram', scaleType: 'log' }, + }; + expect( + convertFormStateToSavedChartConfig(histogramForm, traceSource), + ).toHaveProperty('heatmap', { mode: undefined }); + expect( + convertFormStateToChartConfig( + histogramForm, + [new Date(0), new Date(1000)], + traceSource, + ), + ).toHaveProperty('heatmap', { mode: undefined }); + }); + + it('keep series mode, without PromQL-only settings', () => { + expect( + convertFormStateToSavedChartConfig( + { ...form, heatmap: { mode: 'series', scaleType: 'log' } }, + traceSource, + ), + ).toHaveProperty('heatmap', { mode: 'series' }); + }); +}); + describe('metric formulas (HDX-5080)', () => { const metricSeriesItem = { aggFn: 'avg' as const, @@ -2489,6 +2556,16 @@ describe('getAllowedSourceKinds', () => { ).toEqual([SourceKind.Trace]); }); + it('treats a leftover histogram mode on a builder heatmap as distribution', () => { + expect( + getAllowedSourceKinds({ + configType: 'builder', + displayType: DisplayType.Heatmap, + heatmapMode: 'histogram', + }), + ).toEqual([SourceKind.Trace]); + }); + it('offers only PromQL sources on PromQL heatmap tiles', () => { expect( getAllowedSourceKinds({ @@ -2499,3 +2576,21 @@ describe('getAllowedSourceKinds', () => { ).toEqual([SourceKind.Promql]); }); }); + +describe('promqlExpressionPlaceholder', () => { + it.each([ + ['distribution', /_sum\[5m\]\) \/ rate\(.*_count/], + ['series', /^sum by \(service\)/], + ['histogram', /^sum by \(le\) \(rate\(.*_bucket/], + ] as const)('suggests a %s heatmap expression', (mode, pattern) => { + expect(promqlExpressionPlaceholder(DisplayType.Heatmap, mode)).toMatch( + pattern, + ); + }); + + it('suggests the same expression on every non-heatmap chart', () => { + expect(promqlExpressionPlaceholder(DisplayType.Line, 'histogram')).toBe( + promqlExpressionPlaceholder(DisplayType.Number, 'distribution'), + ); + }); +}); diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts index d1c23ceb65..e8e11f61ed 100644 --- a/packages/app/src/components/ChartEditor/types.ts +++ b/packages/app/src/components/ChartEditor/types.ts @@ -28,9 +28,13 @@ export type SavedChartConfigWithSelectArray = Omit< * `promqlExpressions` is the PromQL counterpart, holding the array form of * `promqlExpression` (which a saved tile may carry as a bare string). **/ -export type ChartEditorFormState = Partial & +export type ChartEditorFormState = Partial< + Omit +> & Partial> & - Partial> & { + Partial> & { + // The PromQL settings are a superset of the builder's. + heatmap?: PromqlSavedChartConfig['heatmap']; alert?: BuilderSavedChartConfig['alert'] & { id?: string; createdBy?: AlertWithCreatedBy['createdBy']; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 2c2e4784a4..4885b61306 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -32,6 +32,7 @@ import { isRangeThresholdType, isTraceSource, PromqlChartConfig, + PromqlHeatmapMode, PromqlSavedChartConfig, PromqlSeries, RawSqlChartConfig, @@ -228,13 +229,13 @@ export function getAllowedSourceKinds({ }: { configType: ChartEditorFormState['configType']; displayType: DisplayType | undefined; - heatmapMode?: HeatmapMode; + heatmapMode?: PromqlHeatmapMode; }): SourceKind[] { if (configType === 'promql' && isPromqlDisplayType(displayType)) { return [SourceKind.Promql]; } if (displayType === DisplayType.Heatmap) { - return [...getHeatmapSourceKinds(heatmapMode)]; + return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; } if (isStringSelectDisplayType(displayType)) { return ROW_LISTING_SOURCE_KINDS; @@ -324,6 +325,7 @@ export function convertFormStateToSavedChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), select: [], where: form.where ?? '', source: source?.id ?? form.source ?? '', @@ -342,6 +344,7 @@ export function convertFormStateToSavedChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), select: isStringSelectDisplayType(form.displayType) ? typeof form.select === 'string' ? form.select @@ -446,6 +449,7 @@ export function convertFormStateToChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), from: source.from, timestampValueExpression: source.timestampValueExpression, dateRange, @@ -554,6 +558,42 @@ function formPromqlExpressions(form: ChartEditorFormState): PromqlSeries[] { })); } +/** The form's heatmap settings, formatted for a builder tile */ +function formBuilderHeatmap( + form: ChartEditorFormState, +): BuilderSavedChartConfig['heatmap'] { + return form.heatmap == null + ? undefined + : { mode: toBuilderHeatmapMode(form.heatmap.mode) }; +} + +/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ +export function toBuilderHeatmapMode( + mode: PromqlHeatmapMode | undefined, +): HeatmapMode | undefined { + return mode === 'histogram' ? undefined : mode; +} + +const DEFAULT_PROMQL_PLACEHOLDER = + "rate(http_requests_total{service='api'}[5m])"; + +const PROMQL_HEATMAP_PLACEHOLDERS: Record = { + distribution: + 'rate(http_request_duration_seconds_sum[5m]) / rate(http_request_duration_seconds_count[5m])', + series: 'sum by (service) (rate(http_requests_total[5m]))', + histogram: 'sum by (le) (rate(http_request_duration_seconds_bucket[5m]))', +}; + +/** An example PromQL expression suited to the display type and heatmap mode. */ +export function promqlExpressionPlaceholder( + displayType: DisplayType | undefined, + heatmapMode: PromqlHeatmapMode, +): string { + return displayType === DisplayType.Heatmap + ? PROMQL_HEATMAP_PLACEHOLDERS[heatmapMode] + : DEFAULT_PROMQL_PLACEHOLDER; +} + /** Heatmap settings are dropped from PromQL tiles that are not heatmaps. */ function formPromqlHeatmap( form: ChartEditorFormState, @@ -762,7 +802,7 @@ export const validateChartForm = ( !isRawSqlChart && !isPromqlChart && form.displayType === DisplayType.Heatmap && - getHeatmapMode(form) === 'distribution' && + getHeatmapMode({ heatmap: formBuilderHeatmap(form) }) === 'distribution' && Array.isArray(form.series) && form.series.length > 0 && !form.series[0]?.valueExpression diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx index eedb420f8b..c51000f5cc 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx @@ -106,8 +106,8 @@ function HeatmapSQLPreview({ }; const query = toHeatmapQuery(configWithTimestamp); - // Only PromQL heatmaps are calculated, and they have no SQL to preview - if (query.mode === 'calculated') { + // Only PromQL heatmaps are calculated or histograms, and they have no SQL + if (query.mode === 'calculated' || query.mode === 'histogram') { return null; } diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 9ccab64d5d..3523e0205e 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -26,7 +26,7 @@ import { DashboardFilter, DisplayType, Filter, - HeatmapMode, + PromqlHeatmapMode, SavedChartConfig, SourceKind, TSource, @@ -73,6 +73,7 @@ import { isPromqlDisplayType, isRawSqlDisplayType, isStringSelectDisplayType, + toBuilderHeatmapMode, validateChartForm, } from '@/components/ChartEditor/utils'; import { @@ -175,7 +176,7 @@ function applyHeatmapDefaults( fallbackValueExpression = '', isPromql = false, }: { - mode: HeatmapMode; + mode: PromqlHeatmapMode; tableSource: TSource | undefined; fallbackValueExpression?: string; isPromql?: boolean; @@ -315,7 +316,7 @@ export default function EditTimeChartForm({ const configType = useWatch({ control, name: 'configType' }); const connection = useWatch({ control, name: 'connection' }); const promqlExpressions = useWatch({ control, name: 'promqlExpressions' }); - const heatmapMode = getHeatmapMode({ + const storedHeatmapMode = getHeatmapMode({ heatmap: useWatch({ control, name: 'heatmap' }), }); @@ -324,6 +325,13 @@ export default function EditTimeChartForm({ configType === 'sql' && isRawSqlDisplayType(displayType); const isPromqlInput = configType === 'promql' && isPromqlDisplayType(displayType); + // Switching from PromQL to builder leaves the PromQL-only histogram mode in + // the form, which the builder treats as distribution. + const builderHeatmapMode = + toBuilderHeatmapMode(storedHeatmapMode) ?? 'distribution'; + const heatmapMode: PromqlHeatmapMode = isPromqlInput + ? storedHeatmapMode + : builderHeatmapMode; const { data: sources } = useSources(); @@ -870,7 +878,7 @@ export default function EditTimeChartForm({ // Switching heatmap mode may trigger a source swap, and should apply new defaults. const onHeatmapModeChange = useCallback( - (mode: HeatmapMode) => { + (mode: PromqlHeatmapMode) => { setValue('heatmap.mode', mode); applyHeatmapDefaults(setValue, getValues, { mode, @@ -1247,7 +1255,7 @@ export default function EditTimeChartForm({ chartConfigForExplanations={chartConfigForExplanations} onSubmit={onSubmit} openDisplaySettings={openDisplaySettings} - heatmapMode={heatmapMode} + heatmapMode={builderHeatmapMode} onHeatmapModeChange={onHeatmapModeChange} /> )} diff --git a/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx b/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx index 13ef68a86d..713fdd0708 100644 --- a/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx @@ -1,28 +1,85 @@ -import { HeatmapMode } from '@hyperdx/common-utils/dist/types'; -import { SegmentedControl } from '@mantine/core'; +import { PromqlHeatmapMode } from '@hyperdx/common-utils/dist/types'; +import { Group, SegmentedControl, Stack, Text, Tooltip } from '@mantine/core'; +import { IconHelpCircle } from '@tabler/icons-react'; -export function HeatmapModeControl({ +type ModeOption = { label: string; value: PromqlHeatmapMode; help: string }; + +const BUILDER_MODE_OPTIONS: ModeOption[] = [ + { + label: 'Distribution', + value: 'distribution', + help: 'Buckets each event by a value, such as duration. Color shows how many events fall in each bucket.', + }, + { + label: 'Series', + value: 'series', + help: 'One row per group. Color shows the aggregated value.', + }, +]; + +const PROMQL_MODE_OPTIONS: ModeOption[] = [ + { + label: 'Distribution', + value: 'distribution', + help: 'Buckets the samples of every series by value. Color shows how many samples fall in each bucket.', + }, + { + label: 'Series', + value: 'series', + help: 'One row per series. Color shows the sample value.', + }, + { + label: 'Histogram', + value: 'histogram', + help: "One row per le bucket of a Prometheus histogram. Color shows the query's value for that bucket alone, such as a rate or count.", + }, +]; + +/** Histogram mode is offered only where `allowHistogram` is set (PromQL). */ +export function HeatmapModeControl({ mode, onModeChange, + allowHistogram = false, }: { - mode: HeatmapMode; - onModeChange: (mode: HeatmapMode) => void; + mode: M; + onModeChange: (mode: M) => void; + allowHistogram?: boolean; }) { + const options = allowHistogram ? PROMQL_MODE_OPTIONS : BUILDER_MODE_OPTIONS; + const isOfferedMode = (value: string): value is M => + options.some(o => o.value === value); return ( - { - if (value === 'distribution' || value === 'series') { - onModeChange(value); + + { + if (isOfferedMode(value)) onModeChange(value); + }} + data={options.map(({ label, value }) => ({ label, value }))} + data-testid="heatmap-mode-control" + /> + + {options.map(({ label, help }) => ( + + {label}: {help} + + ))} + } - }} - data={[ - { label: 'Distribution', value: 'distribution' }, - { label: 'Series', value: 'series' }, - ]} - data-testid="heatmap-mode-control" - /> + > + + + ); } diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/HeatmapModeControl.test.tsx b/packages/app/src/components/DBEditTimeChartForm/__tests__/HeatmapModeControl.test.tsx new file mode 100644 index 0000000000..1db5983ee7 --- /dev/null +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/HeatmapModeControl.test.tsx @@ -0,0 +1,49 @@ +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import { HeatmapModeControl } from '@/components/DBEditTimeChartForm/HeatmapModeControl'; + +describe('HeatmapModeControl', () => { + it('offers no histogram mode by default', () => { + renderWithMantine( + , + ); + expect(screen.getByRole('radio', { name: 'Series' })).toBeInTheDocument(); + expect(screen.queryByRole('radio', { name: 'Histogram' })).toBeNull(); + }); + + it('offers and selects histogram mode when allowed', async () => { + const onModeChange = jest.fn(); + renderWithMantine( + , + ); + await userEvent.click(screen.getByText('Histogram')); + expect(onModeChange).toHaveBeenCalledWith('histogram'); + }); + + it.each([ + [false, ['Distribution', 'Series']], + [true, ['Distribution', 'Series', 'Histogram']], + ])( + 'explains each offered mode on hover (allowHistogram: %s)', + async (allowHistogram, modes) => { + renderWithMantine( + , + ); + await userEvent.hover(screen.getByTestId('heatmap-mode-help')); + const tooltip = await screen.findByRole('tooltip'); + for (const mode of modes) { + expect(tooltip).toHaveTextContent(`${mode}:`); + } + expect(tooltip.textContent?.includes('Histogram:')).toBe(allowHistogram); + }, + ); +}); diff --git a/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx b/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx index 636f1993ba..1a95856dd9 100644 --- a/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx +++ b/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx @@ -5,10 +5,12 @@ import type { HeatmapQuery } from './heatmapQueries'; import type { SelectionBounds } from './selection'; import { useCalculatedHeatmapData } from './useCalculatedHeatmapData'; import { useHeatmapData } from './useHeatmapData'; +import { useHistogramHeatmapData } from './useHistogramHeatmapData'; import { useSeriesHeatmapData } from './useSeriesHeatmapData'; type DistributionQuery = Extract; type CalculatedQuery = Extract; +type HistogramQuery = Extract; type SeriesQuery = Extract; type DistributionOnlyProps = { @@ -113,6 +115,23 @@ function CalculatedHeatmapChart({ ); } +function HistogramHeatmapChart({ + query, + enabled, + ...viewProps +}: HeatmapChartViewProps & { query: HistogramQuery; enabled: boolean }) { + const data = useHistogramHeatmapData({ config: query.config, enabled }); + + return ( + + ); +} + function DBHeatmapChart({ query, enabled = true, @@ -131,6 +150,11 @@ function DBHeatmapChart({ ); } + if (query.mode === 'histogram') { + return ( + + ); + } return ( (numberFormat ? JSON.stringify(numberFormat) : ''), - [numberFormat], - ); + // chart and wiping u.select. + const stableNumberFormat = useStableValue(numberFormat); const tickFormatter = useCallback( - (value: number) => formatHeatmapTick(value, scaleType, numberFormat), - // eslint-disable-next-line react-hooks/exhaustive-deps - [numberFormatKey, scaleType], + (value: number) => formatHeatmapTick(value, scaleType, stableNumberFormat), + [stableNumberFormat, scaleType], + ); + const boundFormatter = useCallback( + (bound: number) => formatBucketBound(bound, stableNumberFormat), + [stableNumberFormat], ); const plotData = useMemo(() => gridToPlotData(grid), [grid]); const rowCount = heatmapRowCount(grid.yAxis); const cellKind = heatmapCellKind(grid.yAxis); - // Key on label content: every refresh builds a fresh labels array, and new - // options make uplot-react recreate the chart instead of updating its data. - const seriesLabelsKey = - grid.yAxis.type === 'series' - ? JSON.stringify(grid.yAxis.labels) - : undefined; - const seriesLabels = useMemo( - () => - seriesLabelsKey === undefined ? undefined : JSON.parse(seriesLabelsKey), - [seriesLabelsKey], + // Fresh grids carry fresh arrays, and new options make uplot-react recreate + // the chart instead of updating its data. + const seriesLabels = useStableValue( + grid.yAxis.type === 'series' ? grid.yAxis.labels : undefined, + ); + const bucketBounds = useStableValue( + grid.yAxis.type === 'buckets' ? grid.yAxis.bounds : undefined, ); const bucketPercentiles = useMemo( () => - grid.yAxis.type === 'numeric' ? computeBucketPercentiles(grid) : null, + grid.yAxis.type === 'series' ? null : computeBucketPercentiles(grid), [grid], ); const options: uPlot.Options = useMemo(() => { const opt = baseHeatmapOptions; const themedSeries = buildSeriesForPalette(palette, cellKind); - const yAxis = heatmapYAxisOptions(seriesLabels, scaleType, tickFormatter); + const yAxis = heatmapYAxisOptions({ + seriesLabels, + bucketBounds, + scaleType, + tickFormatter, + boundFormatter, + }); return { ...opt, ...(yAxis.scale != null @@ -224,6 +222,9 @@ export function HeatmapPlot({ }, }, plugins: [ + // False positive: onPointHighlight reads mouseInsideRef on cursor + // moves, not during render. + // eslint-disable-next-line react-hooks/refs highlightDataPlugin({ margin: 20, cellKind, @@ -287,6 +288,8 @@ export function HeatmapPlot({ palette, hasFilter, seriesLabels, + bucketBounds, + boundFormatter, cellKind, ]); @@ -308,11 +311,21 @@ export function HeatmapPlot({ numberFormat, ), } - : { - kind: 'distribution', - formattedY: tickFormatter(highlightedPoint.yVal), - percentile: bucketPercentiles?.get(highlightedRow), - }; + : bucketBounds != null + ? { + kind: 'bucket', + formattedRange: formatBucketRange( + bucketBounds, + highlightedRow, + boundFormatter, + ), + percentile: bucketPercentiles?.get(highlightedRow), + } + : { + kind: 'distribution', + formattedY: tickFormatter(highlightedPoint.yVal), + percentile: bucketPercentiles?.get(highlightedRow), + }; return (
- Y Value: {cell.formattedY} + {cell.kind === 'bucket' ? ( + <> + Bucket: {cell.formattedRange} + + ) : ( + <> + Y Value: {cell.formattedY} + + )} {cell.percentile != null && ` (p${new Intl.NumberFormat('en-US', { maximumFractionDigits: 1, }).format(cell.percentile)})`}
- Count Value:{' '} - {new Intl.NumberFormat('en-US', { - notation: 'standard', - compactDisplay: 'short', - }).format(point.countVal)} + Count Value: {COUNT_FORMAT.format(point.countVal)}
)} diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/HeatmapTooltip.test.tsx b/packages/app/src/components/DBHeatmapChart/__tests__/HeatmapTooltip.test.tsx index 09475a287a..28dbe42f9a 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/HeatmapTooltip.test.tsx +++ b/packages/app/src/components/DBHeatmapChart/__tests__/HeatmapTooltip.test.tsx @@ -41,6 +41,12 @@ describe('HeatmapTooltip', () => { expect(screen.getByText('1,234')).toBeInTheDocument(); }); + it('keeps significant digits of small counts', () => { + renderTooltip({ point: { ...point, countVal: 0.000278 } }); + + expect(screen.getByText('0.000278')).toBeInTheDocument(); + }); + it('appends the percentile when known', () => { renderTooltip({ cell: { kind: 'distribution', formattedY: 'y=2', percentile: 95.25 }, diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts index 2752400b05..9589f9786e 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts @@ -1,4 +1,7 @@ import { + bucketRowSplits, + formatBucketBound, + formatBucketRange, formatHeatmapTick, logScaleSplits, } from '@/components/DBHeatmapChart/heatmapAxis'; @@ -48,3 +51,63 @@ describe('logScaleSplits', () => { ]); }); }); + +describe('formatBucketBound', () => { + it('labels the open-ended bucket +Inf', () => { + expect(formatBucketBound(Infinity, undefined)).toBe('+Inf'); + }); + + it('keeps the digits that tell adjacent bounds apart', () => { + expect(formatBucketBound(0.0025, undefined)).toBe('0.0025'); + expect(formatBucketBound(2.5, undefined)).toBe('2.5'); + expect(formatBucketBound(2.5, { output: 'number', mantissa: 1 })).toBe( + '2.5', + ); + }); + + it('formats durations', () => { + expect(formatBucketBound(250, { output: 'number', unit: 'ms' })).toBe( + '250ms', + ); + }); +}); + +describe('formatBucketRange', () => { + const format = (bound: number) => formatBucketBound(bound, undefined); + + it('leaves the lowest bucket unbounded below', () => { + expect(formatBucketRange([0.1, 1, Infinity], 0, format)).toBe('≤ 0.1'); + expect(formatBucketRange([-1, 1], 0, format)).toBe('≤ -1'); + }); + + it('spans from the next bound down', () => { + expect(formatBucketRange([0.1, 1, Infinity], 1, format)).toBe('0.1 – 1'); + }); + + it('reads the +Inf bucket as everything above the last finite bound', () => { + expect(formatBucketRange([0.1, 1, Infinity], 2, format)).toBe('> 1'); + }); + + it('reads a lone +Inf bucket as every observation', () => { + expect(formatBucketRange([Infinity], 0, format)).toBe('≤ +Inf'); + }); +}); + +describe('bucketRowSplits', () => { + it('ticks the top of every row when they fit', () => { + expect(bucketRowSplits(4, 200)).toEqual([1, 2, 3, 4]); + }); + + it('thins ticks to evenly spaced rows that fit the height, keeping the top row', () => { + const splits = bucketRowSplits(100, 200); + expect(splits.length).toBeLessThan(100); + expect(splits.at(-1)).toBe(100); + const gaps = new Set(splits.slice(1).map((s, i) => s - splits[i])); + expect(gaps.size).toBe(1); + expect(bucketRowSplits(100, 400).length).toBeGreaterThan(splits.length); + }); + + it('keeps the top tick in a very short plot', () => { + expect(bucketRowSplits(5, 0)).toEqual([5]); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts index f3ca07687d..715f9c9680 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts @@ -150,15 +150,10 @@ describe('rowPlotBounds', () => { expect(hi).toEqual([Math.log(10), Math.log(1000)]); }); - it('draws an open top row as tall as the row below it', () => { - const { lo, hi } = rowPlotBounds({ - type: 'numeric', - scale: 'linear', - edges: [0, 1, 3, Infinity], - openTop: true, - }); - expect(lo).toEqual([0, 1, 3]); - expect(hi).toEqual([1, 3, 5]); + it('gives each histogram bucket a unit-height band, whatever its bound', () => { + expect( + rowPlotBounds({ type: 'buckets', bounds: [0.1, 10, Infinity] }), + ).toEqual({ lo: [0, 1, 2], hi: [1, 2, 3] }); }); it('gives each series row a unit-height band', () => { diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts new file mode 100644 index 0000000000..691418a1b9 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts @@ -0,0 +1,114 @@ +import { gridFromHistogramSamples } from '@/components/DBHeatmapChart/heatmapHistogram'; + +const times = [1000, 2000]; + +describe('gridFromHistogramSamples', () => { + it('sorts buckets by numeric bound, +Inf on top', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '+Inf', t: 1000, v: 10 }, + { le: '10', t: 1000, v: 8 }, + { le: '2.5', t: 1000, v: 3 }, + ], + times, + }); + expect(grid.yAxis).toEqual({ + type: 'buckets', + bounds: [2.5, 10, Infinity], + }); + }); + + it('de-accumulates each bucket from the one below it', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '0.1', t: 1000, v: 2 }, + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '0.1', t: 2000, v: 0 }, + { le: '1', t: 2000, v: 4 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.cells).toEqual([2, 3, 1, 0, 4, 0]); + }); + + it('merges buckets whose le is written differently', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '1.0', t: 2000, v: 4 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [1, Infinity] }); + expect(grid.cells).toEqual([5, 1, 4, 0]); + }); + + it('sums series at the same time whose le is written differently', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 2 }, + { le: '1.0', t: 1000, v: 3 }, + { le: '+Inf', t: 1000, v: 10 }, + ], + times: [1000], + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [1, Infinity] }); + expect(grid.cells).toEqual([5, 5]); + }); + + it('de-accumulates across a missing bucket sample from the next one down', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '0.1', t: 1000, v: 2 }, + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '0.1', t: 2000, v: 1 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.cells.slice(3)).toEqual([1, 0, 3]); + }); + + it('clamps a bucket below the one under it to 0', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 4.9 }, + ], + times, + }); + expect(grid.cells.slice(0, 2)).toEqual([5, 0]); + }); + + it('does not let a dipped bucket inflate the ones above it', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '0.1', t: 1000, v: 5 }, + { le: '1', t: 1000, v: 4.9 }, + { le: '+Inf', t: 1000, v: 8 }, + ], + times: [1000], + }); + expect(grid.cells).toEqual([5, 0, 3]); + }); + + it('draws a single bucket', () => { + const grid = gridFromHistogramSamples({ + samples: [{ le: '+Inf', t: 2000, v: 7 }], + times, + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [Infinity] }); + expect(grid.cells).toEqual([0, 7]); + }); + + it('is empty without samples', () => { + const grid = gridFromHistogramSamples({ samples: [], times }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [] }); + expect(grid.cells).toEqual([]); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts index 753aec8778..a2e676221f 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts @@ -107,6 +107,30 @@ describe('PromQL heatmaps', () => { }); }); + it('are histogram heatmaps in histogram mode', () => { + const histogramConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'histogram' }, + }; + expect(toHeatmapQuery(histogramConfig)).toEqual({ + mode: 'histogram', + config: histogramConfig, + }); + }); + + it('query a histogram heatmap as a histogram, with the expression as written', () => { + const histogramConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'histogram', scaleType: 'log' }, + }; + expect(buildHeatmapSeriesConfig(histogramConfig, '5 minute')).toMatchObject( + { + promqlExpression: config.promqlExpression, + heatmap: { mode: 'histogram' }, + }, + ); + }); + it('query at the heatmap granularity over a bucket-aligned range', () => { expect(buildHeatmapSeriesConfig(config, '5 minute')).toMatchObject({ configType: 'promql', diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts new file mode 100644 index 0000000000..51994a60a1 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts @@ -0,0 +1,76 @@ +import { + DisplayType, + PromqlConfigWithDateRange, +} from '@hyperdx/common-utils/dist/types'; +import { renderHook } from '@testing-library/react'; + +import { useHistogramHeatmapData } from '@/components/DBHeatmapChart/useHistogramHeatmapData'; + +const mockUseQueriedChartConfig = jest.fn(); +jest.mock('@/hooks/useChartConfig', () => ({ + useQueriedChartConfig: (...args: unknown[]) => + mockUseQueriedChartConfig(...args), +})); + +const HOUR = 60 * 60 * 1000; +const T0 = new Date('2026-07-06T00:00:00Z').getTime(); + +const config: PromqlConfigWithDateRange = { + configType: 'promql', + displayType: DisplayType.Heatmap, + promqlExpression: [{ expression: 'sum by (le) (rate(x_bucket[5m]))' }], + connection: 'test-connection', + dateRange: [new Date(T0), new Date(T0 + 2 * HOUR)], + granularity: '1 hour', + heatmap: { mode: 'histogram' }, +}; + +const row = (t: number, le: string, value: number) => ({ + __hdx_time_bucket: new Date(t).toISOString(), + le, + value, +}); + +describe('useHistogramHeatmapData', () => { + beforeEach(() => { + mockUseQueriedChartConfig.mockReset(); + }); + + it('draws a row per bucket from the queried rows', () => { + mockUseQueriedChartConfig.mockReturnValue({ + data: { + data: [ + row(T0, '0.5', 2), + row(T0, '+Inf', 3), + row(T0 + HOUR, '0.5', 1), + row(T0 + HOUR, '+Inf', 5), + ], + }, + isLoading: false, + isPlaceholderData: false, + error: null, + }); + const { result } = renderHook(() => + useHistogramHeatmapData({ config, enabled: true }), + ); + + const { grid } = result.current.view; + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [0.5, Infinity] }); + expect(grid.times.slice(0, 2)).toEqual([T0, T0 + HOUR]); + expect(grid.cells.slice(0, 4)).toEqual([2, 1, 1, 4]); + }); + + it('passes the query error through', () => { + const error = new Error('Several series share le="0.5"'); + mockUseQueriedChartConfig.mockReturnValue({ + data: undefined, + isLoading: false, + isPlaceholderData: false, + error, + }); + const { result } = renderHook(() => + useHistogramHeatmapData({ config, enabled: true }), + ); + expect(result.current.error).toBe(error); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts index dd60245a84..a8736496ca 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts @@ -11,6 +11,11 @@ export const HEATMAP_DURATION_NUMBER_FORMAT = { factor: 0.001, } as const satisfies NumberFormat; +const COMPACT_NUMBER_FORMAT = new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', +}); + /** Format a plot-space y value as a y-axis / tooltip label. */ export function formatHeatmapTick( value: number, @@ -35,10 +40,7 @@ export function formatHeatmapTick( average: true, mantissa: Math.abs(actualValue) >= 1 ? 0 : 2, }) - : new Intl.NumberFormat('en-US', { - notation: 'compact', - compactDisplay: 'short', - }).format(actualValue); + : COMPACT_NUMBER_FORMAT.format(actualValue); } const MIN_LOG_SPLITS = 3; @@ -88,6 +90,62 @@ export function logScaleSplits(yMin: number, yMax: number): number[] { return splits; } +/** + * Format a histogram bucket bound as an axis / tooltip label. Unlike + * `formatHeatmapTick`, this keeps fractional digits above 1, since adjacent + * bounds like 2.5 and 5 must not round to the same label. + */ +export function formatBucketBound( + bound: number, + numberFormat: NumberFormat | undefined, +) { + if (bound === Infinity) return '+Inf'; + if (numberFormat?.unit === 'ms' || numberFormat?.output === 'duration') { + return formatHeatmapTick(bound, 'linear', numberFormat); + } + return numberFormat + ? formatNumber(bound, { ...numberFormat, average: true }) + : COMPACT_NUMBER_FORMAT.format(bound); +} + +/** + * A bucket row's range: from the next bound down to its own, or everything + * above the last finite bound. The lowest bucket has no lower bound, since a + * histogram can record observations below its first bound, including negative + * ones. + */ +export function formatBucketRange( + bounds: number[], + row: number, + format: (bound: number) => string, +) { + const hi = bounds[row]; + if (row === 0) return `≤ ${format(hi)}`; + const lo = bounds[row - 1]; + return hi === Infinity ? `> ${format(lo)}` : `${format(lo)} – ${format(hi)}`; +} + +const MIN_BUCKET_TICK_SPACING_PX = 20; + +/** + * Bucket-axis ticks at the top edge of rows, labeled with their bound. Thinned + * to fit the plot height, counting down from the top row so it always keeps a + * tick. + */ +export function bucketRowSplits( + rowCount: number, + plotHeightPx: number, +): number[] { + const maxTicks = Math.max( + 1, + Math.floor(plotHeightPx / MIN_BUCKET_TICK_SPACING_PX), + ); + const stride = Math.ceil(rowCount / maxTicks); + const splits: number[] = []; + for (let top = rowCount; top >= 1; top -= stride) splits.push(top); + return splits.reverse(); +} + const SERIES_LABEL_MAX_LENGTH = 24; /** Series-axis ticks, one centered on each row. */ @@ -104,14 +162,38 @@ function formatSeriesTick(labels: string[], value: number) { /** * uPlot y scale and y axis overrides for a heatmap's y axis. A series axis is - * pinned to its rows with one label centered on each; a numeric axis formats - * its ticks, placed at powers of 10 on a log scale. + * pinned to its rows with one label centered on each; a bucket axis is pinned + * to its rows with each row's bound labeling its top edge; a numeric axis + * formats its ticks, placed at powers of 10 on a log scale. */ -export function heatmapYAxisOptions( - seriesLabels: string[] | undefined, - scaleType: HeatmapScaleType, - tickFormatter: (value: number) => string, -): { scale?: uPlot.Scale; axis: Pick } { +export function heatmapYAxisOptions({ + seriesLabels, + bucketBounds, + scaleType, + tickFormatter, + boundFormatter, +}: { + seriesLabels: string[] | undefined; + bucketBounds: number[] | undefined; + scaleType: HeatmapScaleType; + tickFormatter: (value: number) => string; + boundFormatter: (bound: number) => string; +}): { scale?: uPlot.Scale; axis: Pick } { + if (bucketBounds != null) { + return { + scale: { auto: false, range: [0, Math.max(1, bucketBounds.length)] }, + axis: { + values: (_u: uPlot, vals: number[]) => + vals.map(v => boundFormatter(bucketBounds[v - 1])), + splits: (u: uPlot) => + bucketRowSplits( + bucketBounds.length, + u.bbox.height / (window.devicePixelRatio || 1), + ), + }, + }; + } + if (seriesLabels != null) { return { scale: { auto: false, range: [0, Math.max(1, seriesLabels.length)] }, diff --git a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts index 81ec3f139b..c1079471bd 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts @@ -8,12 +8,12 @@ export type HeatmapYAxis = scale: HeatmapScaleType; /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ edges: number[]; - /** - * The top row has no upper bound (e.g. a `+Inf` histogram bucket), so - * its last edge is ignored and it is drawn as tall as the row below. - */ - openTop?: boolean; } + /** + * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per + * row; the top bound may be `Infinity` (a `+Inf` bucket). + */ + | { type: 'buckets'; bounds: number[] } | { type: 'series'; labels: string[] }; /** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ @@ -28,8 +28,8 @@ export type HeatmapGrid = { }; /** - * Numeric-axis cells are counts, empty at 0. Series-axis cells are values - * that may be zero or negative, empty when NaN. + * Numeric- and bucket-axis cells are counts, empty at 0. Series-axis cells + * are values that may be zero or negative, empty when NaN. */ export type HeatmapCellKind = 'count' | 'value'; @@ -49,22 +49,27 @@ export const EMPTY_HEATMAP_GRID: HeatmapGrid = { }; export function heatmapRowCount(yAxis: HeatmapYAxis) { - return yAxis.type === 'series' - ? yAxis.labels.length - : Math.max(0, yAxis.edges.length - 1); + switch (yAxis.type) { + case 'series': + return yAxis.labels.length; + case 'buckets': + return yAxis.bounds.length; + case 'numeric': + return Math.max(0, yAxis.edges.length - 1); + } } /** - * Lower and upper bound of each row in plot space. The uPlot y scale is - * linear, so log-scale rows are plotted in natural-log space and tick labels - * are exponentiated back. + * Lower and upper bound of each row in plot space. Series and histogram + * bucket rows are unit-height bands. The uPlot y scale is linear, so log-scale + * rows are plotted in natural-log space and tick labels are exponentiated back. */ export function rowPlotBounds(yAxis: HeatmapYAxis): { lo: number[]; hi: number[]; } { const rows = heatmapRowCount(yAxis); - if (yAxis.type === 'series') { + if (yAxis.type !== 'numeric') { return { lo: Array.from({ length: rows }, (_, r) => r), hi: Array.from({ length: rows }, (_, r) => r + 1), @@ -74,11 +79,6 @@ export function rowPlotBounds(yAxis: HeatmapYAxis): { const toPlot = yAxis.scale === 'log' ? Math.log : (v: number) => v; const lo = yAxis.edges.slice(0, rows).map(toPlot); const hi = yAxis.edges.slice(1, rows + 1).map(toPlot); - if (yAxis.openTop && rows > 0) { - const last = rows - 1; - const height = rows > 1 ? hi[last - 1] - lo[last - 1] : 1; - hi[last] = lo[last] + height; - } return { lo, hi }; } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts b/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts new file mode 100644 index 0000000000..d4858d6e24 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts @@ -0,0 +1,60 @@ +import { parsePromqlLe } from '@hyperdx/common-utils/dist/core/promql'; + +import type { HeatmapGrid } from './heatmapGrid'; +import { makeTimeIndexer } from './heatmapSeriesGrid'; + +/** A sample of one histogram bucket: its `le` label, time (ms) and cumulative value. */ +export type HistogramSample = { le: string; t: number; v: number }; + +/** + * A grid with one row per `le` bucket, lowest bound at the bottom. Bucket + * samples are cumulative (each counts everything at or below its bound), so a + * cell is its bucket's value minus the next bucket down's. + */ +export function gridFromHistogramSamples({ + samples, + times, +}: { + samples: HistogramSample[]; + times: number[]; +}): HeatmapGrid { + const stepMs = times.length > 1 ? times[1] - times[0] : 0; + const timeIndex = makeTimeIndexer(times, stepMs); + + // Keyed by numeric bound, since the same bound can be written differently + // (Prometheus 3 normalizes le="1" to le="1.0"). Such series are summed. + const bounds = [...new Set(samples.map(s => parsePromqlLe(s.le)))] + .filter(bound => !Number.isNaN(bound)) + .sort((a, b) => a - b); + const rowOf = new Map(bounds.map((bound, r) => [bound, r])); + const rows = bounds.length; + + const cumulative = new Array(times.length * rows).fill(NaN); + for (const { le, t, v } of samples) { + const ti = timeIndex(t); + const r = rowOf.get(parsePromqlLe(le)); + if (ti < 0 || r == null || !Number.isFinite(v)) continue; + const i = ti * rows + r; + cumulative[i] = Number.isNaN(cumulative[i]) ? v : cumulative[i] + v; + } + + const cells = new Array(times.length * rows).fill(0); + for (let ti = 0; ti < times.length; ti++) { + let below = 0; + for (let r = 0; r < rows; r++) { + const cum = cumulative[ti * rows + r]; + if (Number.isNaN(cum)) continue; + // rate() and increase() extrapolate each bucket separately, so adjacent + // buckets can come out slightly non-monotonic. + cells[ti * rows + r] = Math.max(0, cum - below); + below = Math.max(below, cum); + } + } + + return { + times, + stepMs, + yAxis: { type: 'buckets', bounds }, + cells, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapPaths.ts b/packages/app/src/components/DBHeatmapChart/heatmapPaths.ts index 621f810e79..51a092a0b6 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapPaths.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapPaths.ts @@ -56,12 +56,16 @@ export function heatmapPaths(opts: { const bottom = valToPosY(y0s[i], scaleY, yDim, yOff); const top = valToPosY(y1s[i], scaleY, yDim, yOff); + // Snap both edges (not just the origin) so adjacent cells share a + // pixel boundary; a fractional size leaves antialiased seams. + const x0 = Math.round(left); + const y0 = Math.round(bottom); rect( fillPaths[fills[i]], - Math.round(left), - Math.round(bottom), - right - left, - top - bottom, + x0, + y0, + Math.round(right) - x0, + Math.round(top) - y0, ); } } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts index fef15c961b..948df6920f 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts @@ -1,9 +1,6 @@ import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getAlignedDateRange } from '@hyperdx/common-utils/dist/core/utils'; -import { - isBuilderChartConfig, - isPromqlChartConfig, -} from '@hyperdx/common-utils/dist/guards'; +import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, @@ -52,7 +49,8 @@ export const PROMQL_DEFAULT_HEATMAP_SCALE_TYPE: HeatmapScaleType = 'linear'; /** * What a heatmap queries, by mode. Distribution heatmaps bucket a value * expression server-side; calculated heatmaps bucket the samples of a PromQL - * range query client-side; series heatmaps query one builder series or PromQL + * range query client-side; histogram heatmaps draw the `le` buckets of a + * PromQL histogram; series heatmaps query one builder series or PromQL * expression per time bucket and draw a row per series. */ export type HeatmapQuery = @@ -66,6 +64,10 @@ export type HeatmapQuery = config: PromqlConfigWithDateRange; scaleType: HeatmapScaleType; } + | { + mode: 'histogram'; + config: PromqlConfigWithDateRange; + } | { mode: 'series'; config: HeatmapSeriesChartConfig; @@ -94,11 +96,15 @@ export function resolveHeatmapGranularity({ } export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { - if (getHeatmapMode(config) === 'series') { + const mode = getHeatmapMode(config); + if (mode === 'series') { return { mode: 'series', config }; } if (isPromqlChartConfig(config)) { + if (mode === 'histogram') { + return { mode: 'histogram', config }; + } return { mode: 'calculated', config, @@ -129,24 +135,36 @@ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { } /** - * The time-chart query behind a series-mode or calculated heatmap, bucketed at - * the heatmap's granularity: `select[0]` and the group by for a builder - * config, or the first expression's range query for PromQL. + * The time-chart query behind a series-mode, calculated or histogram heatmap, + * bucketed at the heatmap's granularity: `select[0]` and the group by for a + * builder config, or the first expression's range query for PromQL. */ export function buildHeatmapSeriesConfig( config: HeatmapSeriesChartConfig, granularity: SQLInterval, ): ChartConfigWithDateRange { + // Heatmaps have no series limit control, a default is applied automatically. + // Heatmap settings only change how the response is drawn, so keep them out + // of the query's cache key: switching mode or scale reuses the data. + // Histogram mode is the exception, since its response is keyed by bucket. + if (isPromqlChartConfig(config)) { + return convertToTimeChartConfig({ + ...config, + granularity, + seriesLimit: undefined, + heatmap: + config.heatmap?.mode === 'histogram' + ? { mode: 'histogram' } + : undefined, + }); + } return convertToTimeChartConfig({ ...config, - ...(isBuilderChartConfig(config) && Array.isArray(config.select) + ...(Array.isArray(config.select) ? { select: config.select.slice(0, 1) } : {}), granularity, - // Heatmaps have no series limit control, a default is applied automatically seriesLimit: undefined, - // Heatmap settings only change how the response is drawn, so keep them - // out of the query's cache key: switching mode or scale reuses the data. heatmap: undefined, }); } diff --git a/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts new file mode 100644 index 0000000000..7baed326ac --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts @@ -0,0 +1,46 @@ +import { useMemo } from 'react'; +import { PromqlConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; + +import { gridFromHistogramSamples, HistogramSample } from './heatmapHistogram'; +import { HeatmapData, HeatmapView, useSettledView } from './useHeatmapView'; +import { useHeatmapSeriesQuery } from './useSeriesHeatmapData'; + +/** + * Query a histogram heatmap: a PromQL range query over histogram buckets, + * drawn as one row per `le` bucket. + */ +export function useHistogramHeatmapData({ + config, + enabled, +}: { + config: PromqlConfigWithDateRange; + enabled: boolean; +}): HeatmapData { + const { generatedTsBuckets, data, isLoading, isPlaceholderData, error } = + useHeatmapSeriesQuery({ config, enabled }); + + const currentView = useMemo(() => { + const samples: HistogramSample[] = (data?.data ?? []).map(row => ({ + le: String(row.le), + t: new Date(row.__hdx_time_bucket).getTime(), + v: Number(row.value), + })); + return { + grid: gridFromHistogramSamples({ + samples, + times: generatedTsBuckets.map(d => d.getTime()), + }), + generatedTsBuckets, + scaleType: 'linear', + effectiveMin: 0, + hiddenSeriesCount: 0, + }; + }, [data, generatedTsBuckets]); + + return { + view: useSettledView(currentView, isPlaceholderData), + isLoading, + isRefreshing: isPlaceholderData, + error, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts index fdc676bdb3..097ab6a0b3 100644 --- a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts +++ b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts @@ -52,8 +52,31 @@ function responseToHeatmapSeries( /** * Query a heatmap's series like a time chart, bucketed at the heatmap - * granularity, as one point list per series. Keeps the `maxSeries` series - * with the largest peaks. + * granularity. The previous response stays in place while refreshing. + */ +export function useHeatmapSeriesQuery({ + config, + enabled, +}: { + config: HeatmapSeriesChartConfig; + enabled: boolean; +}) { + const timeBuckets = useHeatmapTimeBuckets(config); + const seriesConfig = buildHeatmapSeriesConfig( + config, + timeBuckets.granularity, + ); + const query = useQueriedChartConfig(seriesConfig, { + queryKey: ['heatmap_series', seriesConfig], + enabled, + placeholderData: prev => prev, + }); + return { ...timeBuckets, ...query }; +} + +/** + * Query a heatmap's series as one point list per series. Keeps the + * `maxSeries` series with the largest peaks. */ export function useHeatmapSeriesPoints({ config, @@ -64,18 +87,15 @@ export function useHeatmapSeriesPoints({ enabled: boolean; maxSeries: number; }) { - const { granularity, generatedTsBuckets, fromMs, toMs } = - useHeatmapTimeBuckets(config); - - const seriesConfig = buildHeatmapSeriesConfig(config, granularity); - const { data, isLoading, isPlaceholderData, error } = useQueriedChartConfig( - seriesConfig, - { - queryKey: ['heatmap_series', seriesConfig], - enabled, - placeholderData: prev => prev, - }, - ); + const { + generatedTsBuckets, + fromMs, + toMs, + data, + isLoading, + isPlaceholderData, + error, + } = useHeatmapSeriesQuery({ config, enabled }); // Depend on primitive number ms values so a fresh dateRange array doesn't // re-parse the response and rebuild the grid on every render. diff --git a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx index 6a6b263768..3056f4d386 100644 --- a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx +++ b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx @@ -780,6 +780,7 @@ describe('ChartDisplaySettingsDrawer', () => { it.each([ ['hidden', 'distribution', false], + ['hidden', 'histogram', false], ['offered', 'series', true], ] as const)( 'is %s on a PromQL %s heatmap', diff --git a/packages/app/src/hooks/useStableValue.ts b/packages/app/src/hooks/useStableValue.ts new file mode 100644 index 0000000000..29fedb3490 --- /dev/null +++ b/packages/app/src/hooks/useStableValue.ts @@ -0,0 +1,13 @@ +import { useState } from 'react'; +import { isEqual } from 'lodash'; + +/** + * `value`, keeping the previous reference while its content is unchanged, so + * a caller that rebuilds equal objects every render doesn't invalidate memos. + */ +export function useStableValue(value: T): T { + const [stable, setStable] = useState(value); + const isSame = isEqual(stable, value); + if (!isSame) setStable(value); + return isSame ? stable : value; +} diff --git a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts index a2b057d4e3..3689762d53 100644 --- a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts +++ b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts @@ -1,7 +1,11 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { prometheusApi } from '@/api'; -import { queryPromqlChartConfig, toTableRows } from '@/utils/promqlChartQuery'; +import { + queryPromqlChartConfig, + toHistogramRows, + toTableRows, +} from '@/utils/promqlChartQuery'; // The real module is a Mantine/TanStack table component; only the row cap is // needed here. @@ -245,6 +249,53 @@ describe('toTableRows', () => { }); }); +describe('toHistogramRows', () => { + const bucket = (metric: Record, value: number) => ({ + metric, + values: [[1700000000, value]] as [number, number][], + }); + const rangeResult = (...result: ReturnType[]) => ({ + expression: 'sum by (le) (rate(x_bucket[5m]))', + isBucketed: true, + result, + }); + + it('keys each sample by its le label', () => { + const result = toHistogramRows([ + rangeResult(bucket({ le: '0.5' }, 2), bucket({ le: '+Inf' }, 3)), + ]); + expect(result.data).toEqual([ + { __hdx_time_bucket: '2023-11-14T22:13:20.000Z', le: '0.5', value: 2 }, + { __hdx_time_bucket: '2023-11-14T22:13:20.000Z', le: '+Inf', value: 3 }, + ]); + expect(result.meta?.map(({ name }) => name)).toEqual([ + '__hdx_time_bucket', + 'le', + 'value', + ]); + }); + + it.each>([{}, { le: '' }, { le: 'abc' }])( + 'rejects a series without a numeric le: %p', + metric => { + expect(() => toHistogramRows([rangeResult(bucket(metric, 1))])).toThrow( + /need series with an "le" label/, + ); + }, + ); + + it('rejects buckets that were not aggregated by le', () => { + expect(() => + toHistogramRows([ + rangeResult( + bucket({ le: '0.5', instance: 'a' }, 1), + bucket({ le: '0.5', instance: 'b' }, 1), + ), + ]), + ).toThrow('Several series share le="0.5"'); + }); +}); + describe('queryPromqlChartConfig', () => { const dateRange: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), @@ -372,4 +423,43 @@ describe('queryPromqlChartConfig', () => { }, ]); }); + + it('keys a histogram heatmap rows by bucket, not series name', async () => { + jest + .mocked(prometheusApi.queryRange) + .mockReset() + .mockResolvedValue({ + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { le: '+Inf' }, + values: [[dateRange[0].getTime() / 1000, '4']], + }, + ], + }, + }); + + const result = await queryPromqlChartConfig( + { + configType: 'promql', + displayType: DisplayType.Heatmap, + connection: 'conn', + legendTemplate: '{{job}}', + promqlExpression: [{ expression: 'sum by (le) (x_bucket)' }], + heatmap: { mode: 'histogram' }, + }, + dateRange, + new AbortController().signal, + {}, + ); + + expect(prometheusApi.queryRange).toHaveBeenCalledWith( + expect.objectContaining({ query: 'sum by (le) (x_bucket)' }), + ); + expect(result.data).toEqual([ + { __hdx_time_bucket: dateRange[0].toISOString(), le: '+Inf', value: 4 }, + ]); + }); }); diff --git a/packages/app/src/utils/promqlChartQuery.ts b/packages/app/src/utils/promqlChartQuery.ts index ccb912d55f..8d77dde0ce 100644 --- a/packages/app/src/utils/promqlChartQuery.ts +++ b/packages/app/src/utils/promqlChartQuery.ts @@ -2,6 +2,7 @@ import type { QueryAttribution } from '@hyperdx/common-utils/dist/clickhouse'; import { displayTypeSupportsInstantQuery, getQueriedPromqlSeries, + parsePromqlLe, promqlSeriesQueryType, promqlStep, reducePromqlSamples, @@ -262,6 +263,51 @@ export function toTableRows( }; } +/** + * Rows for a histogram heatmap: one per sample, grouped by the series' `le` + * bucket. Every series must be a distinct bucket (le). + */ +export function toHistogramRows( + expressions: ExpressionResult[], +): ChartQueryResult { + const seenLe = new Set(); + const data: Record[] = []; + for (const { result } of expressions) { + for (const { metric, values } of result) { + const le = metric.le; + if (Number.isNaN(parsePromqlLe(le))) { + throw new Error( + 'Histogram heatmaps need series with an "le" label, e.g. sum by (le) (rate(x_bucket[$__rate_interval])).', + ); + } + if (seenLe.has(le)) { + throw new Error( + `Several series share le="${le}". Aggregate them by bucket with sum by (le) (...).`, + ); + } + seenLe.add(le); + for (const [ts, value] of values) { + data.push({ + __hdx_time_bucket: new Date(ts * 1000).toISOString(), + le, + value, + }); + } + } + } + + return { + data, + meta: [ + { name: '__hdx_time_bucket', type: 'DateTime64(3)' }, + { name: 'le', type: 'String' }, + { name: 'value', type: 'Float64' }, + ], + rows: data.length, + isComplete: true, + }; +} + /** * Run a PromQL tile's expressions and shape the result like a ClickHouse * response, so the chart formatters treat it like every other source. @@ -300,6 +346,12 @@ export async function queryPromqlChartConfig( ), ); + // Rows keyed by the series' `le` bucket. Done here so that + // alias and legend templates don't interfere with the le label. + if (substituted.heatmap?.mode === 'histogram') { + return toHistogramRows(results); + } + return substituted.displayType === DisplayType.Table ? toTableRows(results) : toChartRows(results, substituted.legendTemplate?.trim() || undefined); diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index 2c9e53e02d..bab0b7196f 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -75,7 +75,7 @@ export class ChartEditorComponent { } /** Pick the heatmap's mode in its segmented control. */ - async setHeatmapMode(mode: 'Distribution' | 'Series') { + async setHeatmapMode(mode: 'Distribution' | 'Series' | 'Histogram') { await this.editorForm() .getByTestId('heatmap-mode-control') .locator('.mantine-SegmentedControl-label') @@ -105,8 +105,13 @@ export class ChartEditorComponent { return new HeatmapComponent(this.page, this.editorForm()); } + /** The error the preview shows in place of the chart. */ + get previewError(): Locator { + return this.editorForm().getByTestId('chart-error-state'); + } + /** A heatmap mode's (visually hidden) radio, for checking the selection. */ - heatmapModeOption(mode: 'Distribution' | 'Series'): Locator { + heatmapModeOption(mode: 'Distribution' | 'Series' | 'Histogram'): Locator { return this.editorForm() .getByTestId('heatmap-mode-control') .getByRole('radio', { name: mode, exact: true }); diff --git a/packages/app/tests/e2e/components/HeatmapComponent.ts b/packages/app/tests/e2e/components/HeatmapComponent.ts index 6eac1f3e36..4bbb267842 100644 --- a/packages/app/tests/e2e/components/HeatmapComponent.ts +++ b/packages/app/tests/e2e/components/HeatmapComponent.ts @@ -78,7 +78,7 @@ export class HeatmapComponent { } /** A labelled value of the open cell tooltip, e.g. `Count Value`. */ - async hoveredTooltipValue(label: 'Count Value' | 'Y Value') { + async hoveredTooltipValue(label: 'Count Value' | 'Y Value' | 'Bucket') { const row = this.container .getByText(`${label}:`, { exact: true }) .locator('..'); diff --git a/packages/app/tests/e2e/features/promql-heatmap.spec.ts b/packages/app/tests/e2e/features/promql-heatmap.spec.ts index 365e3ef6ae..7424099e68 100644 --- a/packages/app/tests/e2e/features/promql-heatmap.spec.ts +++ b/packages/app/tests/e2e/features/promql-heatmap.spec.ts @@ -1,10 +1,11 @@ /** - * PromQL heatmaps draw either a distribution of every series' samples, - * bucketed client-side, or one row per series of a range query, named by the - * legend template. + * PromQL heatmaps draw a distribution of every series' samples, bucketed + * client-side; one row per series of a range query, named by the legend + * template; or one row per bucket of a Prometheus histogram. * * The seed gives `e2e_service_up` one series per `SERVICES` entry, labelled - * `service`, one sample of 1 a minute. + * `service`, one sample of 1 a minute, and `e2e_request_duration_seconds` a + * cumulative `_bucket` series per service and `le` in 0.1, 0.5, 1 and +Inf. */ import { DisplayType } from '@hyperdx/common-utils/dist/types'; @@ -12,13 +13,23 @@ import { HeatmapComponent } from '../components/HeatmapComponent'; import { DashboardPage } from '../page-objects/DashboardPage'; import { SERVICES } from '../seed-clickhouse'; import { expect, test } from '../utils/base-test'; -import { E2E_PROMQL_METRIC_NAME, PROMQL_SOURCE_NAME } from '../utils/constants'; +import { + E2E_PROMQL_HISTOGRAM_METRIC_NAME, + E2E_PROMQL_METRIC_NAME, + PROMQL_SOURCE_NAME, +} from '../utils/constants'; // Ends in `)` rather than inside the selector so the autocomplete popup closes. const EXPRESSION = `last_over_time(${E2E_PROMQL_METRIC_NAME}{service!=""}[$__interval])`; +// A fixed window holds several of the once-a-minute samples rate() needs. +const BUCKET_RATE = `rate(${E2E_PROMQL_HISTOGRAM_METRIC_NAME}_bucket[5m])`; +const HISTOGRAM_EXPRESSION = `sum by (le) (${BUCKET_RATE})`; -/** Open a new tile on a new dashboard as a PromQL heatmap of `EXPRESSION`. */ -async function startPromqlHeatmapTile(dashboardPage: DashboardPage) { +/** Open a new tile on a new dashboard as a PromQL heatmap of `expression`. */ +async function startPromqlHeatmapTile( + dashboardPage: DashboardPage, + expression = EXPRESSION, +) { const editor = dashboardPage.chartEditor; await dashboardPage.goto(); await dashboardPage.createNewDashboard(); @@ -29,7 +40,7 @@ async function startPromqlHeatmapTile(dashboardPage: DashboardPage) { await editor.switchToPromqlMode(); await editor.selectSource(PROMQL_SOURCE_NAME); await editor.setChartName('PromQL heatmap tile'); - await editor.replacePromqlExpression(EXPRESSION); + await editor.replacePromqlExpression(expression); } /** Save the tile, and check it renders before and after a reload. */ @@ -143,4 +154,62 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { await expect(editor.heatmapScaleOption('Log')).toBeChecked(); }); }); + + test('draws histogram buckets, and persists', async ({ page }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + const assertRendersBuckets = async (heatmap: HeatmapComponent) => { + await expect(heatmap.canvas.first()).toBeVisible({ timeout: 30000 }); + await expect(heatmap.notEnoughDataText).toHaveCount(0); + await heatmap.hoverPopulatedCell(); + expect( + Number(await heatmap.hoveredTooltipValue('Count Value')), + ).toBeGreaterThan(0); + expect(await heatmap.hoveredTooltipValue('Bucket')).toMatch( + /^(0 – 0\.1|0\.1 – 0\.5|0\.5 – 1|> 1)\b/, + ); + }; + + await test.step('Create a PromQL histogram heatmap tile', async () => { + await startPromqlHeatmapTile(dashboardPage, HISTOGRAM_EXPRESSION); + await editor.setHeatmapMode('Histogram'); + await editor.runQuery(false); + }); + + await test.step('The expression is queried as written', async () => { + await editor.openGeneratedPromql(); + await expect + .poll(() => editor.getGeneratedPromqlText()) + .toContain(HISTOGRAM_EXPRESSION); + }); + + await test.step('The preview draws the buckets', async () => { + await assertRendersBuckets(editor.previewHeatmap); + }); + + await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersBuckets); + }); + + await test.step('Histogram mode persists', async () => { + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await expect(editor.heatmapModeOption('Histogram')).toBeChecked(); + }); + }); + + test('explains buckets that were not aggregated by le', async ({ page }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + await startPromqlHeatmapTile(dashboardPage, BUCKET_RATE); + await editor.setHeatmapMode('Histogram'); + await editor.runQuery(false); + + await expect(editor.previewError).toContainText( + 'Several series share le=', + { timeout: 30000 }, + ); + }); }); diff --git a/packages/app/tests/e2e/seed-clickhouse.ts b/packages/app/tests/e2e/seed-clickhouse.ts index 98c537dc6f..9736ffdaea 100644 --- a/packages/app/tests/e2e/seed-clickhouse.ts +++ b/packages/app/tests/e2e/seed-clickhouse.ts @@ -22,6 +22,7 @@ import { E2E_METADATA_MV_LOGS_TABLE, E2E_METRICS_GAUGE_TABLE, E2E_METRICS_SUM_TABLE, + E2E_PROMQL_HISTOGRAM_METRIC_NAME, E2E_PROMQL_METRIC_NAME, E2E_PROMQL_TABLE, E2E_SESSIONS_TABLE, @@ -1101,35 +1102,51 @@ export async function seedClickHouse(): Promise { await seedPromqlSeries(client, startMs, endMs); console.log(` Inserted ${SERVICES.length} PromQL series`); + await seedPromqlHistogram(client, startMs, endMs); + console.log(` Inserted ${SERVICES.length} PromQL histograms`); + console.log('ClickHouse seeding complete'); } /** - * Seed one `E2E_PROMQL_METRIC_NAME` series per service, one sample a minute - * across the window. Written through the TimeSeries table itself, not its inner - * tables: the engine derives each series' `id` and splits samples across inner - * tables in ways that change between ClickHouse versions, and rows written - * around it are invisible to the PromQL engine. + * Insert a metric family and its series into the PromQL table, one sample a + * minute across the window. `valueSql` computes the nth sample's value. Written + * through the TimeSeries table itself, not its inner tables: the engine derives + * each series' `id` and splits samples across inner tables in ways that change + * between ClickHouse versions, and rows written around it are invisible to the + * PromQL engine. */ -async function seedPromqlSeries( +async function insertPromqlMetric( client: ReturnType, startMs: number, endMs: number, + family: { name: string; type: string; unit: string; help: string }, + series: { + metricName: string; + tags: Record; + valueSql: string; + }[], ) { const table = `${E2E_CLICKHOUSE_DATABASE}.${E2E_PROMQL_TABLE}`; await client.query(` INSERT INTO ${table} (metric_family, type, unit, help) - VALUES ('${E2E_PROMQL_METRIC_NAME}', 'gauge', '', 'E2E service liveness') + VALUES ('${family.name}', '${family.type}', '${family.unit}', '${family.help}') `); const stepMs = 60000; const sampleCount = Math.max(1, Math.floor((endMs - startMs) / stepMs)); - const rows = SERVICES.map( - service => - `SELECT '${E2E_PROMQL_METRIC_NAME}', map('service', '${service}'), ` + - `arrayMap(n -> (fromUnixTimestamp64Milli(toInt64(${startMs} + n * ${stepMs})), 1.0), range(${sampleCount}))`, - ).join('\n UNION ALL\n '); + const rows = series + .map(({ metricName, tags, valueSql }) => { + const tagArgs = Object.entries(tags) + .flatMap(([key, value]) => [`'${key}'`, `'${value}'`]) + .join(', '); + return ( + `SELECT '${metricName}', map(${tagArgs}), ` + + `arrayMap(n -> (fromUnixTimestamp64Milli(toInt64(${startMs} + n * ${stepMs})), ${valueSql}), range(${sampleCount}))` + ); + }) + .join('\n UNION ALL\n '); await client.query(` INSERT INTO ${table} (metric_name, tags, time_series) @@ -1137,6 +1154,72 @@ async function seedPromqlSeries( `); } +/** Seed one `E2E_PROMQL_METRIC_NAME` series per service, always 1. */ +async function seedPromqlSeries( + client: ReturnType, + startMs: number, + endMs: number, +) { + await insertPromqlMetric( + client, + startMs, + endMs, + { + name: E2E_PROMQL_METRIC_NAME, + type: 'gauge', + unit: '', + help: 'E2E service liveness', + }, + SERVICES.map(service => ({ + metricName: E2E_PROMQL_METRIC_NAME, + tags: { service }, + valueSql: '1.0', + })), + ); +} + +/** + * Each bucket's per-minute increase, keyed by its `le` label. Cumulative, so a + * minute adds 1 observation <= 0.1, 2 in (0.1, 0.5], 1 in (0.5, 1] and 1 above. + */ +const PROMQL_HISTOGRAM_BUCKET_INCREASES: Record = { + '0.1': 1, + '0.5': 3, + '1': 4, + '+Inf': 5, +}; + +/** + * Seed a classic histogram, `E2E_PROMQL_HISTOGRAM_METRIC_NAME`, per service: + * one cumulative counter series per `le` bucket, growing at a fixed rate. + */ +async function seedPromqlHistogram( + client: ReturnType, + startMs: number, + endMs: number, +) { + await insertPromqlMetric( + client, + startMs, + endMs, + { + name: E2E_PROMQL_HISTOGRAM_METRIC_NAME, + type: 'histogram', + unit: 'seconds', + help: 'E2E request duration', + }, + SERVICES.flatMap(service => + Object.entries(PROMQL_HISTOGRAM_BUCKET_INCREASES).map( + ([le, increase]) => ({ + metricName: `${E2E_PROMQL_HISTOGRAM_METRIC_NAME}_bucket`, + tags: { service, le }, + valueSql: `toFloat64(n * ${increase})`, + }), + ), + ), + ); +} + // Allow running directly for testing if (require.main === module) { seedClickHouse() diff --git a/packages/app/tests/e2e/utils/constants.ts b/packages/app/tests/e2e/utils/constants.ts index 91a105a240..6e43341375 100644 --- a/packages/app/tests/e2e/utils/constants.ts +++ b/packages/app/tests/e2e/utils/constants.ts @@ -60,8 +60,10 @@ export const E2E_METRICS_SUM_TABLE = 'e2e_otel_metrics_sum'; // tables above are not usable for PromQL. The seeder writes through the // timeSeries* table functions, which take this same table name. export const E2E_PROMQL_TABLE = 'e2e_promql'; -/** The one metric name seeded into the PromQL table, one series per service. */ +/** A gauge seeded into the PromQL table, one series per service. */ export const E2E_PROMQL_METRIC_NAME = 'e2e_service_up'; +/** A classic histogram seeded into the PromQL table as `_bucket` series. */ +export const E2E_PROMQL_HISTOGRAM_METRIC_NAME = 'e2e_request_duration_seconds'; // A second database holding OTEL-shaped metric tables, so the metric table // autofill tests can switch the source form's database and see tables detected // from the new one. The table names differ from the `default` database's so an diff --git a/packages/common-utils/src/core/__tests__/promql.test.ts b/packages/common-utils/src/core/__tests__/promql.test.ts index a0c1c8a6cf..2f19318b0d 100644 --- a/packages/common-utils/src/core/__tests__/promql.test.ts +++ b/packages/common-utils/src/core/__tests__/promql.test.ts @@ -6,6 +6,7 @@ import { getPromqlSeries, getQueriedPromqlSeries, isRangeQuery, + parsePromqlLe, PROMQL_MACROS, promqlSeriesQueryType, promqlStep, @@ -250,6 +251,22 @@ describe('promqlStep with a window', () => { }); }); +describe('parsePromqlLe', () => { + it.each([ + ['0.5', 0.5], + ['10', 10], + ['1e+06', 1e6], + ['-1', -1], + ['+Inf', Infinity], + ])('parses %s', (le, expected) => { + expect(parsePromqlLe(le)).toBe(expected); + }); + + it.each([undefined, '', ' ', 'abc'])('is NaN for %p', le => { + expect(parsePromqlLe(le)).toBeNaN(); + }); +}); + describe('PROMQL_MACROS', () => { const hour: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), diff --git a/packages/common-utils/src/core/heatmap.ts b/packages/common-utils/src/core/heatmap.ts index 72a761b668..2018299a00 100644 --- a/packages/common-utils/src/core/heatmap.ts +++ b/packages/common-utils/src/core/heatmap.ts @@ -1,8 +1,8 @@ -import type { HeatmapMode } from '@/types'; +import type { PromqlHeatmapMode } from '@/types'; -/** A builder heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ -export function getHeatmapMode(config: { - heatmap?: { mode?: HeatmapMode }; -}): HeatmapMode { +/** A heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ +export function getHeatmapMode(config: { + heatmap?: { mode?: M }; +}): M | 'distribution' { return config.heatmap?.mode ?? 'distribution'; } diff --git a/packages/common-utils/src/core/promql.ts b/packages/common-utils/src/core/promql.ts index 6406cc6b7c..7f070801b5 100644 --- a/packages/common-utils/src/core/promql.ts +++ b/packages/common-utils/src/core/promql.ts @@ -133,6 +133,17 @@ export const promqlStep = ( ): string => `${promqlStepSeconds(granularity, dateRange, minGranularitySeconds)}s`; +/** + * A histogram bucket's `le` label as its upper bound. Prometheus writes the + * open-ended bucket as `+Inf`, which `Number` and `parseFloat` don't accept. + * NaN for a missing or non-numeric label. + */ +export function parsePromqlLe(le: string | undefined): number { + if (le === '+Inf') return Infinity; + if (le == null || le.trim() === '') return NaN; + return Number(le); +} + /** * Prometheus' default scrape interval, which `$__rate_interval` assumes when * the PromQL source has no minimum auto granularity set. diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 07c779cd55..26935e84a8 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1708,6 +1708,13 @@ export type RatioMode = z.infer; export const HeatmapModeSchema = z.enum(['distribution', 'series']); export type HeatmapMode = z.infer; +// PromQL heatmaps can also draw pre-bucketed Prometheus histograms (`le` buckets). +export const PromqlHeatmapModeSchema = z.enum([ + ...HeatmapModeSchema.options, + 'histogram', +]); +export type PromqlHeatmapMode = z.infer; + export const HeatmapSettingsSchema = z.object({ mode: HeatmapModeSchema.optional(), }); @@ -1912,6 +1919,7 @@ const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ step: z.string().optional(), legendTemplate: z.string().max(MAX_LEGEND_TEMPLATE_LENGTH).optional(), heatmap: HeatmapSettingsSchema.extend({ + mode: PromqlHeatmapModeSchema.optional(), scaleType: HeatmapScaleTypeSchema.optional(), }).optional(), });