diff --git a/.changeset/nice-cows-hug.md b/.changeset/nice-cows-hug.md new file mode 100644 index 0000000000..74f51a6c2f --- /dev/null +++ b/.changeset/nice-cows-hug.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Support distribution heatmaps calculated from promql data diff --git a/.changeset/promql-histogram-heatmap.md b/.changeset/promql-histogram-heatmap.md new file mode 100644 index 0000000000..a86ac25ecf --- /dev/null +++ b/.changeset/promql-histogram-heatmap.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +--- + +feat: Support histogram heatmaps from PromQL bucket metrics diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 6be88f101e..6aa957e671 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -8,9 +8,9 @@ import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { ChartConfigWithDateRange, DisplayType, - HeatmapMode, MAX_LEGEND_TEMPLATE_LENGTH, NumberFormat, + PromqlHeatmapMode, } from '@hyperdx/common-utils/dist/types'; import { Alert, @@ -65,7 +65,8 @@ export type ChartConfigDisplaySettings = Pick< // PromQL-only: Handlebars template over each series' Prometheus label set // that renders the legend/tooltip name. legendTemplate?: string; - // Distribution heatmaps only; stored on the heatmap's select[0]. + // Distribution heatmaps only; stored on a builder heatmap's select[0] and + // on a PromQL heatmap's `heatmap.scaleType`. heatmapScaleType?: HeatmapScaleType; }; @@ -88,8 +89,10 @@ interface ChartDisplaySettingsDrawerProps { configType?: 'sql' | 'builder' | 'promql'; /** Whether a PromQL tile's queried expression runs over a range. */ promqlUsesRange?: boolean; - /** A builder heatmap's mode; only distribution heatmaps have a numeric y axis to scale. */ - heatmapMode?: HeatmapMode; + /** A heatmap's mode; only distribution heatmaps have a numeric y axis to scale. */ + heatmapMode?: PromqlHeatmapMode; + /** The heatmap scale used when the tile has none set. */ + defaultHeatmapScaleType?: HeatmapScaleType; previousDateRange?: [Date, Date]; onChange: (settings: ChartConfigDisplaySettings, isDirty: boolean) => void; onClose: () => void; @@ -98,7 +101,8 @@ interface ChartDisplaySettingsDrawerProps { function applyDefaultSettings( settings: ChartConfigDisplaySettings, - fallbackNumberFormat?: NumberFormat, + fallbackNumberFormat: NumberFormat | undefined, + defaultHeatmapScaleType: HeatmapScaleType, ): DrawerFormValues { return { numberFormat: @@ -121,7 +125,7 @@ function applyDefaultSettings( ? attachLocalIds(settings.colorRules) : undefined, backgroundChart: settings.backgroundChart, - heatmapScaleType: settings.heatmapScaleType ?? 'log', + heatmapScaleType: settings.heatmapScaleType ?? defaultHeatmapScaleType, }; } @@ -132,6 +136,7 @@ export default function ChartDisplaySettingsDrawer({ configType, promqlUsesRange = false, heatmapMode, + defaultHeatmapScaleType = 'log', defaultNumberFormat, onChange, onClose, @@ -139,8 +144,13 @@ export default function ChartDisplaySettingsDrawer({ isPerSeriesNumberFormatAllowed = false, }: ChartDisplaySettingsDrawerProps) { const appliedDefaults = useMemo( - () => applyDefaultSettings(settings, defaultNumberFormat), - [settings, defaultNumberFormat], + () => + applyDefaultSettings( + settings, + defaultNumberFormat, + defaultHeatmapScaleType, + ), + [settings, defaultNumberFormat, defaultHeatmapScaleType], ); const { @@ -197,9 +207,10 @@ export default function ChartDisplaySettingsDrawer({ applyDefaultSettings( {} as ChartConfigDisplaySettings, defaultNumberFormat, + defaultHeatmapScaleType, ), ); - }, [reset, defaultNumberFormat]); + }, [reset, defaultNumberFormat, defaultHeatmapScaleType]); const isTimeChart = isTimeSeriesDisplayType(displayType); @@ -212,12 +223,18 @@ export default function ChartDisplaySettingsDrawer({ const isClientSideSeriesLimit = showSeriesLimit && (configType === 'sql' || configType === 'promql'); + const isDistributionHeatmap = + displayType === DisplayType.Heatmap && heatmapMode === 'distribution'; + // Every PromQL display that surfaces a series name. A number tile shows one - // value and a table gives each label its own column, so neither has a legend. + // value, a table gives each label its own column, a distribution heatmap + // merges every series into one, and a histogram heatmap labels rows by + // bucket, so none has a legend. const showLegendTemplate = configType === 'promql' && displayType !== DisplayType.Number && - displayType !== DisplayType.Table; + displayType !== DisplayType.Table && + !(displayType === DisplayType.Heatmap && heatmapMode !== 'series'); // On pie/bar builder charts, seriesLimit becomes a plain SQL LIMIT on the // number of slices/bars; on PromQL it trims the reduced series client-side @@ -254,8 +271,7 @@ export default function ChartDisplaySettingsDrawer({ : 'Available on query-builder number tiles.'; // Log scale only means something on a numeric (distribution mode) y axis - const showHeatmapScale = - displayType === DisplayType.Heatmap && heatmapMode === 'distribution'; + const showHeatmapScale = isDistributionHeatmap; return ( ( )} diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 07313da232..5dac49f90a 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -11,18 +11,22 @@ import { } from '@hyperdx/common-utils/dist/core/promql'; import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { + DisplayType, MAX_PROMQL_EXPRESSIONS, + PromqlHeatmapMode, SourceKind, } from '@hyperdx/common-utils/dist/types'; -import { Button, Divider, Flex, Group, Text } from '@mantine/core'; +import { Box, Button, Divider, Flex, Group, Text } from '@mantine/core'; import { IconCirclePlus } from '@tabler/icons-react'; +import { HeatmapModeControl } from '@/components/DBEditTimeChartForm/HeatmapModeControl'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; import { useSource } from '@/source'; import PromqlExpressionEditor from './PromqlExpressionEditor'; import { ChartEditorFormState } from './types'; +import { promqlExpressionPlaceholder } from './utils'; export default function PromqlChartEditor({ control, @@ -30,12 +34,16 @@ export default function PromqlChartEditor({ allowedSourceKinds, onSubmit, onOpenDisplaySettings, + heatmapMode, + onHeatmapModeChange, }: { control: Control; getValues: UseFormGetValues; allowedSourceKinds: SourceKind[]; onSubmit: (suppressErrorNotification?: boolean) => void; onOpenDisplaySettings: () => void; + heatmapMode: PromqlHeatmapMode; + onHeatmapModeChange: (mode: PromqlHeatmapMode) => void; }) { const { fields: expressions, @@ -106,6 +114,15 @@ export default function PromqlChartEditor({ allowedSourceKinds={allowedSourceKinds} /> + {displayType === DisplayType.Heatmap && ( + + + + )} {expressions.map((field, index) => ( 0} isInstantQuerySupported={displayTypeSupportsInstantQuery({ displayType, diff --git a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx index c2bea4b95d..61c5d707cd 100644 --- a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx @@ -12,6 +12,7 @@ export default function PromqlExpressionEditor({ index, length, metricNames, + placeholder, isIgnored, isInstantQuerySupported, isReducerSupported, @@ -26,6 +27,7 @@ export default function PromqlExpressionEditor({ index: number; length: number; metricNames: string[] | undefined; + placeholder: string; /** Whether the display type leaves this expression unqueried. */ isIgnored: boolean; /** Whether the display type lets this expression choose between instant and range queries. */ @@ -62,7 +64,7 @@ export default function PromqlExpressionEditor({ value={expressionField.value ?? ''} onChange={expressionField.onChange} onSubmit={() => onSubmit()} - placeholder="rate(http_requests_total{service='api'}[5m])" + placeholder={placeholder} metricNames={metricNames} enableMacros /> diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index ce841c260f..a45eb39922 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -19,6 +19,7 @@ import { convertFormStateToSavedChartConfig, convertSavedChartConfigToFormState, getAllowedSourceKinds, + promqlExpressionPlaceholder, validateChartForm, } from '@/components/ChartEditor/utils'; @@ -668,12 +669,12 @@ describe('PromQL expressions', () => { series: [], }); - it('saves a heatmap with only its first expression queried and no heatmap mode', () => { + it('saves a heatmap with only its first expression queried and its heatmap settings', () => { const result = convertFormStateToSavedChartConfig( { ...promqlForm([{ expression: 'up' }], DisplayType.Heatmap), legendTemplate: '{{pod}}', - heatmap: { mode: 'distribution' }, + heatmap: { mode: 'distribution', scaleType: 'log' }, granularity: '5 minute', }, undefined, @@ -684,8 +685,52 @@ describe('PromQL expressions', () => { promqlExpression: [{ expression: 'up' }], legendTemplate: '{{pod}}', granularity: '5 minute', + heatmap: { mode: 'distribution', scaleType: 'log' }, }); - expect(result).not.toHaveProperty('heatmap'); + }); + + it('saves a histogram heatmap', () => { + expect( + convertFormStateToSavedChartConfig( + { + ...promqlForm( + [{ expression: 'sum by (le) (x_bucket)' }], + DisplayType.Heatmap, + ), + heatmap: { mode: 'histogram' }, + }, + undefined, + ), + ).toMatchObject({ heatmap: { mode: 'histogram' } }); + }); + + it('queries a heatmap with its heatmap settings', () => { + const result = convertFormStateToChartConfig( + { + ...promqlForm([{ expression: 'up' }], DisplayType.Heatmap), + heatmap: { mode: 'series' }, + }, + [new Date(0), new Date(1000)], + undefined, + ); + expect(result).toMatchObject({ heatmap: { mode: 'series' } }); + }); + + it('drops heatmap settings from PromQL tiles that are not heatmaps', () => { + const form = { + ...promqlForm([{ expression: 'up' }], DisplayType.Line), + heatmap: { mode: 'series' as const }, + }; + expect( + convertFormStateToSavedChartConfig(form, undefined), + ).not.toHaveProperty('heatmap', expect.anything()); + expect( + convertFormStateToChartConfig( + form, + [new Date(0), new Date(1000)], + undefined, + ), + ).not.toHaveProperty('heatmap', expect.anything()); }); it('saves the form rows as the expression list', () => { @@ -1900,6 +1945,22 @@ describe('validateChartForm', () => { expect.objectContaining({ path: 'series.0.valueExpression' }), ); }); + + it('requires a value expression on a builder heatmap left in histogram mode', () => { + const errors = validateChartForm( + makeForm({ + displayType: DisplayType.Heatmap, + source: 'source-trace', + heatmap: { mode: 'histogram' }, + series: [{ ...seriesItem, valueExpression: '' }], + }), + traceSource, + jest.fn(), + ); + expect(errors).toContainEqual( + expect.objectContaining({ path: 'series.0.valueExpression' }), + ); + }); }); describe('color round-trip (sql/promql Number tile)', () => { @@ -2053,6 +2114,41 @@ describe('heatmap round-trip', () => { }); }); +describe('builder heatmap settings', () => { + const form: ChartEditorFormState = { + displayType: DisplayType.Heatmap, + source: 'source-trace', + where: '', + series: [{ ...seriesItem, valueExpression: 'Duration' }], + }; + + it('fall back to distribution mode from PromQL-only histogram mode', () => { + const histogramForm: ChartEditorFormState = { + ...form, + heatmap: { mode: 'histogram', scaleType: 'log' }, + }; + expect( + convertFormStateToSavedChartConfig(histogramForm, traceSource), + ).toHaveProperty('heatmap', { mode: undefined }); + expect( + convertFormStateToChartConfig( + histogramForm, + [new Date(0), new Date(1000)], + traceSource, + ), + ).toHaveProperty('heatmap', { mode: undefined }); + }); + + it('keep series mode, without PromQL-only settings', () => { + expect( + convertFormStateToSavedChartConfig( + { ...form, heatmap: { mode: 'series', scaleType: 'log' } }, + traceSource, + ), + ).toHaveProperty('heatmap', { mode: 'series' }); + }); +}); + describe('metric formulas (HDX-5080)', () => { const metricSeriesItem = { aggFn: 'avg' as const, @@ -2460,6 +2556,16 @@ describe('getAllowedSourceKinds', () => { ).toEqual([SourceKind.Trace]); }); + it('treats a leftover histogram mode on a builder heatmap as distribution', () => { + expect( + getAllowedSourceKinds({ + configType: 'builder', + displayType: DisplayType.Heatmap, + heatmapMode: 'histogram', + }), + ).toEqual([SourceKind.Trace]); + }); + it('offers only PromQL sources on PromQL heatmap tiles', () => { expect( getAllowedSourceKinds({ @@ -2470,3 +2576,21 @@ describe('getAllowedSourceKinds', () => { ).toEqual([SourceKind.Promql]); }); }); + +describe('promqlExpressionPlaceholder', () => { + it.each([ + ['distribution', /_sum\[5m\]\) \/ rate\(.*_count/], + ['series', /^sum by \(service\)/], + ['histogram', /^sum by \(le\) \(rate\(.*_bucket/], + ] as const)('suggests a %s heatmap expression', (mode, pattern) => { + expect(promqlExpressionPlaceholder(DisplayType.Heatmap, mode)).toMatch( + pattern, + ); + }); + + it('suggests the same expression on every non-heatmap chart', () => { + expect(promqlExpressionPlaceholder(DisplayType.Line, 'histogram')).toBe( + promqlExpressionPlaceholder(DisplayType.Number, 'distribution'), + ); + }); +}); diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts index d1c23ceb65..e8e11f61ed 100644 --- a/packages/app/src/components/ChartEditor/types.ts +++ b/packages/app/src/components/ChartEditor/types.ts @@ -28,9 +28,13 @@ export type SavedChartConfigWithSelectArray = Omit< * `promqlExpressions` is the PromQL counterpart, holding the array form of * `promqlExpression` (which a saved tile may carry as a bare string). **/ -export type ChartEditorFormState = Partial & +export type ChartEditorFormState = Partial< + Omit +> & Partial> & - Partial> & { + Partial> & { + // The PromQL settings are a superset of the builder's. + heatmap?: PromqlSavedChartConfig['heatmap']; alert?: BuilderSavedChartConfig['alert'] & { id?: string; createdBy?: AlertWithCreatedBy['createdBy']; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 0709cf6974..4885b61306 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -32,6 +32,7 @@ import { isRangeThresholdType, isTraceSource, PromqlChartConfig, + PromqlHeatmapMode, PromqlSavedChartConfig, PromqlSeries, RawSqlChartConfig, @@ -228,13 +229,13 @@ export function getAllowedSourceKinds({ }: { configType: ChartEditorFormState['configType']; displayType: DisplayType | undefined; - heatmapMode?: HeatmapMode; + heatmapMode?: PromqlHeatmapMode; }): SourceKind[] { if (configType === 'promql' && isPromqlDisplayType(displayType)) { return [SourceKind.Promql]; } if (displayType === DisplayType.Heatmap) { - return [...getHeatmapSourceKinds(heatmapMode)]; + return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; } if (isStringSelectDisplayType(displayType)) { return ROW_LISTING_SOURCE_KINDS; @@ -280,6 +281,7 @@ export function convertFormStateToSavedChartConfig( connection: form.connection ?? '', source: form.source || undefined, legendTemplate: form.legendTemplate?.trim() || undefined, + heatmap: formPromqlHeatmap(form), }; return promqlConfig; @@ -323,6 +325,7 @@ export function convertFormStateToSavedChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), select: [], where: form.where ?? '', source: source?.id ?? form.source ?? '', @@ -341,6 +344,7 @@ export function convertFormStateToSavedChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), select: isStringSelectDisplayType(form.displayType) ? typeof form.select === 'string' ? form.select @@ -380,6 +384,7 @@ export function convertFormStateToChartConfig( source: form.source || undefined, from: source?.from, legendTemplate: form.legendTemplate?.trim() || undefined, + heatmap: formPromqlHeatmap(form), }; return { ...promqlConfig, dateRange }; @@ -444,6 +449,7 @@ export function convertFormStateToChartConfig( 'promqlExpression', 'promqlExpressions', ]), + heatmap: formBuilderHeatmap(form), from: source.from, timestampValueExpression: source.timestampValueExpression, dateRange, @@ -552,6 +558,49 @@ function formPromqlExpressions(form: ChartEditorFormState): PromqlSeries[] { })); } +/** The form's heatmap settings, formatted for a builder tile */ +function formBuilderHeatmap( + form: ChartEditorFormState, +): BuilderSavedChartConfig['heatmap'] { + return form.heatmap == null + ? undefined + : { mode: toBuilderHeatmapMode(form.heatmap.mode) }; +} + +/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ +export function toBuilderHeatmapMode( + mode: PromqlHeatmapMode | undefined, +): HeatmapMode | undefined { + return mode === 'histogram' ? undefined : mode; +} + +const DEFAULT_PROMQL_PLACEHOLDER = + "rate(http_requests_total{service='api'}[5m])"; + +const PROMQL_HEATMAP_PLACEHOLDERS: Record = { + distribution: + 'rate(http_request_duration_seconds_sum[5m]) / rate(http_request_duration_seconds_count[5m])', + series: 'sum by (service) (rate(http_requests_total[5m]))', + histogram: 'sum by (le) (rate(http_request_duration_seconds_bucket[5m]))', +}; + +/** An example PromQL expression suited to the display type and heatmap mode. */ +export function promqlExpressionPlaceholder( + displayType: DisplayType | undefined, + heatmapMode: PromqlHeatmapMode, +): string { + return displayType === DisplayType.Heatmap + ? PROMQL_HEATMAP_PLACEHOLDERS[heatmapMode] + : DEFAULT_PROMQL_PLACEHOLDER; +} + +/** Heatmap settings are dropped from PromQL tiles that are not heatmaps. */ +function formPromqlHeatmap( + form: ChartEditorFormState, +): PromqlSavedChartConfig['heatmap'] { + return form.displayType === DisplayType.Heatmap ? form.heatmap : undefined; +} + export const validateChartForm = ( form: ChartEditorFormState, source: TSource | undefined, @@ -753,7 +802,7 @@ export const validateChartForm = ( !isRawSqlChart && !isPromqlChart && form.displayType === DisplayType.Heatmap && - getHeatmapMode(form) === 'distribution' && + getHeatmapMode({ heatmap: formBuilderHeatmap(form) }) === 'distribution' && Array.isArray(form.series) && form.series.length > 0 && !form.series[0]?.valueExpression diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx index d6ead0b592..c51000f5cc 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx @@ -106,6 +106,11 @@ function HeatmapSQLPreview({ }; const query = toHeatmapQuery(configWithTimestamp); + // Only PromQL heatmaps are calculated or histograms, and they have no SQL + if (query.mode === 'calculated' || query.mode === 'histogram') { + return null; + } + if (query.mode === 'series') { const seriesConfig = { ...query.config, dateRange }; return ( diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 55d7f2d62c..3523e0205e 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -18,7 +18,6 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { displayTypeRequiresSource, - getHeatmapSourceKinds, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -27,7 +26,7 @@ import { DashboardFilter, DisplayType, Filter, - HeatmapMode, + PromqlHeatmapMode, SavedChartConfig, SourceKind, TSource, @@ -74,9 +73,13 @@ import { isPromqlDisplayType, isRawSqlDisplayType, isStringSelectDisplayType, + toBuilderHeatmapMode, validateChartForm, } from '@/components/ChartEditor/utils'; -import { HEATMAP_DURATION_NUMBER_FORMAT } from '@/components/DBHeatmapChart'; +import { + HEATMAP_DURATION_NUMBER_FORMAT, + PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, +} from '@/components/DBHeatmapChart'; import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { InputControlled } from '@/components/InputControlled'; import SaveToDashboardModal from '@/components/SaveToDashboardModal'; @@ -161,7 +164,8 @@ const ALERT_IGNORES_DASHBOARD_FILTERS = * Populate form state with the standard series for a heatmap mode. * Distribution heatmaps bucket the trace duration (or `fallbackValueExpression` * on a trace source without one) with a duration number format; series - * heatmaps start from a plain count. + * heatmaps start from a plain count. PromQL distribution heatmaps bucket + * arbitrary samples, so they get no duration format. */ function applyHeatmapDefaults( setValue: UseFormSetValue, @@ -170,10 +174,12 @@ function applyHeatmapDefaults( mode, tableSource, fallbackValueExpression = '', + isPromql = false, }: { - mode: HeatmapMode; + mode: PromqlHeatmapMode; tableSource: TSource | undefined; fallbackValueExpression?: string; + isPromql?: boolean; }, ) { const isTraceDuration = @@ -194,13 +200,13 @@ function applyHeatmapDefaults( setValue('select', heatmapSeries); setValue('series', heatmapSeries); - if (mode === 'distribution') { + if (mode === 'distribution' && !isPromql) { setValue('series.0.countExpression', 'count()'); setValue('numberFormat', { ...HEATMAP_DURATION_NUMBER_FORMAT }); } else if ( isEqual(getValues('numberFormat'), HEATMAP_DURATION_NUMBER_FORMAT) ) { - // Unselect the default-applied duration format for series mode + // Unselect the default-applied duration format for builder series-mode setValue('numberFormat', undefined); } } @@ -310,7 +316,7 @@ export default function EditTimeChartForm({ const configType = useWatch({ control, name: 'configType' }); const connection = useWatch({ control, name: 'connection' }); const promqlExpressions = useWatch({ control, name: 'promqlExpressions' }); - const formHeatmapMode = getHeatmapMode({ + const storedHeatmapMode = getHeatmapMode({ heatmap: useWatch({ control, name: 'heatmap' }), }); @@ -319,8 +325,13 @@ export default function EditTimeChartForm({ configType === 'sql' && isRawSqlDisplayType(displayType); const isPromqlInput = configType === 'promql' && isPromqlDisplayType(displayType); - // PromQL heatmaps only support series mode - const heatmapMode: HeatmapMode = isPromqlInput ? 'series' : formHeatmapMode; + // Switching from PromQL to builder leaves the PromQL-only histogram mode in + // the form, which the builder treats as distribution. + const builderHeatmapMode = + toBuilderHeatmapMode(storedHeatmapMode) ?? 'distribution'; + const heatmapMode: PromqlHeatmapMode = isPromqlInput + ? storedHeatmapMode + : builderHeatmapMode; const { data: sources } = useSources(); @@ -382,7 +393,8 @@ export default function EditTimeChartForm({ colorRules, backgroundChart, legendTemplate, - heatmapScaleType, + builderHeatmapScaleType, + promqlHeatmapScaleType, ] = useWatch({ control, name: [ @@ -399,8 +411,12 @@ export default function EditTimeChartForm({ 'backgroundChart', 'legendTemplate', 'series.0.heatmapScaleType', + 'heatmap.scaleType', ], }); + const heatmapScaleType = isPromqlInput + ? promqlHeatmapScaleType + : builderHeatmapScaleType; // Format auto-detected purely from the datasource (e.g. duration for a trace // Duration column), used as the drawer's fallback when no explicit @@ -825,6 +841,7 @@ export default function EditTimeChartForm({ fallbackValueExpression: Array.isArray(select) ? (select[0]?.valueExpression ?? '') : '', + isPromql: isPromqlInput, }); } else if (!Array.isArray(select)) { const defaultSeries: SavedChartConfigWithSelectArray['select'] = [ @@ -856,21 +873,38 @@ export default function EditTimeChartForm({ configType, tableSource, heatmapMode, + isPromqlInput, ]); // Switching heatmap mode may trigger a source swap, and should apply new defaults. const onHeatmapModeChange = useCallback( - (mode: HeatmapMode) => { + (mode: PromqlHeatmapMode) => { setValue('heatmap.mode', mode); - applyHeatmapDefaults(setValue, getValues, { mode, tableSource }); + applyHeatmapDefaults(setValue, getValues, { + mode, + tableSource, + isPromql: isPromqlInput, + }); if ( tableSource != null && - getHeatmapSourceKinds(mode).includes(tableSource.kind) + getAllowedSourceKinds({ + configType, + displayType, + heatmapMode: mode, + }).includes(tableSource.kind) ) { onSubmit(true); } }, - [setValue, getValues, tableSource, onSubmit], + [ + setValue, + getValues, + tableSource, + onSubmit, + configType, + displayType, + isPromqlInput, + ], ); // Handle auto-submitting and form state updates when the source changes. @@ -995,7 +1029,11 @@ export default function EditTimeChartForm({ displayType === DisplayType.Heatmap && heatmapMode === 'distribution' ) { - setValue('series.0.heatmapScaleType', heatmapScaleType); + if (isPromqlInput) { + setValue('heatmap.scaleType', heatmapScaleType); + } else { + setValue('series.0.heatmapScaleType', heatmapScaleType); + } } // Display settings live in a separate drawer form, so RHF can't track // them. Latch dirty state only when the drawer reports actual changes. @@ -1005,7 +1043,15 @@ export default function EditTimeChartForm({ } onSubmit(); }, - [setValue, onDirtyChange, onSubmit, configType, displayType, heatmapMode], + [ + setValue, + onDirtyChange, + onSubmit, + configType, + displayType, + heatmapMode, + isPromqlInput, + ], ); const tableConnection = useMemo( @@ -1159,6 +1205,8 @@ export default function EditTimeChartForm({ allowedSourceKinds={allowedSourceKinds} onSubmit={onSubmit} onOpenDisplaySettings={openDisplaySettings} + heatmapMode={heatmapMode} + onHeatmapModeChange={onHeatmapModeChange} /> ) : isRawSqlInput ? ( )} @@ -1276,6 +1324,9 @@ export default function EditTimeChartForm({ displayType, })} heatmapMode={heatmapMode} + defaultHeatmapScaleType={ + isPromqlInput ? PROMQL_DEFAULT_HEATMAP_SCALE_TYPE : undefined + } onChange={handleUpdateDisplaySettings} onClose={closeDisplaySettings} // Heatmaps format with the chart-level number format only. diff --git a/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx b/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx new file mode 100644 index 0000000000..c9dcc46d50 --- /dev/null +++ b/packages/app/src/components/DBEditTimeChartForm/HeatmapModeControl.tsx @@ -0,0 +1,85 @@ +import { PromqlHeatmapMode } from '@hyperdx/common-utils/dist/types'; +import { Group, SegmentedControl, Stack, Text, Tooltip } from '@mantine/core'; +import { IconHelpCircle } from '@tabler/icons-react'; + +type ModeOption = { label: string; value: PromqlHeatmapMode; help: string }; + +const BUILDER_MODE_OPTIONS: ModeOption[] = [ + { + label: 'Distribution', + value: 'distribution', + help: 'Buckets each event by a value, such as duration. Color shows how many events fall in each bucket.', + }, + { + label: 'Series', + value: 'series', + help: 'One row per group. Color shows the aggregated value.', + }, +]; + +const PROMQL_MODE_OPTIONS: ModeOption[] = [ + { + label: 'Distribution', + value: 'distribution', + help: 'Buckets the samples of every series by value. Color shows how many samples fall in each bucket.', + }, + { + label: 'Series', + value: 'series', + help: 'One row per series. Color shows the sample value.', + }, + { + label: 'Histogram', + value: 'histogram', + help: 'One row per bucket of a Prometheus histogram, from series aggregated by their le label. Color shows the count in each bucket.', + }, +]; + +/** Histogram mode is offered only where `allowHistogram` is set (PromQL). */ +export function HeatmapModeControl({ + mode, + onModeChange, + allowHistogram = false, +}: { + mode: M; + onModeChange: (mode: M) => void; + allowHistogram?: boolean; +}) { + const options = allowHistogram ? PROMQL_MODE_OPTIONS : BUILDER_MODE_OPTIONS; + const isOfferedMode = (value: string): value is M => + options.some(o => o.value === value); + return ( + + { + if (isOfferedMode(value)) onModeChange(value); + }} + data={options.map(({ label, value }) => ({ label, value }))} + data-testid="heatmap-mode-control" + /> + + {options.map(({ label, help }) => ( + + {label}: {help} + + ))} + + } + > + + + + ); +} diff --git a/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx index dde2aab494..dd3439b1f6 100644 --- a/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/HeatmapSeriesEditor.tsx @@ -8,15 +8,7 @@ import { } from 'react-hook-form'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { HeatmapMode, TSource } from '@hyperdx/common-utils/dist/types'; -import { - Button, - Divider, - Flex, - SegmentedControl, - Stack, - Text, - Tooltip, -} from '@mantine/core'; +import { Button, Divider, Flex, Stack, Text, Tooltip } from '@mantine/core'; import { IconHelpCircle } from '@tabler/icons-react'; import { ChartEditorFormState } from '@/components/ChartEditor/types'; @@ -25,6 +17,7 @@ import SearchWhereInput from '@/components/SearchInput/SearchWhereInput'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; import { ChartSeriesEditor } from './ChartSeriesEditor'; +import { HeatmapModeControl } from './HeatmapModeControl'; type HeatmapSeriesEditorProps = { control: Control; @@ -70,21 +63,7 @@ export function HeatmapSeriesEditor({ return ( - { - if (value === 'distribution' || value === 'series') { - onModeChange(value); - } - }} - data={[ - { label: 'Distribution', value: 'distribution' }, - { label: 'Series', value: 'series' }, - ]} - data-testid="heatmap-mode-control" - /> + {mode === 'series' ? ( { + it('offers no histogram mode by default', () => { + renderWithMantine( + , + ); + expect(screen.getByRole('radio', { name: 'Series' })).toBeInTheDocument(); + expect(screen.queryByRole('radio', { name: 'Histogram' })).toBeNull(); + }); + + it('offers and selects histogram mode when allowed', async () => { + const onModeChange = jest.fn(); + renderWithMantine( + , + ); + await userEvent.click(screen.getByText('Histogram')); + expect(onModeChange).toHaveBeenCalledWith('histogram'); + }); + + it.each([ + [false, ['Distribution', 'Series']], + [true, ['Distribution', 'Series', 'Histogram']], + ])( + 'explains each offered mode on hover (allowHistogram: %s)', + async (allowHistogram, modes) => { + renderWithMantine( + , + ); + await userEvent.hover(screen.getByTestId('heatmap-mode-help')); + const tooltip = await screen.findByRole('tooltip'); + for (const mode of modes) { + expect(tooltip).toHaveTextContent(`${mode}:`); + } + expect(tooltip.textContent?.includes('Histogram:')).toBe(allowHistogram); + }, + ); +}); diff --git a/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx b/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx index 6ebb4f6118..1a95856dd9 100644 --- a/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx +++ b/packages/app/src/components/DBHeatmapChart/DBHeatmapChart.tsx @@ -3,10 +3,14 @@ import dynamic from 'next/dynamic'; import { HeatmapChartView, HeatmapChartViewProps } from './HeatmapChartView'; import type { HeatmapQuery } from './heatmapQueries'; import type { SelectionBounds } from './selection'; +import { useCalculatedHeatmapData } from './useCalculatedHeatmapData'; import { useHeatmapData } from './useHeatmapData'; +import { useHistogramHeatmapData } from './useHistogramHeatmapData'; import { useSeriesHeatmapData } from './useSeriesHeatmapData'; type DistributionQuery = Extract; +type CalculatedQuery = Extract; +type HistogramQuery = Extract; type SeriesQuery = Extract; type DistributionOnlyProps = { @@ -90,6 +94,44 @@ function SeriesHeatmapChart({ ); } +function CalculatedHeatmapChart({ + query, + enabled, + ...viewProps +}: HeatmapChartViewProps & { query: CalculatedQuery; enabled: boolean }) { + const data = useCalculatedHeatmapData({ + config: query.config, + scaleType: query.scaleType, + enabled, + }); + + return ( + + ); +} + +function HistogramHeatmapChart({ + query, + enabled, + ...viewProps +}: HeatmapChartViewProps & { query: HistogramQuery; enabled: boolean }) { + const data = useHistogramHeatmapData({ config: query.config, enabled }); + + return ( + + ); +} + function DBHeatmapChart({ query, enabled = true, @@ -98,9 +140,22 @@ function DBHeatmapChart({ selectionBounds, ...viewProps }: DBHeatmapChartProps) { - return query.mode === 'series' ? ( - - ) : ( + if (query.mode === 'series') { + return ( + + ); + } + if (query.mode === 'calculated') { + return ( + + ); + } + if (query.mode === 'histogram') { + return ( + + ); + } + return ( (numberFormat ? JSON.stringify(numberFormat) : ''), - [numberFormat], - ); + // chart and wiping u.select. + const stableNumberFormat = useContentStable(numberFormat); const tickFormatter = useCallback( - (value: number) => formatHeatmapTick(value, scaleType, numberFormat), - // eslint-disable-next-line react-hooks/exhaustive-deps - [numberFormatKey, scaleType], + (value: number) => formatHeatmapTick(value, scaleType, stableNumberFormat), + [stableNumberFormat, scaleType], + ); + const boundFormatter = useCallback( + (bound: number) => formatBucketBound(bound, stableNumberFormat), + [stableNumberFormat], ); const plotData = useMemo(() => gridToPlotData(grid), [grid]); const rowCount = heatmapRowCount(grid.yAxis); const cellKind = heatmapCellKind(grid.yAxis); - // Key on label content: every refresh builds a fresh labels array, and new - // options make uplot-react recreate the chart instead of updating its data. - const seriesLabelsKey = - grid.yAxis.type === 'series' - ? JSON.stringify(grid.yAxis.labels) - : undefined; - const seriesLabels = useMemo( - () => - seriesLabelsKey === undefined ? undefined : JSON.parse(seriesLabelsKey), - [seriesLabelsKey], + // Fresh grids carry fresh arrays, and new options make uplot-react recreate + // the chart instead of updating its data. + const seriesLabels = useContentStable( + grid.yAxis.type === 'series' ? grid.yAxis.labels : undefined, + ); + const bucketBounds = useContentStable( + grid.yAxis.type === 'buckets' ? grid.yAxis.bounds : undefined, ); const bucketPercentiles = useMemo( () => - grid.yAxis.type === 'numeric' ? computeBucketPercentiles(grid) : null, + grid.yAxis.type === 'series' ? null : computeBucketPercentiles(grid), [grid], ); const options: uPlot.Options = useMemo(() => { const opt = baseHeatmapOptions; const themedSeries = buildSeriesForPalette(palette, cellKind); - const yAxis = heatmapYAxisOptions(seriesLabels, scaleType, tickFormatter); + const yAxis = heatmapYAxisOptions({ + seriesLabels, + bucketBounds, + scaleType, + tickFormatter, + boundFormatter, + }); return { ...opt, ...(yAxis.scale != null @@ -224,6 +222,9 @@ export function HeatmapPlot({ }, }, plugins: [ + // False positive: onPointHighlight reads mouseInsideRef on cursor + // moves, not during render. + // eslint-disable-next-line react-hooks/refs highlightDataPlugin({ margin: 20, cellKind, @@ -287,6 +288,8 @@ export function HeatmapPlot({ palette, hasFilter, seriesLabels, + bucketBounds, + boundFormatter, cellKind, ]); @@ -308,11 +311,21 @@ export function HeatmapPlot({ numberFormat, ), } - : { - kind: 'distribution', - formattedY: tickFormatter(highlightedPoint.yVal), - percentile: bucketPercentiles?.get(highlightedRow), - }; + : bucketBounds != null + ? { + kind: 'bucket', + formattedRange: formatBucketRange( + bucketBounds, + highlightedRow, + boundFormatter, + ), + percentile: bucketPercentiles?.get(highlightedRow), + } + : { + kind: 'distribution', + formattedY: tickFormatter(highlightedPoint.yVal), + percentile: bucketPercentiles?.get(highlightedRow), + }; return (
- Y Value: {cell.formattedY} + {cell.kind === 'bucket' ? ( + <> + Bucket: {cell.formattedRange} + + ) : ( + <> + Y Value: {cell.formattedY} + + )} {cell.percentile != null && ` (p${new Intl.NumberFormat('en-US', { maximumFractionDigits: 1, diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts index 3b4deae2dc..fc10cb8636 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapAxis.test.ts @@ -1,4 +1,7 @@ import { + bucketRowSplits, + formatBucketBound, + formatBucketRange, formatHeatmapTick, logScaleSplits, } from '@/components/DBHeatmapChart/heatmapAxis'; @@ -32,4 +35,83 @@ describe('logScaleSplits', () => { const splits = logScaleSplits(Math.log(2), Math.log(40)); expect(splits.map(Math.exp).map(v => Math.round(v))).toEqual([3, 10, 30]); }); + + it('uses every mantissa within a range narrower than a decade', () => { + const splits = logScaleSplits(Math.log(0.3), Math.log(0.66)); + expect(splits.map(v => +Math.exp(v).toFixed(2))).toEqual([ + 0.3, 0.4, 0.5, 0.6, + ]); + }); + + it('spaces nice values evenly within a range narrower than a mantissa step', () => { + const splits = logScaleSplits(Math.log(0.3), Math.log(0.33)); + expect(splits.length).toBeGreaterThanOrEqual(3); + expect(splits.map(v => +Math.exp(v).toFixed(3))).toEqual([ + 0.3, 0.31, 0.32, 0.33, + ]); + }); +}); + +describe('formatBucketBound', () => { + it('labels the open-ended bucket +Inf', () => { + expect(formatBucketBound(Infinity, undefined)).toBe('+Inf'); + }); + + it('keeps the digits that tell adjacent bounds apart', () => { + expect(formatBucketBound(0.0025, undefined)).toBe('0.0025'); + expect(formatBucketBound(2.5, undefined)).toBe('2.5'); + expect(formatBucketBound(2.5, { output: 'number', mantissa: 1 })).toBe( + '2.5', + ); + }); + + it('formats durations', () => { + expect(formatBucketBound(250, { output: 'number', unit: 'ms' })).toBe( + '250ms', + ); + }); +}); + +describe('formatBucketRange', () => { + const format = (bound: number) => formatBucketBound(bound, undefined); + + it('starts the lowest bucket at 0', () => { + expect(formatBucketRange([0.1, 1, Infinity], 0, format)).toBe('0 – 0.1'); + }); + + it('spans from the next bound down', () => { + expect(formatBucketRange([0.1, 1, Infinity], 1, format)).toBe('0.1 – 1'); + }); + + it('reads the +Inf bucket as everything above the last finite bound', () => { + expect(formatBucketRange([0.1, 1, Infinity], 2, format)).toBe('> 1'); + }); + + it('leaves a non-positive lowest bucket unbounded below', () => { + expect(formatBucketRange([-1, 1], 0, format)).toBe('≤ -1'); + expect(formatBucketRange([0, 1], 0, format)).toBe('≤ 0'); + }); + + it('reads a lone +Inf bucket as every observation', () => { + expect(formatBucketRange([Infinity], 0, format)).toBe('≤ +Inf'); + }); +}); + +describe('bucketRowSplits', () => { + it('ticks the top of every row when they fit', () => { + expect(bucketRowSplits(4, 200)).toEqual([1, 2, 3, 4]); + }); + + it('thins ticks to evenly spaced rows that fit the height, keeping the top row', () => { + const splits = bucketRowSplits(100, 200); + expect(splits.length).toBeLessThan(100); + expect(splits.at(-1)).toBe(100); + const gaps = new Set(splits.slice(1).map((s, i) => s - splits[i])); + expect(gaps.size).toBe(1); + expect(bucketRowSplits(100, 400).length).toBeGreaterThan(splits.length); + }); + + it('keeps the top tick in a very short plot', () => { + expect(bucketRowSplits(5, 0)).toEqual([5]); + }); }); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts new file mode 100644 index 0000000000..78f3389524 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapCalculate.test.ts @@ -0,0 +1,280 @@ +import { + CALCULATED_TARGET_BUCKETS, + gridFromSamples, +} from '@/components/DBHeatmapChart/heatmapCalculate'; +import { EMPTY_HEATMAP_GRID } from '@/components/DBHeatmapChart/heatmapGrid'; + +const STEP = 60_000; +const times = [0, STEP, 2 * STEP]; + +const series = (name: string, points: [number, number][]) => ({ + name, + points: points.map(([t, v]) => ({ t, v })), +}); + +const linearEdges = (values: number[]) => { + const { grid } = gridFromSamples({ + series: [ + series( + 'a', + values.map(v => [0, v]), + ), + ], + times, + scaleType: 'linear', + }); + return grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; +}; + +describe('gridFromSamples', () => { + it.each([ + [95, 10], + [9, 1], + [1, 0.1], + [0.45, 0.05], + [1000, 100], + [130, 10], + [150, 15], + ])('splits [0, %p] into about 10 linear buckets of %p', (max, bucketSize) => { + const edges = linearEdges([0, max]); + expect(edges[0]).toBe(0); + expect(edges[1]).toBe(bucketSize); + }); + + it('counts the samples of every series per time column and linear bucket', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [STEP, 50], + ]), + series('b', [ + [0, 5], + [STEP, 55], + [2 * STEP, 95], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis).toEqual({ + type: 'numeric', + scale: 'linear', + edges: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], + }); + expect(effectiveMin).toBe(0); + const rows = 10; + const column = (ti: number) => + grid.cells.slice(ti * rows, ti * rows + rows); + expect(column(0)).toEqual([2, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(column(1)).toEqual([0, 0, 0, 0, 0, 2, 0, 0, 0, 0]); + expect(column(2)).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + }); + + it('puts a value on a bucket edge in the bucket it starts', () => { + const { grid } = gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [0, 0.3], + [0, 0.9], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis.type === 'numeric' && grid.yAxis.edges).toEqual([ + 0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, + ]); + expect(grid.cells.slice(0, 10)).toEqual([1, 0, 0, 1, 0, 0, 0, 0, 0, 1]); + }); + + it('starts the buckets at a multiple of the bucket size', () => { + const edges = linearEdges([-12, 75]); + expect(edges[0]).toBe(-18); + expect(edges.at(-1)).toBe(81); + }); + + it('draws a constant value as one bucket sized by its magnitude', () => { + const { grid } = gridFromSamples({ + series: [series('a', [[0, 1]]), series('b', [[0, 1]])], + times, + scaleType: 'linear', + }); + + expect(grid.yAxis).toEqual({ + type: 'numeric', + scale: 'linear', + edges: [1, 1.1], + }); + expect(grid.cells).toEqual([2, 0, 0]); + }); + + it('keeps the edges of buckets smaller than 1e-6', () => { + const edges = linearEdges([1e-7, 5e-7, 9e-7]); + expect(edges[0]).toBe(8.5e-8); + expect(edges[1]).toBe(1.7e-7); + expect(edges.at(-1)).toBe(9.35e-7); + expect(linearEdges([3e-6, 3e-6])).toEqual([3e-6, 3.3e-6]); + }); + + it('buckets a range wider than 1e17', () => { + const edges = linearEdges([0, 9.2e18]); + expect(edges[0]).toBe(0); + expect(edges[1]).toBe(1e18); + expect(edges.at(-1)).toBe(1e19); + }); + + it('buckets a range near the largest finite magnitude', () => { + const edges = linearEdges([0, 1e300]); + expect(edges[1]).toBe(1e299); + expect(edges).toHaveLength(12); + }); + + it('places samples between column starts in the column containing them', () => { + const { grid } = gridFromSamples({ + series: [series('a', [[STEP + 1000, 1]])], + times, + scaleType: 'linear', + }); + + expect(grid.cells).toEqual([0, 1, 0]); + }); + + it('skips non-finite values and samples outside the time range', () => { + const { grid } = gridFromSamples({ + series: [ + series('a', [ + [0, NaN], + [0, Infinity], + [10 * STEP, 5], + [0, 1], + ]), + ], + times, + scaleType: 'linear', + }); + + expect(grid.cells).toEqual([1, 0, 0]); + }); + + it('is empty without samples', () => { + expect( + gridFromSamples({ series: [], times, scaleType: 'linear' }).grid, + ).toBe(EMPTY_HEATMAP_GRID); + }); + + describe('log scale', () => { + it('spaces buckets geometrically from the effective min to the max', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 1], + [0, 10], + [STEP, 100], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(1); + const edges = grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; + expect(grid.yAxis.type === 'numeric' && grid.yAxis.scale).toBe('log'); + expect(edges).toHaveLength(CALCULATED_TARGET_BUCKETS + 1); + expect(edges[0]).toBe(1); + expect(edges[CALCULATED_TARGET_BUCKETS / 2]).toBeCloseTo(10); + expect(edges[CALCULATED_TARGET_BUCKETS]).toBeCloseTo(100); + + const rows = CALCULATED_TARGET_BUCKETS; + expect(grid.cells[0]).toBe(1); + expect(grid.cells[rows / 2]).toBe(1); + expect(grid.cells[rows + rows - 1]).toBe(1); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(3); + }); + + it('clamps small values into the bottom bucket and drops non-positive ones', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, -5], + [0, 0], + [0, 1e-9], + [0, 1000], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBeCloseTo(0.1); + expect(grid.cells[0]).toBe(1); + expect(grid.cells[CALCULATED_TARGET_BUCKETS - 1]).toBe(1); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(2); + }); + + it('bounds the buckets by the positive samples only', () => { + const { effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + ...Array.from({ length: 50 }, (): [number, number] => [0, 0]), + [0, 10], + [0, 1000], + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(10); + }); + + it('draws a constant value in the top bucket', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [series('a', [[0, 1]]), series('b', [[0, 1]])], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBeCloseTo(1e-4); + const edges = grid.yAxis.type === 'numeric' ? grid.yAxis.edges : []; + expect(edges.at(-1)).toBeCloseTo(1); + expect(grid.cells[CALCULATED_TARGET_BUCKETS - 1]).toBe(2); + expect(grid.cells.reduce((a, b) => a + b, 0)).toBe(2); + }); + + it('starts at the smallest sample when the low quantile reaches the max', () => { + const { grid, effectiveMin } = gridFromSamples({ + series: [ + series('a', [ + [0, 0.5], + ...Array.from({ length: 100 }, (): [number, number] => [0, 1]), + ]), + ], + times, + scaleType: 'log', + }); + + expect(effectiveMin).toBe(0.5); + expect(grid.cells[0]).toBe(1); + expect(grid.cells[CALCULATED_TARGET_BUCKETS - 1]).toBe(100); + }); + + it('is empty when there is no positive range to bucket', () => { + expect( + gridFromSamples({ + series: [ + series('a', [ + [0, 0], + [0, -1], + ]), + ], + times, + scaleType: 'log', + }).grid, + ).toBe(EMPTY_HEATMAP_GRID); + }); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts index f3ca07687d..715f9c9680 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapGrid.test.ts @@ -150,15 +150,10 @@ describe('rowPlotBounds', () => { expect(hi).toEqual([Math.log(10), Math.log(1000)]); }); - it('draws an open top row as tall as the row below it', () => { - const { lo, hi } = rowPlotBounds({ - type: 'numeric', - scale: 'linear', - edges: [0, 1, 3, Infinity], - openTop: true, - }); - expect(lo).toEqual([0, 1, 3]); - expect(hi).toEqual([1, 3, 5]); + it('gives each histogram bucket a unit-height band, whatever its bound', () => { + expect( + rowPlotBounds({ type: 'buckets', bounds: [0.1, 10, Infinity] }), + ).toEqual({ lo: [0, 1, 2], hi: [1, 2, 3] }); }); it('gives each series row a unit-height band', () => { diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts new file mode 100644 index 0000000000..7d43a90a47 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapHistogram.test.ts @@ -0,0 +1,102 @@ +import { gridFromHistogramSamples } from '@/components/DBHeatmapChart/heatmapHistogram'; + +const times = [1000, 2000]; + +describe('gridFromHistogramSamples', () => { + it('sorts buckets by numeric bound, +Inf on top', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '+Inf', t: 1000, v: 10 }, + { le: '10', t: 1000, v: 8 }, + { le: '2.5', t: 1000, v: 3 }, + ], + times, + }); + expect(grid.yAxis).toEqual({ + type: 'buckets', + bounds: [2.5, 10, Infinity], + }); + }); + + it('de-accumulates each bucket from the one below it', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '0.1', t: 1000, v: 2 }, + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '0.1', t: 2000, v: 0 }, + { le: '1', t: 2000, v: 4 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.cells).toEqual([2, 3, 1, 0, 4, 0]); + }); + + it('merges buckets whose le is written differently', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '1.0', t: 2000, v: 4 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [1, Infinity] }); + expect(grid.cells).toEqual([5, 1, 4, 0]); + }); + + it('sums series at the same time whose le is written differently', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 2 }, + { le: '1.0', t: 1000, v: 3 }, + { le: '+Inf', t: 1000, v: 10 }, + ], + times: [1000], + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [1, Infinity] }); + expect(grid.cells).toEqual([5, 5]); + }); + + it('de-accumulates across a missing bucket sample from the next one down', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '0.1', t: 1000, v: 2 }, + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 6 }, + { le: '0.1', t: 2000, v: 1 }, + { le: '+Inf', t: 2000, v: 4 }, + ], + times, + }); + expect(grid.cells.slice(3)).toEqual([1, 0, 3]); + }); + + it('clamps a bucket below the one under it to 0', () => { + const grid = gridFromHistogramSamples({ + samples: [ + { le: '1', t: 1000, v: 5 }, + { le: '+Inf', t: 1000, v: 4.9 }, + ], + times, + }); + expect(grid.cells.slice(0, 2)).toEqual([5, 0]); + }); + + it('draws a single bucket', () => { + const grid = gridFromHistogramSamples({ + samples: [{ le: '+Inf', t: 2000, v: 7 }], + times, + }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [Infinity] }); + expect(grid.cells).toEqual([0, 7]); + }); + + it('is empty without samples', () => { + const grid = gridFromHistogramSamples({ samples: [], times }); + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [] }); + expect(grid.cells).toEqual([]); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts index 42e0903e9e..a2e676221f 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts @@ -76,8 +76,59 @@ describe('PromQL heatmaps', () => { legendTemplate: '{{job}}', }; - it('are always series heatmaps', () => { - expect(toHeatmapQuery(config)).toEqual({ mode: 'series', config }); + it('are calculated distribution heatmaps by default, on a linear scale', () => { + expect(toHeatmapQuery(config)).toEqual({ + mode: 'calculated', + config, + scaleType: 'linear', + }); + }); + + it('use the configured distribution scale', () => { + const logConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'distribution', scaleType: 'log' }, + }; + expect(toHeatmapQuery(logConfig)).toEqual({ + mode: 'calculated', + config: logConfig, + scaleType: 'log', + }); + }); + + it('are series heatmaps in series mode', () => { + const seriesModeConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'series' }, + }; + expect(toHeatmapQuery(seriesModeConfig)).toEqual({ + mode: 'series', + config: seriesModeConfig, + }); + }); + + it('are histogram heatmaps in histogram mode', () => { + const histogramConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'histogram' }, + }; + expect(toHeatmapQuery(histogramConfig)).toEqual({ + mode: 'histogram', + config: histogramConfig, + }); + }); + + it('query a histogram heatmap as a histogram, with the expression as written', () => { + const histogramConfig: PromqlConfigWithDateRange = { + ...config, + heatmap: { mode: 'histogram', scaleType: 'log' }, + }; + expect(buildHeatmapSeriesConfig(histogramConfig, '5 minute')).toMatchObject( + { + promqlExpression: config.promqlExpression, + heatmap: { mode: 'histogram' }, + }, + ); }); it('query at the heatmap granularity over a bucket-aligned range', () => { diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts new file mode 100644 index 0000000000..d5147ddbff --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts @@ -0,0 +1,123 @@ +import { + DisplayType, + PromqlConfigWithDateRange, +} from '@hyperdx/common-utils/dist/types'; +import { renderHook } from '@testing-library/react'; + +import { useCalculatedHeatmapData } from '@/components/DBHeatmapChart/useCalculatedHeatmapData'; +import { useSeriesHeatmapData } from '@/components/DBHeatmapChart/useSeriesHeatmapData'; + +const mockUseQueriedChartConfig = jest.fn(); +jest.mock('@/hooks/useChartConfig', () => ({ + useQueriedChartConfig: (...args: unknown[]) => + mockUseQueriedChartConfig(...args), +})); + +const HOUR = 60 * 60 * 1000; +const T0 = new Date('2026-07-06T00:00:00Z').getTime(); +const dateRange: [Date, Date] = [new Date(T0), new Date(T0 + 2 * HOUR)]; + +const config: PromqlConfigWithDateRange = { + configType: 'promql', + displayType: DisplayType.Heatmap, + promqlExpression: [{ expression: 'up' }], + connection: 'test-connection', + dateRange, + granularity: 'auto', +}; + +const meta = [ + { name: '__hdx_time_bucket', type: 'DateTime64(3)' }, + { name: 'value', type: 'Float64' }, + { name: 'series_name', type: 'String' }, +]; + +function mockResponse(data: Record[]) { + mockUseQueriedChartConfig.mockReturnValue({ + data: { meta, data }, + isLoading: false, + isPlaceholderData: false, + error: null, + }); +} + +const row = (t: number, value: number, series_name: string) => ({ + __hdx_time_bucket: new Date(t).toISOString(), + value, + series_name, +}); + +describe('useCalculatedHeatmapData', () => { + beforeEach(() => { + mockUseQueriedChartConfig.mockReset(); + }); + + it('counts the series samples in each time column and value bucket', () => { + mockResponse([row(T0, 0, 'a'), row(T0, 5, 'b'), row(T0 + HOUR, 95, 'a')]); + const { result } = renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + + const { grid, generatedTsBuckets } = result.current.view; + const t1 = generatedTsBuckets.map(d => d.getTime()).indexOf(T0 + HOUR); + expect(grid.yAxis).toMatchObject({ type: 'numeric', scale: 'linear' }); + const rows = 10; + expect(grid.cells.slice(0, rows)).toEqual([2, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + expect(grid.cells.slice(t1 * rows, t1 * rows + rows)).toEqual([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + ]); + }); + + it('counts every series', () => { + mockResponse( + Array.from({ length: 60 }, (_, i) => row(T0, 1, `series-${i}`)), + ); + const { result } = renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + + expect(result.current.view.grid.cells[0]).toBe(60); + expect(result.current.view.hiddenSeriesCount).toBe(0); + }); + + it('re-buckets a scale change without changing the query', () => { + mockResponse([row(T0, 1, 'a'), row(T0, 100, 'b')]); + const { result, rerender } = renderHook( + ({ scaleType }: { scaleType: 'linear' | 'log' }) => + useCalculatedHeatmapData({ + config: { ...config, heatmap: { scaleType } }, + scaleType, + enabled: true, + }), + { initialProps: { scaleType: 'linear' } }, + ); + const [linearQuery] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + rerender({ scaleType: 'log' }); + + const [logQuery] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + expect(logQuery).toEqual(linearQuery); + expect(result.current.view.grid.yAxis).toMatchObject({ + type: 'numeric', + scale: 'log', + }); + }); + + it('shares its query with a series heatmap of the same expression', () => { + mockResponse([]); + renderHook(() => + useCalculatedHeatmapData({ config, scaleType: 'linear', enabled: true }), + ); + const [, calculatedOptions] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + renderHook(() => + useSeriesHeatmapData({ + config: { ...config, heatmap: { mode: 'series' } }, + enabled: true, + }), + ); + const [, seriesOptions] = mockUseQueriedChartConfig.mock.calls.at(-1)!; + + expect(seriesOptions.queryKey).toEqual(calculatedOptions.queryKey); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts new file mode 100644 index 0000000000..51994a60a1 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts @@ -0,0 +1,76 @@ +import { + DisplayType, + PromqlConfigWithDateRange, +} from '@hyperdx/common-utils/dist/types'; +import { renderHook } from '@testing-library/react'; + +import { useHistogramHeatmapData } from '@/components/DBHeatmapChart/useHistogramHeatmapData'; + +const mockUseQueriedChartConfig = jest.fn(); +jest.mock('@/hooks/useChartConfig', () => ({ + useQueriedChartConfig: (...args: unknown[]) => + mockUseQueriedChartConfig(...args), +})); + +const HOUR = 60 * 60 * 1000; +const T0 = new Date('2026-07-06T00:00:00Z').getTime(); + +const config: PromqlConfigWithDateRange = { + configType: 'promql', + displayType: DisplayType.Heatmap, + promqlExpression: [{ expression: 'sum by (le) (rate(x_bucket[5m]))' }], + connection: 'test-connection', + dateRange: [new Date(T0), new Date(T0 + 2 * HOUR)], + granularity: '1 hour', + heatmap: { mode: 'histogram' }, +}; + +const row = (t: number, le: string, value: number) => ({ + __hdx_time_bucket: new Date(t).toISOString(), + le, + value, +}); + +describe('useHistogramHeatmapData', () => { + beforeEach(() => { + mockUseQueriedChartConfig.mockReset(); + }); + + it('draws a row per bucket from the queried rows', () => { + mockUseQueriedChartConfig.mockReturnValue({ + data: { + data: [ + row(T0, '0.5', 2), + row(T0, '+Inf', 3), + row(T0 + HOUR, '0.5', 1), + row(T0 + HOUR, '+Inf', 5), + ], + }, + isLoading: false, + isPlaceholderData: false, + error: null, + }); + const { result } = renderHook(() => + useHistogramHeatmapData({ config, enabled: true }), + ); + + const { grid } = result.current.view; + expect(grid.yAxis).toEqual({ type: 'buckets', bounds: [0.5, Infinity] }); + expect(grid.times.slice(0, 2)).toEqual([T0, T0 + HOUR]); + expect(grid.cells.slice(0, 4)).toEqual([2, 1, 1, 4]); + }); + + it('passes the query error through', () => { + const error = new Error('Several series share le="0.5"'); + mockUseQueriedChartConfig.mockReturnValue({ + data: undefined, + isLoading: false, + isPlaceholderData: false, + error, + }); + const { result } = renderHook(() => + useHistogramHeatmapData({ config, enabled: true }), + ); + expect(result.current.error).toBe(error); + }); +}); diff --git a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts index d9cd40f319..26e4549f76 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapAxis.ts @@ -11,6 +11,11 @@ export const HEATMAP_DURATION_NUMBER_FORMAT = { factor: 0.001, } as const satisfies NumberFormat; +const COMPACT_NUMBER_FORMAT = new Intl.NumberFormat('en-US', { + notation: 'compact', + compactDisplay: 'short', +}); + /** Format a plot-space y value as a y-axis / tooltip label. */ export function formatHeatmapTick( value: number, @@ -35,48 +40,116 @@ export function formatHeatmapTick( average: true, mantissa: Math.abs(actualValue) >= 1 ? 0 : 2, }) - : new Intl.NumberFormat('en-US', { - notation: 'compact', - compactDisplay: 'short', - }).format(actualValue); + : COMPACT_NUMBER_FORMAT.format(actualValue); } +const MIN_LOG_SPLITS = 3; + +/** Split mantissas within a power of 10, sparsest first. */ +const LOG_SPLIT_MANTISSAS = [ + [1], // 1, 10, 100... + [1, 3], // 1, 3, 10, 30, 100... + [1, 2, 5], // 1, 2, 5, 10, 20, 50, 100... + [1, 2, 3, 5], + [1, 2, 3, 4, 5, 6, 7, 8, 9], +]; + /** - * Log-scale y-axis splits at powers of 10 (0.01, 0.1, 1, 10, 100…) so labels - * are clean round numbers instead of arbitrary positions in log space. - * `yMin`/`yMax` and the returned splits are natural-log values. + * Log-scale y-axis splits at round numbers: powers of 10 (0.01, 0.1, 1, 10…) + * when the range spans enough decades, otherwise denser mantissas within each + * decade, and evenly spaced nice values when the range is narrower than one + * mantissa step. `yMin`/`yMax` and the returned splits are natural-log values. */ export function logScaleSplits(yMin: number, yMax: number): number[] { const realMin = Math.exp(yMin); const realMax = Math.exp(yMax); - const splits: number[] = []; - // Generate powers of 10 within range const startExp = Math.floor(Math.log10(realMin)); const endExp = Math.ceil(Math.log10(realMax)); - for (let e = startExp; e <= endExp; e++) { - const v = Math.pow(10, e); - const logV = Math.log(v); - if (logV >= yMin && logV <= yMax) { - splits.push(logV); - } - } - // If too few splits, add intermediate values (×3) - if (splits.length < 3) { + + // Try increasingly dense sets of mantissas until the minimum + // number of log splits (axis ticks) is reached. + for (const mantissas of LOG_SPLIT_MANTISSAS) { + const splits: number[] = []; for (let e = startExp; e <= endExp; e++) { - for (const mult of [1, 3]) { - const v = mult * Math.pow(10, e); - const logV = Math.log(v); - if (logV >= yMin && logV <= yMax) { - splits.push(logV); - } + for (const mult of mantissas) { + const logV = Math.log(mult * Math.pow(10, e)); + if (logV >= yMin && logV <= yMax) splits.push(logV); } } - // Deduplicate and sort - return [...new Set(splits)].sort((a, b) => a - b); + if (splits.length >= MIN_LOG_SPLITS) return splits; + } + + // Fallback to linear splits when the log-scale splits are insufficient. + const maxIncr = (realMax - realMin) / MIN_LOG_SPLITS; + const mag = Math.pow(10, Math.floor(Math.log10(maxIncr))); + const incr = [5, 2, 1].map(m => m * mag).find(i => i <= maxIncr) ?? mag; + const splits: number[] = []; + for (let k = Math.ceil(realMin / incr); k * incr <= realMax; k++) { + splits.push(Math.log(k * incr)); } return splits; } +/** + * Format a histogram bucket bound as an axis / tooltip label. Unlike + * `formatHeatmapTick`, this keeps fractional digits above 1, since adjacent + * bounds like 2.5 and 5 must not round to the same label. + */ +export function formatBucketBound( + bound: number, + numberFormat: NumberFormat | undefined, +) { + if (bound === Infinity) return '+Inf'; + if (numberFormat?.unit === 'ms' || numberFormat?.output === 'duration') { + return formatHeatmapTick(bound, 'linear', numberFormat); + } + return numberFormat + ? formatNumber(bound, { ...numberFormat, average: true }) + : COMPACT_NUMBER_FORMAT.format(bound); +} + +/** + * A bucket row's range: from the next bound down to its own, or everything + * above the last finite bound. The lowest bucket starts at 0 when its bound is + * positive, since observations like durations and sizes are non-negative; + * otherwise it has no lower bound. + */ +export function formatBucketRange( + bounds: number[], + row: number, + format: (bound: number) => string, +) { + const hi = bounds[row]; + if (row === 0) { + return hi > 0 && hi !== Infinity + ? `${format(0)} – ${format(hi)}` + : `≤ ${format(hi)}`; + } + const lo = bounds[row - 1]; + return hi === Infinity ? `> ${format(lo)}` : `${format(lo)} – ${format(hi)}`; +} + +const MIN_BUCKET_TICK_SPACING_PX = 20; + +/** + * Bucket-axis ticks at the top edge of rows, labeled with their bound. Thinned + * to fit the plot height, counting down from the top row so it always keeps a + * tick. + */ +export function bucketRowSplits( + rowCount: number, + plotHeightPx: number, +): number[] { + const maxTicks = Math.max( + 1, + Math.floor(plotHeightPx / MIN_BUCKET_TICK_SPACING_PX), + ); + const stride = Math.ceil(rowCount / maxTicks); + const splits: number[] = []; + for (let top = rowCount; top >= 1; top -= stride) splits.push(top); + return splits.reverse(); +} + const SERIES_LABEL_MAX_LENGTH = 24; /** Series-axis ticks, one centered on each row. */ @@ -93,14 +166,38 @@ function formatSeriesTick(labels: string[], value: number) { /** * uPlot y scale and y axis overrides for a heatmap's y axis. A series axis is - * pinned to its rows with one label centered on each; a numeric axis formats - * its ticks, placed at powers of 10 on a log scale. + * pinned to its rows with one label centered on each; a bucket axis is pinned + * to its rows with each row's bound labeling its top edge; a numeric axis + * formats its ticks, placed at powers of 10 on a log scale. */ -export function heatmapYAxisOptions( - seriesLabels: string[] | undefined, - scaleType: HeatmapScaleType, - tickFormatter: (value: number) => string, -): { scale?: uPlot.Scale; axis: Pick } { +export function heatmapYAxisOptions({ + seriesLabels, + bucketBounds, + scaleType, + tickFormatter, + boundFormatter, +}: { + seriesLabels: string[] | undefined; + bucketBounds: number[] | undefined; + scaleType: HeatmapScaleType; + tickFormatter: (value: number) => string; + boundFormatter: (bound: number) => string; +}): { scale?: uPlot.Scale; axis: Pick } { + if (bucketBounds != null) { + return { + scale: { auto: false, range: [0, Math.max(1, bucketBounds.length)] }, + axis: { + values: (_u: uPlot, vals: number[]) => + vals.map(v => boundFormatter(bucketBounds[v - 1])), + splits: (u: uPlot) => + bucketRowSplits( + bucketBounds.length, + u.bbox.height / (window.devicePixelRatio || 1), + ), + }, + }; + } + if (seriesLabels != null) { return { scale: { auto: false, range: [0, Math.max(1, seriesLabels.length)] }, diff --git a/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts b/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts new file mode 100644 index 0000000000..2ae686dc12 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/heatmapCalculate.ts @@ -0,0 +1,177 @@ +import { range, round } from 'lodash'; + +import { computeEffectiveMin, heatmapLowQuantile } from './heatmapBounds'; +import { + EMPTY_HEATMAP_GRID, + HeatmapGrid, + HeatmapScaleType, +} from './heatmapGrid'; +import { HeatmapSeries, makeTimeIndexer } from './heatmapSeriesGrid'; + +export const CALCULATED_TARGET_BUCKETS = 10; + +/** Decimal places required to represent multiples of the given nice y step */ +function yStepDecimals(yStep: number) { + return Math.max(0, 1 - Math.floor(Math.log10(yStep))); +} + +const ALL_MULTS = [ + 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5, 9, 9.5, +]; + +/** + * ...0.01, 0.015, 0.02, ..., 0.95, 1, 2, 3, ..., 9, 10, 15, 20, ... + * + * Covers the full range of numbers representable in JavaScript floats. + */ +const NICE_Y_STEPS = range(-16, 308).flatMap(exp => + ALL_MULTS.map(m => round(m * 10 ** exp, yStepDecimals(m * 10 ** exp))).filter( + yStep => exp < 0 || Number.isInteger(yStep), + ), +); + +/** The largest nice y step that splits `[min, max]` into at least `count` buckets. */ +function niceYStep(min: number, max: number, count: number) { + const approx = (max - min) / Math.max(count - 1, 1); + const idx = NICE_Y_STEPS.findIndex(yStep => yStep > approx); + if (idx === -1) return NICE_Y_STEPS[NICE_Y_STEPS.length - 1]; + return NICE_Y_STEPS[Math.max(idx - 1, 0)]; +} + +/** Index of the `yStep`-tall row holding `value`. */ +function yStepIndex(value: number, yStep: number) { + return Math.floor(round(value / yStep, 9)); +} + +/** + * Linear buckets: at least `CALCULATED_TARGET_BUCKETS` rows of a nice y step, + * aligned to multiples of the step so edges are round numbers. + */ +function linearBuckets(min: number, max: number) { + // A constant value has no range to split, so size its single row by the + // value's magnitude instead of the smallest step, which would be invisible. + const yStep = + max > min + ? niceYStep(min, max, CALCULATED_TARGET_BUCKETS) + : niceYStep(0, Math.abs(min) || 1, CALCULATED_TARGET_BUCKETS); + const decimals = yStepDecimals(yStep); + const minIdx = yStepIndex(min, yStep); + const rows = yStepIndex(max, yStep) - minIdx + 1; + const edges = Array.from({ length: rows + 1 }, (_, k) => + round((minIdx + k) * yStep, decimals), + ); + + return { edges, rowOf: (v: number) => yStepIndex(v, yStep) - minIdx }; +} + +/** + * Log buckets over positive samples, bounded like a builder distribution + * heatmap's: `CALCULATED_TARGET_BUCKETS` geometric rows from `effectiveMin` + * to the max, with smaller values in the bottom row. + */ +function logBuckets(values: Float64Array) { + const q = heatmapLowQuantile('log'); + const min = values[Math.floor(q * (values.length - 1))]; + const max = values[values.length - 1]; + let effectiveMin = computeEffectiveMin(min, max, 'log'); + if (!(max > effectiveMin)) { + // The low quantile reaches the max when nearly every sample has the same + // value; fall back to the smallest sample, then the range cap. + effectiveMin = computeEffectiveMin( + values[0] < max ? values[0] : 0, + max, + 'log', + ); + } + if (!(max > effectiveMin)) return undefined; + + const n = CALCULATED_TARGET_BUCKETS; + const logRange = Math.log(max / effectiveMin); + const edges = Array.from( + { length: n + 1 }, + (_, k) => effectiveMin * Math.pow(max / effectiveMin, k / n), + ); + + return { + edges, + rowOf: (v: number) => + Math.floor((n * Math.log(v / effectiveMin)) / logRange), + }; +} + +/** + * A distribution grid of every series' samples: each cell counts the samples + * in a time column whose values fall in a row's bucket. + */ +export function gridFromSamples({ + series, + times, + scaleType, +}: { + series: HeatmapSeries[]; + times: number[]; + scaleType: HeatmapScaleType; +}): { grid: HeatmapGrid; effectiveMin: number } { + const stepMs = times.length > 1 ? times[1] - times[0] : 0; + const timeIndex = makeTimeIndexer(times, stepMs); + + // Two passes over the points, one for the bounds and one to count, so no + // per-sample copy is held between them. + const forEachSample = (fn: (ti: number, v: number) => void) => { + for (const s of series) { + for (const { t, v } of s.points) { + const ti = timeIndex(t); + // Non-positive values have no log, so log buckets drop them + if (ti < 0 || !Number.isFinite(v) || (scaleType === 'log' && v <= 0)) { + continue; + } + fn(ti, v); + } + } + }; + + // Log bounds need a quantile, so they keep every value + const values = + scaleType === 'log' + ? new Float64Array(series.reduce((n, s) => n + s.points.length, 0)) + : undefined; + let count = 0; + let min = Infinity; + let max = -Infinity; + forEachSample((_, v) => { + if (values != null) values[count] = v; + count++; + min = Math.min(min, v); + max = Math.max(max, v); + }); + if (count === 0) { + return { grid: EMPTY_HEATMAP_GRID, effectiveMin: 0 }; + } + + const buckets = + values != null + ? logBuckets(values.subarray(0, count).sort()) + : linearBuckets(min, max); + if (buckets == null) { + return { grid: EMPTY_HEATMAP_GRID, effectiveMin: 0 }; + } + + const rows = buckets.edges.length - 1; + const cells = new Array(times.length * rows).fill(0); + forEachSample((ti, v) => { + // Log values below the effective min fall in the bottom row, and the max + // computes to one past the top row + const r = Math.min(rows - 1, Math.max(0, buckets.rowOf(v))); + cells[ti * rows + r]++; + }); + + return { + grid: { + times, + stepMs, + yAxis: { type: 'numeric', scale: scaleType, edges: buckets.edges }, + cells, + }, + effectiveMin: buckets.edges[0], + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts index 81ec3f139b..c1079471bd 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapGrid.ts @@ -8,12 +8,12 @@ export type HeatmapYAxis = scale: HeatmapScaleType; /** `rows + 1` ascending row boundaries in data space. Rows may be uneven. */ edges: number[]; - /** - * The top row has no upper bound (e.g. a `+Inf` histogram bucket), so - * its last edge is ignored and it is drawn as tall as the row below. - */ - openTop?: boolean; } + /** + * Pre-bucketed histogram rows, drawn evenly. One ascending upper bound per + * row; the top bound may be `Infinity` (a `+Inf` bucket). + */ + | { type: 'buckets'; bounds: number[] } | { type: 'series'; labels: string[] }; /** Source-agnostic heatmap data: one column per time bucket, one row per y bucket or series. */ @@ -28,8 +28,8 @@ export type HeatmapGrid = { }; /** - * Numeric-axis cells are counts, empty at 0. Series-axis cells are values - * that may be zero or negative, empty when NaN. + * Numeric- and bucket-axis cells are counts, empty at 0. Series-axis cells + * are values that may be zero or negative, empty when NaN. */ export type HeatmapCellKind = 'count' | 'value'; @@ -49,22 +49,27 @@ export const EMPTY_HEATMAP_GRID: HeatmapGrid = { }; export function heatmapRowCount(yAxis: HeatmapYAxis) { - return yAxis.type === 'series' - ? yAxis.labels.length - : Math.max(0, yAxis.edges.length - 1); + switch (yAxis.type) { + case 'series': + return yAxis.labels.length; + case 'buckets': + return yAxis.bounds.length; + case 'numeric': + return Math.max(0, yAxis.edges.length - 1); + } } /** - * Lower and upper bound of each row in plot space. The uPlot y scale is - * linear, so log-scale rows are plotted in natural-log space and tick labels - * are exponentiated back. + * Lower and upper bound of each row in plot space. Series and histogram + * bucket rows are unit-height bands. The uPlot y scale is linear, so log-scale + * rows are plotted in natural-log space and tick labels are exponentiated back. */ export function rowPlotBounds(yAxis: HeatmapYAxis): { lo: number[]; hi: number[]; } { const rows = heatmapRowCount(yAxis); - if (yAxis.type === 'series') { + if (yAxis.type !== 'numeric') { return { lo: Array.from({ length: rows }, (_, r) => r), hi: Array.from({ length: rows }, (_, r) => r + 1), @@ -74,11 +79,6 @@ export function rowPlotBounds(yAxis: HeatmapYAxis): { const toPlot = yAxis.scale === 'log' ? Math.log : (v: number) => v; const lo = yAxis.edges.slice(0, rows).map(toPlot); const hi = yAxis.edges.slice(1, rows + 1).map(toPlot); - if (yAxis.openTop && rows > 0) { - const last = rows - 1; - const height = rows > 1 ? hi[last - 1] - lo[last - 1] : 1; - hi[last] = lo[last] + height; - } return { lo, hi }; } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts b/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts new file mode 100644 index 0000000000..104b04b71f --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/heatmapHistogram.ts @@ -0,0 +1,60 @@ +import { parsePromqlLe } from '@hyperdx/common-utils/dist/core/promql'; + +import type { HeatmapGrid } from './heatmapGrid'; +import { makeTimeIndexer } from './heatmapSeriesGrid'; + +/** A sample of one histogram bucket: its `le` label, time (ms) and cumulative value. */ +export type HistogramSample = { le: string; t: number; v: number }; + +/** + * A grid with one row per `le` bucket, lowest bound at the bottom. Bucket + * samples are cumulative (each counts everything at or below its bound), so a + * cell is its bucket's value minus the next bucket down's. + */ +export function gridFromHistogramSamples({ + samples, + times, +}: { + samples: HistogramSample[]; + times: number[]; +}): HeatmapGrid { + const stepMs = times.length > 1 ? times[1] - times[0] : 0; + const timeIndex = makeTimeIndexer(times, stepMs); + + // Keyed by numeric bound, since the same bound can be written differently + // (Prometheus 3 normalizes le="1" to le="1.0"). Such series are summed. + const bounds = [...new Set(samples.map(s => parsePromqlLe(s.le)))] + .filter(bound => !Number.isNaN(bound)) + .sort((a, b) => a - b); + const rowOf = new Map(bounds.map((bound, r) => [bound, r])); + const rows = bounds.length; + + const cumulative = new Array(times.length * rows).fill(NaN); + for (const { le, t, v } of samples) { + const ti = timeIndex(t); + const r = rowOf.get(parsePromqlLe(le)); + if (ti < 0 || r == null || !Number.isFinite(v)) continue; + const i = ti * rows + r; + cumulative[i] = Number.isNaN(cumulative[i]) ? v : cumulative[i] + v; + } + + const cells = new Array(times.length * rows).fill(0); + for (let ti = 0; ti < times.length; ti++) { + let below = 0; + for (let r = 0; r < rows; r++) { + const cum = cumulative[ti * rows + r]; + if (Number.isNaN(cum)) continue; + // rate() and increase() extrapolate each bucket separately, so adjacent + // buckets can come out slightly non-monotonic. + cells[ti * rows + r] = Math.max(0, cum - below); + below = cum; + } + } + + return { + times, + stepMs, + yAxis: { type: 'buckets', bounds }, + cells, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts index 1a7a114e39..948df6920f 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapQueries.ts @@ -1,9 +1,6 @@ import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getAlignedDateRange } from '@hyperdx/common-utils/dist/core/utils'; -import { - isBuilderChartConfig, - isPromqlChartConfig, -} from '@hyperdx/common-utils/dist/guards'; +import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, @@ -46,9 +43,14 @@ export type HeatmapSeriesChartConfig = | BuilderChartConfigWithDateRange | PromqlConfigWithDateRange; +/** Linear, since PromQL samples often include zeros, which log scale drops. */ +export const PROMQL_DEFAULT_HEATMAP_SCALE_TYPE: HeatmapScaleType = 'linear'; + /** * What a heatmap queries, by mode. Distribution heatmaps bucket a value - * expression server-side; series heatmaps query one builder series or PromQL + * expression server-side; calculated heatmaps bucket the samples of a PromQL + * range query client-side; histogram heatmaps draw the `le` buckets of a + * PromQL histogram; series heatmaps query one builder series or PromQL * expression per time bucket and draw a row per series. */ export type HeatmapQuery = @@ -57,6 +59,15 @@ export type HeatmapQuery = config: HeatmapChartConfig; scaleType: HeatmapScaleType; } + | { + mode: 'calculated'; + config: PromqlConfigWithDateRange; + scaleType: HeatmapScaleType; + } + | { + mode: 'histogram'; + config: PromqlConfigWithDateRange; + } | { mode: 'series'; config: HeatmapSeriesChartConfig; @@ -84,12 +95,23 @@ export function resolveHeatmapGranularity({ ); } -/** PromQL heatmaps only support series mode. */ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { - if (isPromqlChartConfig(config) || getHeatmapMode(config) === 'series') { + const mode = getHeatmapMode(config); + if (mode === 'series') { return { mode: 'series', config }; } + if (isPromqlChartConfig(config)) { + if (mode === 'histogram') { + return { mode: 'histogram', config }; + } + return { + mode: 'calculated', + config, + scaleType: config.heatmap?.scaleType ?? PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, + }; + } + const firstSelect = Array.isArray(config.select) ? config.select[0] : undefined; @@ -113,22 +135,37 @@ export function toHeatmapQuery(config: HeatmapSeriesChartConfig): HeatmapQuery { } /** - * The time-chart query behind a series-mode heatmap, bucketed at the heatmap's - * granularity: `select[0]` and the group by for a builder config, or the first - * expression's range query for PromQL. + * The time-chart query behind a series-mode, calculated or histogram heatmap, + * bucketed at the heatmap's granularity: `select[0]` and the group by for a + * builder config, or the first expression's range query for PromQL. */ export function buildHeatmapSeriesConfig( config: HeatmapSeriesChartConfig, granularity: SQLInterval, ): ChartConfigWithDateRange { + // Heatmaps have no series limit control, a default is applied automatically. + // Heatmap settings only change how the response is drawn, so keep them out + // of the query's cache key: switching mode or scale reuses the data. + // Histogram mode is the exception, since its response is keyed by bucket. + if (isPromqlChartConfig(config)) { + return convertToTimeChartConfig({ + ...config, + granularity, + seriesLimit: undefined, + heatmap: + config.heatmap?.mode === 'histogram' + ? { mode: 'histogram' } + : undefined, + }); + } return convertToTimeChartConfig({ ...config, - ...(isBuilderChartConfig(config) && Array.isArray(config.select) + ...(Array.isArray(config.select) ? { select: config.select.slice(0, 1) } : {}), granularity, - // Heatmaps have no series limit control, a default is applied automatically seriesLimit: undefined, + heatmap: undefined, }); } diff --git a/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts b/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts index 5ddc233325..88775ca3b4 100644 --- a/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts +++ b/packages/app/src/components/DBHeatmapChart/heatmapSeriesGrid.ts @@ -10,7 +10,7 @@ const naturalCompare = new Intl.Collator(undefined, { numeric: true }).compare; * normally match `times` exactly; the arithmetic fallback covers timestamps * that land inside a column instead of on its start. */ -function makeTimeIndexer(times: number[], stepMs: number) { +export function makeTimeIndexer(times: number[], stepMs: number) { const indexByTime = new Map(times.map((t, i) => [t, i])); return (t: number) => { const exact = indexByTime.get(t); diff --git a/packages/app/src/components/DBHeatmapChart/index.ts b/packages/app/src/components/DBHeatmapChart/index.ts index f0b63103d7..112380d1b0 100644 --- a/packages/app/src/components/DBHeatmapChart/index.ts +++ b/packages/app/src/components/DBHeatmapChart/index.ts @@ -8,6 +8,7 @@ export { buildHeatmapBoundsConfig, buildHeatmapBucketConfig, buildHeatmapSeriesConfig, + PROMQL_DEFAULT_HEATMAP_SCALE_TYPE, resolveHeatmapGranularity, toHeatmapQuery, } from './heatmapQueries'; diff --git a/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts new file mode 100644 index 0000000000..3867f42bf2 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/useCalculatedHeatmapData.ts @@ -0,0 +1,53 @@ +import { useMemo } from 'react'; +import { PromqlConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; + +import { gridFromSamples } from './heatmapCalculate'; +import { HeatmapScaleType } from './heatmapGrid'; +import { HeatmapData, HeatmapView, useSettledView } from './useHeatmapView'; +import { useHeatmapSeriesPoints } from './useSeriesHeatmapData'; + +/** + * Query a calculated (PromQL distribution) heatmap: the same range query as a + * series heatmap, with every series' samples bucketed client-side into a + * value distribution per time column. + */ +export function useCalculatedHeatmapData({ + config, + scaleType, + enabled, +}: { + config: PromqlConfigWithDateRange; + scaleType: HeatmapScaleType; + enabled: boolean; +}): HeatmapData { + const { series, generatedTsBuckets, isLoading, isPlaceholderData, error } = + useHeatmapSeriesPoints({ + config, + enabled, + // Every series counts towards the distribution, so none are dropped. This is a + // client-side limit, so capping it wouldn't affect how much data is in memory. + maxSeries: Number.POSITIVE_INFINITY, + }); + + const currentView = useMemo(() => { + const { grid, effectiveMin } = gridFromSamples({ + series, + times: generatedTsBuckets.map(d => d.getTime()), + scaleType, + }); + return { + grid, + generatedTsBuckets, + scaleType, + effectiveMin, + hiddenSeriesCount: 0, + }; + }, [series, generatedTsBuckets, scaleType]); + + return { + view: useSettledView(currentView, isPlaceholderData), + isLoading, + isRefreshing: isPlaceholderData, + error, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/useContentStable.ts b/packages/app/src/components/DBHeatmapChart/useContentStable.ts new file mode 100644 index 0000000000..5c0921e801 --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/useContentStable.ts @@ -0,0 +1,13 @@ +import { useState } from 'react'; +import { isEqual } from 'lodash'; + +/** + * `value`, keeping the previous reference while its content is unchanged, so + * a caller that rebuilds equal objects every render doesn't invalidate memos. + */ +export function useContentStable(value: T): T { + const [stable, setStable] = useState(value); + const isSame = isEqual(stable, value); + if (!isSame) setStable(value); + return isSame ? stable : value; +} diff --git a/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts new file mode 100644 index 0000000000..7baed326ac --- /dev/null +++ b/packages/app/src/components/DBHeatmapChart/useHistogramHeatmapData.ts @@ -0,0 +1,46 @@ +import { useMemo } from 'react'; +import { PromqlConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; + +import { gridFromHistogramSamples, HistogramSample } from './heatmapHistogram'; +import { HeatmapData, HeatmapView, useSettledView } from './useHeatmapView'; +import { useHeatmapSeriesQuery } from './useSeriesHeatmapData'; + +/** + * Query a histogram heatmap: a PromQL range query over histogram buckets, + * drawn as one row per `le` bucket. + */ +export function useHistogramHeatmapData({ + config, + enabled, +}: { + config: PromqlConfigWithDateRange; + enabled: boolean; +}): HeatmapData { + const { generatedTsBuckets, data, isLoading, isPlaceholderData, error } = + useHeatmapSeriesQuery({ config, enabled }); + + const currentView = useMemo(() => { + const samples: HistogramSample[] = (data?.data ?? []).map(row => ({ + le: String(row.le), + t: new Date(row.__hdx_time_bucket).getTime(), + v: Number(row.value), + })); + return { + grid: gridFromHistogramSamples({ + samples, + times: generatedTsBuckets.map(d => d.getTime()), + }), + generatedTsBuckets, + scaleType: 'linear', + effectiveMin: 0, + hiddenSeriesCount: 0, + }; + }, [data, generatedTsBuckets]); + + return { + view: useSettledView(currentView, isPlaceholderData), + isLoading, + isRefreshing: isPlaceholderData, + error, + }; +} diff --git a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts index f418f88d26..097ab6a0b3 100644 --- a/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts +++ b/packages/app/src/components/DBHeatmapChart/useSeriesHeatmapData.ts @@ -20,18 +20,19 @@ const HEATMAP_MAX_SERIES = 50; /** * Group time-chart rows into one point list per series, keeping the - * `HEATMAP_MAX_SERIES` series with the largest peaks. + * `maxSeries` series with the largest peaks. */ function responseToHeatmapSeries( response: ResponseJSON>, dateRange: [Date, Date], + maxSeries: number, ): { series: HeatmapSeries[]; hiddenSeriesCount: number } { const { graphResults, timestampColumn, lineData, hiddenSeriesCount } = formatResponseForTimeChart({ currentPeriodResponse: response, dateRange, generateEmptyBuckets: false, - maxSeries: HEATMAP_MAX_SERIES, + maxSeries, }); const series = lineData.map(line => ({ @@ -50,29 +51,51 @@ function responseToHeatmapSeries( } /** - * Query a series-mode heatmap. Queried like a time chart, formatting the - * response into a heatmap grid with series on the y axis, time on the - * x-axis, and value determining the cell value. + * Query a heatmap's series like a time chart, bucketed at the heatmap + * granularity. The previous response stays in place while refreshing. */ -export function useSeriesHeatmapData({ +export function useHeatmapSeriesQuery({ config, enabled, }: { config: HeatmapSeriesChartConfig; enabled: boolean; -}): HeatmapData { - const { granularity, generatedTsBuckets, fromMs, toMs } = - useHeatmapTimeBuckets(config); - - const seriesConfig = buildHeatmapSeriesConfig(config, granularity); - const { data, isLoading, isPlaceholderData, error } = useQueriedChartConfig( - seriesConfig, - { - queryKey: ['heatmap_series', seriesConfig], - enabled, - placeholderData: prev => prev, - }, +}) { + const timeBuckets = useHeatmapTimeBuckets(config); + const seriesConfig = buildHeatmapSeriesConfig( + config, + timeBuckets.granularity, ); + const query = useQueriedChartConfig(seriesConfig, { + queryKey: ['heatmap_series', seriesConfig], + enabled, + placeholderData: prev => prev, + }); + return { ...timeBuckets, ...query }; +} + +/** + * Query a heatmap's series as one point list per series. Keeps the + * `maxSeries` series with the largest peaks. + */ +export function useHeatmapSeriesPoints({ + config, + enabled, + maxSeries, +}: { + config: HeatmapSeriesChartConfig; + enabled: boolean; + maxSeries: number; +}) { + const { + generatedTsBuckets, + fromMs, + toMs, + data, + isLoading, + isPlaceholderData, + error, + } = useHeatmapSeriesQuery({ config, enabled }); // Depend on primitive number ms values so a fresh dateRange array doesn't // re-parse the response and rebuild the grid on every render. @@ -83,7 +106,11 @@ export function useSeriesHeatmapData({ }>(() => { if (data == null) return { series: [], hiddenSeriesCount: 0 }; try { - return responseToHeatmapSeries(data, [new Date(fromMs), new Date(toMs)]); + return responseToHeatmapSeries( + data, + [new Date(fromMs), new Date(toMs)], + maxSeries, + ); } catch (e) { return { series: [], @@ -91,26 +118,61 @@ export function useSeriesHeatmapData({ error: e instanceof Error ? e : new Error(String(e)), }; } - }, [data, fromMs, toMs]); + }, [data, fromMs, toMs, maxSeries]); + + return { + series: parsed.series, + hiddenSeriesCount: parsed.hiddenSeriesCount, + generatedTsBuckets, + isLoading, + isPlaceholderData, + error: error ?? parsed.error, + }; +} + +/** + * Query a series-mode heatmap. Queried like a time chart, formatting the + * response into a heatmap grid with series on the y axis, time on the + * x-axis, and value determining the cell value. + */ +export function useSeriesHeatmapData({ + config, + enabled, +}: { + config: HeatmapSeriesChartConfig; + enabled: boolean; +}): HeatmapData { + const { + series, + hiddenSeriesCount, + generatedTsBuckets, + isLoading, + isPlaceholderData, + error, + } = useHeatmapSeriesPoints({ + config, + enabled, + maxSeries: HEATMAP_MAX_SERIES, + }); const currentView = useMemo( () => ({ grid: gridFromSeries( - parsed.series, + series, generatedTsBuckets.map(d => d.getTime()), ), generatedTsBuckets, scaleType: 'linear', effectiveMin: 0, - hiddenSeriesCount: parsed.hiddenSeriesCount, + hiddenSeriesCount, }), - [parsed, generatedTsBuckets], + [series, hiddenSeriesCount, generatedTsBuckets], ); return { view: useSettledView(currentView, isPlaceholderData), isLoading, isRefreshing: isPlaceholderData, - error: error ?? parsed.error, + error, }; } diff --git a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx index 6bcd400664..3056f4d386 100644 --- a/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx +++ b/packages/app/src/components/__tests__/ChartDisplaySettingsDrawer.test.tsx @@ -76,6 +76,30 @@ describe('ChartDisplaySettingsDrawer', () => { heatmapScaleType: 'linear', }); }); + + it.each(['linear', 'log'] as const)( + 'selects the %s default scale when the tile has none', + async scale => { + const onChange = jest.fn(); + const user = userEvent.setup(); + renderWithMantine( + , + ); + + await user.click(screen.getByRole('button', { name: /apply/i })); + + expect(onChange.mock.calls[0][0]).toMatchObject({ + heatmapScaleType: scale, + }); + }, + ); }); describe('color picker section', () => { @@ -754,6 +778,27 @@ describe('ChartDisplaySettingsDrawer', () => { ).not.toBeInTheDocument(); }); + it.each([ + ['hidden', 'distribution', false], + ['hidden', 'histogram', false], + ['offered', 'series', true], + ] as const)( + 'is %s on a PromQL %s heatmap', + (_label, heatmapMode, isShown) => { + renderWithMantine( + , + ); + + expect(screen.queryByTestId('legend-template-input') != null).toBe( + isShown, + ); + }, + ); + it('blocks Apply when the template exceeds the persisted length cap', async () => { const onChange = jest.fn(); const onClose = jest.fn(); diff --git a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts index a2b057d4e3..3689762d53 100644 --- a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts +++ b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts @@ -1,7 +1,11 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { prometheusApi } from '@/api'; -import { queryPromqlChartConfig, toTableRows } from '@/utils/promqlChartQuery'; +import { + queryPromqlChartConfig, + toHistogramRows, + toTableRows, +} from '@/utils/promqlChartQuery'; // The real module is a Mantine/TanStack table component; only the row cap is // needed here. @@ -245,6 +249,53 @@ describe('toTableRows', () => { }); }); +describe('toHistogramRows', () => { + const bucket = (metric: Record, value: number) => ({ + metric, + values: [[1700000000, value]] as [number, number][], + }); + const rangeResult = (...result: ReturnType[]) => ({ + expression: 'sum by (le) (rate(x_bucket[5m]))', + isBucketed: true, + result, + }); + + it('keys each sample by its le label', () => { + const result = toHistogramRows([ + rangeResult(bucket({ le: '0.5' }, 2), bucket({ le: '+Inf' }, 3)), + ]); + expect(result.data).toEqual([ + { __hdx_time_bucket: '2023-11-14T22:13:20.000Z', le: '0.5', value: 2 }, + { __hdx_time_bucket: '2023-11-14T22:13:20.000Z', le: '+Inf', value: 3 }, + ]); + expect(result.meta?.map(({ name }) => name)).toEqual([ + '__hdx_time_bucket', + 'le', + 'value', + ]); + }); + + it.each>([{}, { le: '' }, { le: 'abc' }])( + 'rejects a series without a numeric le: %p', + metric => { + expect(() => toHistogramRows([rangeResult(bucket(metric, 1))])).toThrow( + /need series with an "le" label/, + ); + }, + ); + + it('rejects buckets that were not aggregated by le', () => { + expect(() => + toHistogramRows([ + rangeResult( + bucket({ le: '0.5', instance: 'a' }, 1), + bucket({ le: '0.5', instance: 'b' }, 1), + ), + ]), + ).toThrow('Several series share le="0.5"'); + }); +}); + describe('queryPromqlChartConfig', () => { const dateRange: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), @@ -372,4 +423,43 @@ describe('queryPromqlChartConfig', () => { }, ]); }); + + it('keys a histogram heatmap rows by bucket, not series name', async () => { + jest + .mocked(prometheusApi.queryRange) + .mockReset() + .mockResolvedValue({ + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { le: '+Inf' }, + values: [[dateRange[0].getTime() / 1000, '4']], + }, + ], + }, + }); + + const result = await queryPromqlChartConfig( + { + configType: 'promql', + displayType: DisplayType.Heatmap, + connection: 'conn', + legendTemplate: '{{job}}', + promqlExpression: [{ expression: 'sum by (le) (x_bucket)' }], + heatmap: { mode: 'histogram' }, + }, + dateRange, + new AbortController().signal, + {}, + ); + + expect(prometheusApi.queryRange).toHaveBeenCalledWith( + expect.objectContaining({ query: 'sum by (le) (x_bucket)' }), + ); + expect(result.data).toEqual([ + { __hdx_time_bucket: dateRange[0].toISOString(), le: '+Inf', value: 4 }, + ]); + }); }); diff --git a/packages/app/src/utils/promqlChartQuery.ts b/packages/app/src/utils/promqlChartQuery.ts index ccb912d55f..8d77dde0ce 100644 --- a/packages/app/src/utils/promqlChartQuery.ts +++ b/packages/app/src/utils/promqlChartQuery.ts @@ -2,6 +2,7 @@ import type { QueryAttribution } from '@hyperdx/common-utils/dist/clickhouse'; import { displayTypeSupportsInstantQuery, getQueriedPromqlSeries, + parsePromqlLe, promqlSeriesQueryType, promqlStep, reducePromqlSamples, @@ -262,6 +263,51 @@ export function toTableRows( }; } +/** + * Rows for a histogram heatmap: one per sample, grouped by the series' `le` + * bucket. Every series must be a distinct bucket (le). + */ +export function toHistogramRows( + expressions: ExpressionResult[], +): ChartQueryResult { + const seenLe = new Set(); + const data: Record[] = []; + for (const { result } of expressions) { + for (const { metric, values } of result) { + const le = metric.le; + if (Number.isNaN(parsePromqlLe(le))) { + throw new Error( + 'Histogram heatmaps need series with an "le" label, e.g. sum by (le) (rate(x_bucket[$__rate_interval])).', + ); + } + if (seenLe.has(le)) { + throw new Error( + `Several series share le="${le}". Aggregate them by bucket with sum by (le) (...).`, + ); + } + seenLe.add(le); + for (const [ts, value] of values) { + data.push({ + __hdx_time_bucket: new Date(ts * 1000).toISOString(), + le, + value, + }); + } + } + } + + return { + data, + meta: [ + { name: '__hdx_time_bucket', type: 'DateTime64(3)' }, + { name: 'le', type: 'String' }, + { name: 'value', type: 'Float64' }, + ], + rows: data.length, + isComplete: true, + }; +} + /** * Run a PromQL tile's expressions and shape the result like a ClickHouse * response, so the chart formatters treat it like every other source. @@ -300,6 +346,12 @@ export async function queryPromqlChartConfig( ), ); + // Rows keyed by the series' `le` bucket. Done here so that + // alias and legend templates don't interfere with the le label. + if (substituted.heatmap?.mode === 'histogram') { + return toHistogramRows(results); + } + return substituted.displayType === DisplayType.Table ? toTableRows(results) : toChartRows(results, substituted.legendTemplate?.trim() || undefined); diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index 9fd6b5ad8b..bab0b7196f 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -13,6 +13,7 @@ import { import { switchWhereToLucene } from '../utils/lucene-autocomplete'; import { addTagViaPicker, removeTagViaPicker } from '../utils/tags'; +import { HeatmapComponent } from './HeatmapComponent'; import { WebhookAlertModalComponent } from './WebhookAlertModalComponent'; import { WhereInputComponent } from './WhereInputComponent'; @@ -73,8 +74,8 @@ export class ChartEditorComponent { await this.chartTypeInput.getByRole('tab', { name: tabName }).click(); } - /** Pick the builder heatmap's mode in its segmented control. */ - async setHeatmapMode(mode: 'Distribution' | 'Series') { + /** Pick the heatmap's mode in its segmented control. */ + async setHeatmapMode(mode: 'Distribution' | 'Series' | 'Histogram') { await this.editorForm() .getByTestId('heatmap-mode-control') .locator('.mantine-SegmentedControl-label') @@ -99,8 +100,18 @@ export class ChartEditorComponent { .locator('.cm-content'); } + /** The heatmap the editor previews. */ + get previewHeatmap(): HeatmapComponent { + return new HeatmapComponent(this.page, this.editorForm()); + } + + /** The error the preview shows in place of the chart. */ + get previewError(): Locator { + return this.editorForm().getByTestId('chart-error-state'); + } + /** A heatmap mode's (visually hidden) radio, for checking the selection. */ - heatmapModeOption(mode: 'Distribution' | 'Series'): Locator { + heatmapModeOption(mode: 'Distribution' | 'Series' | 'Histogram'): Locator { return this.editorForm() .getByTestId('heatmap-mode-control') .getByRole('radio', { name: mode, exact: true }); diff --git a/packages/app/tests/e2e/components/HeatmapComponent.ts b/packages/app/tests/e2e/components/HeatmapComponent.ts index 6d38251270..4bbb267842 100644 --- a/packages/app/tests/e2e/components/HeatmapComponent.ts +++ b/packages/app/tests/e2e/components/HeatmapComponent.ts @@ -77,6 +77,15 @@ export class HeatmapComponent { return ((await row.textContent()) ?? '').replace(/^Series:\s*/, '').trim(); } + /** A labelled value of the open cell tooltip, e.g. `Count Value`. */ + async hoveredTooltipValue(label: 'Count Value' | 'Y Value' | 'Bucket') { + const row = this.container + .getByText(`${label}:`, { exact: true }) + .locator('..'); + await expect(row).toBeVisible(); + return ((await row.textContent()) ?? '').replace(`${label}:`, '').trim(); + } + /** * Hover the top series-axis label, which the canvas draws, so its DOM * tooltip opens. Returns that tooltip. diff --git a/packages/app/tests/e2e/features/promql-heatmap.spec.ts b/packages/app/tests/e2e/features/promql-heatmap.spec.ts index 9d2db47213..7424099e68 100644 --- a/packages/app/tests/e2e/features/promql-heatmap.spec.ts +++ b/packages/app/tests/e2e/features/promql-heatmap.spec.ts @@ -1,9 +1,11 @@ /** - * PromQL heatmaps are series heatmaps: one row per series of a range query, - * named by the legend template. There is no distribution mode to pick. + * PromQL heatmaps draw a distribution of every series' samples, bucketed + * client-side; one row per series of a range query, named by the legend + * template; or one row per bucket of a Prometheus histogram. * * The seed gives `e2e_service_up` one series per `SERVICES` entry, labelled - * `service`, one sample of 1 a minute. + * `service`, one sample of 1 a minute, and `e2e_request_duration_seconds` a + * cumulative `_bucket` series per service and `le` in 0.1, 0.5, 1 and +Inf. */ import { DisplayType } from '@hyperdx/common-utils/dist/types'; @@ -11,10 +13,48 @@ import { HeatmapComponent } from '../components/HeatmapComponent'; import { DashboardPage } from '../page-objects/DashboardPage'; import { SERVICES } from '../seed-clickhouse'; import { expect, test } from '../utils/base-test'; -import { E2E_PROMQL_METRIC_NAME, PROMQL_SOURCE_NAME } from '../utils/constants'; +import { + E2E_PROMQL_HISTOGRAM_METRIC_NAME, + E2E_PROMQL_METRIC_NAME, + PROMQL_SOURCE_NAME, +} from '../utils/constants'; // Ends in `)` rather than inside the selector so the autocomplete popup closes. const EXPRESSION = `last_over_time(${E2E_PROMQL_METRIC_NAME}{service!=""}[$__interval])`; +// A fixed window holds several of the once-a-minute samples rate() needs. +const BUCKET_RATE = `rate(${E2E_PROMQL_HISTOGRAM_METRIC_NAME}_bucket[5m])`; +const HISTOGRAM_EXPRESSION = `sum by (le) (${BUCKET_RATE})`; + +/** Open a new tile on a new dashboard as a PromQL heatmap of `expression`. */ +async function startPromqlHeatmapTile( + dashboardPage: DashboardPage, + expression = EXPRESSION, +) { + const editor = dashboardPage.chartEditor; + await dashboardPage.goto(); + await dashboardPage.createNewDashboard(); + await dashboardPage.addTile(); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setChartType(DisplayType.Heatmap); + await editor.switchToPromqlMode(); + await editor.selectSource(PROMQL_SOURCE_NAME); + await editor.setChartName('PromQL heatmap tile'); + await editor.replacePromqlExpression(expression); +} + +/** Save the tile, and check it renders before and after a reload. */ +async function saveAndAssertRenders( + dashboardPage: DashboardPage, + assertRenders: (heatmap: HeatmapComponent) => Promise, +) { + await dashboardPage.chartEditor.save(); + await expect(dashboardPage.getTiles()).toHaveCount(1, { timeout: 10000 }); + await expect(dashboardPage.getTileError()).toHaveCount(0); + await assertRenders(dashboardPage.getTileHeatmap()); + await dashboardPage.reload(); + await assertRenders(dashboardPage.getTileHeatmap()); +} test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { test('draws a row per series, and persists', async ({ page }) => { @@ -35,22 +75,13 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }; await test.step('Create a PromQL heatmap tile', async () => { - await dashboardPage.goto(); - await dashboardPage.createNewDashboard(); - await dashboardPage.addTile(); - await expect(editor.nameInput).toBeVisible(); - await editor.waitForDataToLoad(); - await editor.setChartType(DisplayType.Heatmap); - await editor.switchToPromqlMode(); - await editor.selectSource(PROMQL_SOURCE_NAME); - await editor.setChartName('PromQL heatmap tile'); - await editor.replacePromqlExpression(EXPRESSION); + await startPromqlHeatmapTile(dashboardPage); + await editor.setHeatmapMode('Series'); await editor.setLegendTemplate('{{service}}'); await editor.runQuery(false); }); - await test.step('The editor offers no heatmap mode or instant query', async () => { - await expect(page.getByTestId('heatmap-mode-control')).toHaveCount(0); + await test.step('The editor offers no instant query', async () => { await expect(page.getByTestId('promql-query-type-control-0')).toHaveCount( 0, ); @@ -58,9 +89,7 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }); await test.step('The preview draws a row per series', async () => { - await assertRendersServices( - new HeatmapComponent(page, page.getByTestId('tile-editor-form')), - ); + await assertRendersServices(editor.previewHeatmap); await editor.openGeneratedPromql(); await expect .poll(() => editor.getGeneratedPromqlText()) @@ -68,14 +97,119 @@ test.describe('PromQL heatmaps', { tag: ['@dashboard', '@full-stack'] }, () => { }); await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersServices); + }); + }); + + test('draws a distribution of the samples, and persists', async ({ + page, + }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + // Every service reports 1, so a populated cell counts at most one sample + // per service, in a row around 1. + const assertRendersDistribution = async (heatmap: HeatmapComponent) => { + await expect(heatmap.canvas.first()).toBeVisible(); + await expect(heatmap.notEnoughDataText).toHaveCount(0); + await heatmap.hoverPopulatedCell(); + const count = Number(await heatmap.hoveredTooltipValue('Count Value')); + expect(count).toBeGreaterThan(0); + expect(count).toBeLessThanOrEqual(SERVICES.length); + expect(await heatmap.hoveredTooltipValue('Y Value')).toMatch( + /^1(\.\d+)?\b/, + ); + }; + + await test.step('Create a PromQL heatmap tile', async () => { + await startPromqlHeatmapTile(dashboardPage); + await editor.runQuery(false); + }); + + await test.step('Distribution is the default mode, on a linear scale', async () => { + await expect(editor.heatmapModeOption('Distribution')).toBeChecked(); + await editor.openDisplaySettings(); + await expect(editor.heatmapScaleOption('Linear')).toBeChecked(); + await editor.applyDisplaySettings(); + }); + + await test.step('The preview draws the distribution', async () => { + await assertRendersDistribution(editor.previewHeatmap); + }); + + await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersDistribution); + }); + + await test.step('The chosen scale persists', async () => { + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setHeatmapScale('Log'); await editor.save(); - await expect(dashboardPage.getTiles()).toHaveCount(1, { - timeout: 10000, - }); - await expect(dashboardPage.getTileError()).toHaveCount(0); - await assertRendersServices(dashboardPage.getTileHeatmap()); await dashboardPage.reload(); - await assertRendersServices(dashboardPage.getTileHeatmap()); + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await editor.openDisplaySettings(); + await expect(editor.heatmapScaleOption('Log')).toBeChecked(); + }); + }); + + test('draws histogram buckets, and persists', async ({ page }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + const assertRendersBuckets = async (heatmap: HeatmapComponent) => { + await expect(heatmap.canvas.first()).toBeVisible({ timeout: 30000 }); + await expect(heatmap.notEnoughDataText).toHaveCount(0); + await heatmap.hoverPopulatedCell(); + expect( + Number(await heatmap.hoveredTooltipValue('Count Value')), + ).toBeGreaterThan(0); + expect(await heatmap.hoveredTooltipValue('Bucket')).toMatch( + /^(0 – 0\.1|0\.1 – 0\.5|0\.5 – 1|> 1)\b/, + ); + }; + + await test.step('Create a PromQL histogram heatmap tile', async () => { + await startPromqlHeatmapTile(dashboardPage, HISTOGRAM_EXPRESSION); + await editor.setHeatmapMode('Histogram'); + await editor.runQuery(false); }); + + await test.step('The expression is queried as written', async () => { + await editor.openGeneratedPromql(); + await expect + .poll(() => editor.getGeneratedPromqlText()) + .toContain(HISTOGRAM_EXPRESSION); + }); + + await test.step('The preview draws the buckets', async () => { + await assertRendersBuckets(editor.previewHeatmap); + }); + + await test.step('The saved tile renders, and still renders after reload', async () => { + await saveAndAssertRenders(dashboardPage, assertRendersBuckets); + }); + + await test.step('Histogram mode persists', async () => { + await dashboardPage.editTile(0); + await expect(editor.nameInput).toBeVisible(); + await expect(editor.heatmapModeOption('Histogram')).toBeChecked(); + }); + }); + + test('explains buckets that were not aggregated by le', async ({ page }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + + await startPromqlHeatmapTile(dashboardPage, BUCKET_RATE); + await editor.setHeatmapMode('Histogram'); + await editor.runQuery(false); + + await expect(editor.previewError).toContainText( + 'Several series share le=', + { timeout: 30000 }, + ); }); }); diff --git a/packages/app/tests/e2e/seed-clickhouse.ts b/packages/app/tests/e2e/seed-clickhouse.ts index 98c537dc6f..9736ffdaea 100644 --- a/packages/app/tests/e2e/seed-clickhouse.ts +++ b/packages/app/tests/e2e/seed-clickhouse.ts @@ -22,6 +22,7 @@ import { E2E_METADATA_MV_LOGS_TABLE, E2E_METRICS_GAUGE_TABLE, E2E_METRICS_SUM_TABLE, + E2E_PROMQL_HISTOGRAM_METRIC_NAME, E2E_PROMQL_METRIC_NAME, E2E_PROMQL_TABLE, E2E_SESSIONS_TABLE, @@ -1101,35 +1102,51 @@ export async function seedClickHouse(): Promise { await seedPromqlSeries(client, startMs, endMs); console.log(` Inserted ${SERVICES.length} PromQL series`); + await seedPromqlHistogram(client, startMs, endMs); + console.log(` Inserted ${SERVICES.length} PromQL histograms`); + console.log('ClickHouse seeding complete'); } /** - * Seed one `E2E_PROMQL_METRIC_NAME` series per service, one sample a minute - * across the window. Written through the TimeSeries table itself, not its inner - * tables: the engine derives each series' `id` and splits samples across inner - * tables in ways that change between ClickHouse versions, and rows written - * around it are invisible to the PromQL engine. + * Insert a metric family and its series into the PromQL table, one sample a + * minute across the window. `valueSql` computes the nth sample's value. Written + * through the TimeSeries table itself, not its inner tables: the engine derives + * each series' `id` and splits samples across inner tables in ways that change + * between ClickHouse versions, and rows written around it are invisible to the + * PromQL engine. */ -async function seedPromqlSeries( +async function insertPromqlMetric( client: ReturnType, startMs: number, endMs: number, + family: { name: string; type: string; unit: string; help: string }, + series: { + metricName: string; + tags: Record; + valueSql: string; + }[], ) { const table = `${E2E_CLICKHOUSE_DATABASE}.${E2E_PROMQL_TABLE}`; await client.query(` INSERT INTO ${table} (metric_family, type, unit, help) - VALUES ('${E2E_PROMQL_METRIC_NAME}', 'gauge', '', 'E2E service liveness') + VALUES ('${family.name}', '${family.type}', '${family.unit}', '${family.help}') `); const stepMs = 60000; const sampleCount = Math.max(1, Math.floor((endMs - startMs) / stepMs)); - const rows = SERVICES.map( - service => - `SELECT '${E2E_PROMQL_METRIC_NAME}', map('service', '${service}'), ` + - `arrayMap(n -> (fromUnixTimestamp64Milli(toInt64(${startMs} + n * ${stepMs})), 1.0), range(${sampleCount}))`, - ).join('\n UNION ALL\n '); + const rows = series + .map(({ metricName, tags, valueSql }) => { + const tagArgs = Object.entries(tags) + .flatMap(([key, value]) => [`'${key}'`, `'${value}'`]) + .join(', '); + return ( + `SELECT '${metricName}', map(${tagArgs}), ` + + `arrayMap(n -> (fromUnixTimestamp64Milli(toInt64(${startMs} + n * ${stepMs})), ${valueSql}), range(${sampleCount}))` + ); + }) + .join('\n UNION ALL\n '); await client.query(` INSERT INTO ${table} (metric_name, tags, time_series) @@ -1137,6 +1154,72 @@ async function seedPromqlSeries( `); } +/** Seed one `E2E_PROMQL_METRIC_NAME` series per service, always 1. */ +async function seedPromqlSeries( + client: ReturnType, + startMs: number, + endMs: number, +) { + await insertPromqlMetric( + client, + startMs, + endMs, + { + name: E2E_PROMQL_METRIC_NAME, + type: 'gauge', + unit: '', + help: 'E2E service liveness', + }, + SERVICES.map(service => ({ + metricName: E2E_PROMQL_METRIC_NAME, + tags: { service }, + valueSql: '1.0', + })), + ); +} + +/** + * Each bucket's per-minute increase, keyed by its `le` label. Cumulative, so a + * minute adds 1 observation <= 0.1, 2 in (0.1, 0.5], 1 in (0.5, 1] and 1 above. + */ +const PROMQL_HISTOGRAM_BUCKET_INCREASES: Record = { + '0.1': 1, + '0.5': 3, + '1': 4, + '+Inf': 5, +}; + +/** + * Seed a classic histogram, `E2E_PROMQL_HISTOGRAM_METRIC_NAME`, per service: + * one cumulative counter series per `le` bucket, growing at a fixed rate. + */ +async function seedPromqlHistogram( + client: ReturnType, + startMs: number, + endMs: number, +) { + await insertPromqlMetric( + client, + startMs, + endMs, + { + name: E2E_PROMQL_HISTOGRAM_METRIC_NAME, + type: 'histogram', + unit: 'seconds', + help: 'E2E request duration', + }, + SERVICES.flatMap(service => + Object.entries(PROMQL_HISTOGRAM_BUCKET_INCREASES).map( + ([le, increase]) => ({ + metricName: `${E2E_PROMQL_HISTOGRAM_METRIC_NAME}_bucket`, + tags: { service, le }, + valueSql: `toFloat64(n * ${increase})`, + }), + ), + ), + ); +} + // Allow running directly for testing if (require.main === module) { seedClickHouse() diff --git a/packages/app/tests/e2e/utils/constants.ts b/packages/app/tests/e2e/utils/constants.ts index 91a105a240..6e43341375 100644 --- a/packages/app/tests/e2e/utils/constants.ts +++ b/packages/app/tests/e2e/utils/constants.ts @@ -60,8 +60,10 @@ export const E2E_METRICS_SUM_TABLE = 'e2e_otel_metrics_sum'; // tables above are not usable for PromQL. The seeder writes through the // timeSeries* table functions, which take this same table name. export const E2E_PROMQL_TABLE = 'e2e_promql'; -/** The one metric name seeded into the PromQL table, one series per service. */ +/** A gauge seeded into the PromQL table, one series per service. */ export const E2E_PROMQL_METRIC_NAME = 'e2e_service_up'; +/** A classic histogram seeded into the PromQL table as `_bucket` series. */ +export const E2E_PROMQL_HISTOGRAM_METRIC_NAME = 'e2e_request_duration_seconds'; // A second database holding OTEL-shaped metric tables, so the metric table // autofill tests can switch the source form's database and see tables detected // from the new one. The table names differ from the `default` database's so an diff --git a/packages/common-utils/src/core/__tests__/promql.test.ts b/packages/common-utils/src/core/__tests__/promql.test.ts index a0c1c8a6cf..2f19318b0d 100644 --- a/packages/common-utils/src/core/__tests__/promql.test.ts +++ b/packages/common-utils/src/core/__tests__/promql.test.ts @@ -6,6 +6,7 @@ import { getPromqlSeries, getQueriedPromqlSeries, isRangeQuery, + parsePromqlLe, PROMQL_MACROS, promqlSeriesQueryType, promqlStep, @@ -250,6 +251,22 @@ describe('promqlStep with a window', () => { }); }); +describe('parsePromqlLe', () => { + it.each([ + ['0.5', 0.5], + ['10', 10], + ['1e+06', 1e6], + ['-1', -1], + ['+Inf', Infinity], + ])('parses %s', (le, expected) => { + expect(parsePromqlLe(le)).toBe(expected); + }); + + it.each([undefined, '', ' ', 'abc'])('is NaN for %p', le => { + expect(parsePromqlLe(le)).toBeNaN(); + }); +}); + describe('PROMQL_MACROS', () => { const hour: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), diff --git a/packages/common-utils/src/core/heatmap.ts b/packages/common-utils/src/core/heatmap.ts index 72a761b668..2018299a00 100644 --- a/packages/common-utils/src/core/heatmap.ts +++ b/packages/common-utils/src/core/heatmap.ts @@ -1,8 +1,8 @@ -import type { HeatmapMode } from '@/types'; +import type { PromqlHeatmapMode } from '@/types'; -/** A builder heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ -export function getHeatmapMode(config: { - heatmap?: { mode?: HeatmapMode }; -}): HeatmapMode { +/** A heatmap's mode; tiles saved before modes existed are distribution heatmaps. */ +export function getHeatmapMode(config: { + heatmap?: { mode?: M }; +}): M | 'distribution' { return config.heatmap?.mode ?? 'distribution'; } diff --git a/packages/common-utils/src/core/promql.ts b/packages/common-utils/src/core/promql.ts index 6406cc6b7c..7f070801b5 100644 --- a/packages/common-utils/src/core/promql.ts +++ b/packages/common-utils/src/core/promql.ts @@ -133,6 +133,17 @@ export const promqlStep = ( ): string => `${promqlStepSeconds(granularity, dateRange, minGranularitySeconds)}s`; +/** + * A histogram bucket's `le` label as its upper bound. Prometheus writes the + * open-ended bucket as `+Inf`, which `Number` and `parseFloat` don't accept. + * NaN for a missing or non-numeric label. + */ +export function parsePromqlLe(le: string | undefined): number { + if (le === '+Inf') return Infinity; + if (le == null || le.trim() === '') return NaN; + return Number(le); +} + /** * Prometheus' default scrape interval, which `$__rate_interval` assumes when * the PromQL source has no minimum auto granularity set. diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 1d2ab24e29..d07eebf420 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -397,6 +397,8 @@ export const ColorConditionSchema = z.discriminatedUnion('operator', [ export type ColorCondition = z.infer; +export const HeatmapScaleTypeSchema = z.enum(['log', 'linear']); + export const DerivedColumnSchema = z.intersection( RootValueExpressionSchema, z.object({ @@ -406,7 +408,7 @@ export const DerivedColumnSchema = z.intersection( metricNameSql: z.string().optional(), // Heatmap-specific fields (optional, only used when displayType is Heatmap) countExpression: z.string().optional(), - heatmapScaleType: z.enum(['log', 'linear']).optional(), + heatmapScaleType: HeatmapScaleTypeSchema.optional(), numberFormat: NumberFormatSchema.optional(), // Per-column palette-token color, applied by the renderer to this // column's cells on table tiles only (gated in the series editor by @@ -1667,9 +1669,20 @@ export type RatioMode = z.infer; export const HeatmapModeSchema = z.enum(['distribution', 'series']); export type HeatmapMode = z.infer; +// PromQL heatmaps can also draw pre-bucketed Prometheus histograms (`le` buckets). +export const PromqlHeatmapModeSchema = z.enum([ + ...HeatmapModeSchema.options, + 'histogram', +]); +export type PromqlHeatmapMode = z.infer; + +export const HeatmapSettingsSchema = z.object({ + mode: HeatmapModeSchema.optional(), +}); + export const _ChartConfigSchema = SharedChartSettingsSchema.extend({ // Heatmap tiles only; an absent mode is 'distribution' (see getHeatmapMode). - heatmap: z.object({ mode: HeatmapModeSchema.optional() }).optional(), + heatmap: HeatmapSettingsSchema.optional(), timestampValueExpression: z.string(), implicitColumnExpression: z.string().optional(), // Fallback expression for bare-text Lucene search when no implicit column is @@ -1866,6 +1879,10 @@ const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ source: z.string().optional(), step: z.string().optional(), legendTemplate: z.string().max(MAX_LEGEND_TEMPLATE_LENGTH).optional(), + heatmap: HeatmapSettingsSchema.extend({ + mode: PromqlHeatmapModeSchema.optional(), + scaleType: HeatmapScaleTypeSchema.optional(), + }).optional(), }); /** Schema describing PromQL chart configs with runtime-only fields */