diff --git a/.changeset/promql-macros.md b/.changeset/promql-macros.md new file mode 100644 index 0000000000..185f7d3206 --- /dev/null +++ b/.changeset/promql-macros.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +--- + +feat: Support macros in PromQL charts diff --git a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx index eb029ae6eb..c2bea4b95d 100644 --- a/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlExpressionEditor.tsx @@ -64,6 +64,7 @@ export default function PromqlExpressionEditor({ onSubmit={() => onSubmit()} placeholder="rate(http_requests_total{service='api'}[5m])" metricNames={metricNames} + enableMacros /> {isIgnored ? ( diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx b/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx index 0d1593c903..c750a03d34 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx @@ -415,7 +415,7 @@ describe('ChartPreviewPanel', () => { await userEvent.hover(wrapper!); expect( - await screen.findByText(/Variables could not be expanded/), + await screen.findByText(/Expression could not be expanded/), ).toBeInTheDocument(); expect( screen.queryByTestId('chart-promql-preview'), diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts index 27a94ff390..2ce5342b07 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts @@ -222,6 +222,7 @@ describe('buildRenderedPromqlExpression', () => { promqlExpression?: PromqlExpressionList; displayType?: DisplayType; variables?: ChartVariable[]; + granularity?: string; } = {}, ): ChartConfigWithDateRange => ({ configType: 'promql', @@ -302,6 +303,47 @@ describe('buildRenderedPromqlExpression', () => { ).toEqual(['up{service=~"api"}', 'errors{service=~"api"}']); }); + it('expands macros with the granularity the chart queries with', () => { + const expression = 'rate(up[$__rate_interval]) / $__interval / $__range'; + const render = (displayType: DisplayType, granularity?: string) => + buildRenderedPromqlExpression( + promqlConfig({ + displayType, + granularity, + promqlExpression: [{ expression }], + }), + )?.expressions?.[0].expression; + + expect(render(DisplayType.Line, '5 minute')).toBe( + 'rate(up[315s]) / 300s / 86400s', + ); + // `auto` resolves to the chart's 80-bucket granularity (30 minutes for a + // day), not promqlStep's own fallback. + expect(render(DisplayType.Line, 'auto')).toBe( + 'rate(up[1815s]) / 1800s / 86400s', + ); + expect(render(DisplayType.Number, 'auto')).toBe( + 'rate(up[1815s]) / 1800s / 86400s', + ); + }); + + it('expands macros with the granularity a table queries with', () => { + const seventyMinutes: [Date, Date] = [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:10:00Z'), + ]; + expect( + buildRenderedPromqlExpression({ + ...promqlConfig({ + displayType: DisplayType.Table, + granularity: 'auto', + promqlExpression: [{ expression: 'rate(up[$__interval])' }], + }), + dateRange: seventyMinutes, + })?.expressions?.[0].expression, + ).toBe('rate(up[60s])'); + }); + it('reports a substitution failure instead of an expression', () => { const result = buildRenderedPromqlExpression( promqlConfig({ @@ -310,7 +352,7 @@ describe('buildRenderedPromqlExpression', () => { }), ); expect(result?.expressions).toBeUndefined(); - expect(result?.error).toMatch(/Variables could not be expanded/); + expect(result?.error).toMatch(/Expression could not be expanded/); }); }); diff --git a/packages/app/src/components/DBEditTimeChartForm/utils.ts b/packages/app/src/components/DBEditTimeChartForm/utils.ts index 17d8f23934..629c59fc6c 100644 --- a/packages/app/src/components/DBEditTimeChartForm/utils.ts +++ b/packages/app/src/components/DBEditTimeChartForm/utils.ts @@ -4,6 +4,7 @@ import { TableConnectionChoice, } from '@hyperdx/common-utils/dist/core/metadata'; import { getQueriedPromqlSeries } from '@hyperdx/common-utils/dist/core/promql'; +import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; import { configConsumesBroadcastFilters, getBlockingRequiredFilterNames, @@ -14,6 +15,7 @@ import { isRawSqlChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { substitutePromqlChartConfigTemplates } from '@hyperdx/common-utils/dist/macros'; import { BuilderChartConfigWithDateRange, ChartAlertBaseSchema, @@ -21,22 +23,23 @@ import { ChartConfigWithOptTimestamp, ChartVariable, DashboardFilter, + DateRange, DisplayType, Filter, + PromqlChartConfig, SavedChartConfig, SelectList, SourceKind, TSource, validateAlertScheduleOffsetMinutes, } from '@hyperdx/common-utils/dist/types'; -import { - filterReferencedVariables, - substitutePromqlChartConfigVariables, -} from '@hyperdx/common-utils/dist/variables'; +import { filterReferencedVariables } from '@hyperdx/common-utils/dist/variables'; import { convertToCategoricalChartConfig, convertToNumberChartConfig, + convertToPromqlNumberChartConfig, + convertToPromqlTableChartConfig, convertToTableChartConfig, convertToTimeChartConfig, tryExpandConfigVariables, @@ -263,7 +266,28 @@ export type RenderedPromqlExpression = | { expressions: RenderedPromqlEntry[]; error?: never } | { expressions?: never; error: string }; -/** The expressions a PromQL tile is queried with, with variables substituted. */ +/** + * The config a PromQL chart of this display type actually queries with. Macros + * depend on the resolved granularity and date range, so the preview must + * resolve them the same way the chart does. + */ +function toQueriedPromqlConfig( + config: PromqlChartConfig & DateRange, +): PromqlChartConfig & DateRange { + if (config.displayType === DisplayType.Number) { + return convertToPromqlNumberChartConfig(config, { withReducer: false }); + } + if (config.displayType === DisplayType.Table) { + return convertToPromqlTableChartConfig(config); + } + if (isTimeSeriesDisplayType(config.displayType)) { + const converted = convertToTimeChartConfig(config); + return isPromqlChartConfig(converted) ? converted : config; + } + return config; +} + +/** The expressions a PromQL tile is queried with, with macros and variables substituted. */ export function buildRenderedPromqlExpression( queriedConfig: ChartConfigWithDateRange | undefined, ): RenderedPromqlExpression | undefined { @@ -272,7 +296,9 @@ export function buildRenderedPromqlExpression( } try { - const substituted = substitutePromqlChartConfigVariables(queriedConfig); + const substituted = substitutePromqlChartConfigTemplates( + toQueriedPromqlConfig(queriedConfig), + ); return { expressions: getQueriedPromqlSeries(substituted).map((series, index) => ({ id: String(index), @@ -287,8 +313,8 @@ export function buildRenderedPromqlExpression( return { error: e instanceof Error - ? `Variables could not be expanded: ${e.message}` - : 'Variables could not be expanded.', + ? `Expression could not be expanded: ${e.message}` + : 'Expression could not be expanded.', }; } } diff --git a/packages/app/src/components/PromQLEditor/PromQLEditor.tsx b/packages/app/src/components/PromQLEditor/PromQLEditor.tsx index 4bfc0fbb58..d8167f9b52 100644 --- a/packages/app/src/components/PromQLEditor/PromQLEditor.tsx +++ b/packages/app/src/components/PromQLEditor/PromQLEditor.tsx @@ -23,13 +23,17 @@ import { createCodeMirrorStyleTheme, DEFAULT_CODE_MIRROR_BASIC_SETUP, } from '@/components/SQLEditor/utils'; -import { usePromqlVariableCompletions } from '@/components/SQLEditor/variableCompletions'; +import { + PROMQL_MACRO_COMPLETIONS, + usePromqlVariableCompletions, +} from '@/components/SQLEditor/variableCompletions'; import { useVariableValidation, VariableIssueIndicator, variableValidationState, } from '@/components/SQLEditor/variableValidation'; +import { promqlTemplateExtension } from './templateMaskedLanguage'; import { createVariableCompletionSource } from './variableCompletionSource'; import styles from '@/components/SQLEditor/SQLInlineEditor.module.scss'; @@ -47,6 +51,8 @@ type PromQLEditorProps = { * modal or other scroll container, which would otherwise clip the popup. */ parentRef?: HTMLElement | null; + /** Suggest `$__interval` and the other chart macros. */ + enableMacros?: boolean; }; const MAX_EDITOR_HEIGHT = '150px'; @@ -113,30 +119,37 @@ export default function PromQLEditor({ onSubmit, metricNames, parentRef, + enableMacros, }: PromQLEditorProps) { const { colorScheme } = useMantineColorScheme(); const ref = useRef(null); const compartmentRef = useRef(new Compartment()); const [isFocused, setIsFocused] = useState(false); const variableCompletions = usePromqlVariableCompletions(); + const templateCompletions = useMemo( + () => + enableMacros + ? [...PROMQL_MACRO_COMPLETIONS, ...variableCompletions] + : variableCompletions, + [enableMacros, variableCompletions], + ); const variableIssues = useVariableValidation(value, { language: 'promql' }); const updateAutocomplete = useCallback( (viewRef: EditorView) => { const override: CompletionSource[] = []; - if (variableCompletions.length > 0) { - override.push(createVariableCompletionSource(variableCompletions)); + if (templateCompletions.length > 0) { + override.push(createVariableCompletionSource(templateCompletions)); } if (metricNames?.length) { override.push(debounceAndPruneAutocompleteResults(metricNames)); } - // PromQL's own function/keyword completion, re-registered explicitly. - // `PromQLExtension.asExtension()` registers it through - // `language.data.of({ autocomplete })`, which an `override` array - // replaces outright — so without this it is silently lost. + // PromQL's own function/keyword completion. `promqlTemplateExtension` + // installs the language without it, and an `override` array would + // replace a `language.data` registration outright anyway. override.push(context => promqlExtension.getComplete().promQL(context)); viewRef.dispatch({ @@ -145,7 +158,7 @@ export default function PromQLEditor({ ), }); }, - [metricNames, variableCompletions], + [metricNames, templateCompletions], ); useEffect(() => { @@ -160,8 +173,10 @@ export default function PromQLEditor({ createCodeMirrorStyleTheme(MAX_EDITOR_HEIGHT), EditorView.lineWrapping, - // PromQL syntax highlighting - promqlExtension.asExtension(), + // PromQL syntax highlighting and linting, variable- and macro-aware + promqlTemplateExtension(promqlExtension, { + enableMacros: !!enableMacros, + }), // Autocomplete sources (via compartment for hot-swapping) // eslint-disable-next-line react-hooks/refs @@ -190,7 +205,7 @@ export default function PromQLEditor({ }, ]), ], - [onSubmit, parentRef], + [onSubmit, parentRef, enableMacros], ); const onClickCodeMirror = useCallback(() => { diff --git a/packages/app/src/components/PromQLEditor/__tests__/templateMaskedLanguage.test.ts b/packages/app/src/components/PromQLEditor/__tests__/templateMaskedLanguage.test.ts new file mode 100644 index 0000000000..88f41578b7 --- /dev/null +++ b/packages/app/src/components/PromQLEditor/__tests__/templateMaskedLanguage.test.ts @@ -0,0 +1,102 @@ +import { + findMaskedTemplateRanges, + maskPromqlTemplates, + templateMaskedPromqlLanguage, +} from '@/components/PromQLEditor/templateMaskedLanguage'; + +const errorSpans = (doc: string, enableMacros = true) => { + const spans: string[] = []; + templateMaskedPromqlLanguage({ enableMacros }) + .parser.parse(doc) + .iterate({ + enter: node => { + if (node.type.isError) spans.push(doc.slice(node.from, node.to)); + }, + }); + return spans; +}; + +describe('findMaskedTemplateRanges', () => { + it('covers each macro and reference exactly', () => { + const expression = 'topk($k, rate(up[$__rate_interval]))'; + expect( + findMaskedTemplateRanges(expression, { enableMacros: true }).map( + ({ from, to }) => expression.slice(from, to), + ), + ).toEqual(['$k', '$__rate_interval']); + }); + + it('leaves references inside label strings alone', () => { + expect( + findMaskedTemplateRanges('up{service=~"$service"}', { + enableMacros: true, + }), + ).toEqual([]); + }); +}); + +describe('maskPromqlTemplates', () => { + it('keeps the expression length', () => { + const expression = 'rate(up{a="$a"}[$__interval]) * $k'; + const masked = maskPromqlTemplates(expression, { enableMacros: true }); + expect(masked).toBe('rate(up{a="$a"}[11111111111]) * aa'); + expect(masked).toHaveLength(expression.length); + }); + + it('masks a variable in a label-name position as an identifier', () => { + expect( + maskPromqlTemplates('sum by (${service:csv}) (up)', { + enableMacros: true, + }), + ).toBe('sum by (aaaaaaaaaaaaaa) (up)'); + }); +}); + +describe('templateMaskedPromqlLanguage', () => { + it('parses macros in range and offset positions', () => { + expect( + errorSpans( + 'sum(rate(up[$__range])) + avg_over_time(up[$__interval] offset $__rate_interval)', + ), + ).toEqual([]); + }); + + it('parses variables in metric and label name positions', () => { + expect(errorSpans('sum by (${service:csv}) (up)')).toEqual([]); + expect(errorSpans('sum by (job, $label) ($metric{${key}="x"})')).toEqual( + [], + ); + expect(errorSpans('a / on(${keys:csv}) group_left($extra) b')).toEqual([]); + }); + + it('rejects a variable in a duration position', () => { + expect(errorSpans('rate(up[$window])')).not.toEqual([]); + }); + + it('keeps node offsets on the original text', () => { + const doc = 'rate(up[$__range])'; + const spans: string[] = []; + templateMaskedPromqlLanguage({ enableMacros: true }) + .parser.parse(doc) + .iterate({ + enter: node => { + if (node.name === 'NumberDurationLiteralInDurationContext') { + spans.push(doc.slice(node.from, node.to)); + } + }, + }); + expect(spans).toEqual(['$__range']); + }); + + it('rejects macros when macros are disabled', () => { + expect(errorSpans('rate(up[$__interval])', false)).not.toEqual([]); + }); + + it('parses a macro given arguments', () => { + expect(errorSpans('rate(up[$__interval(1)])')).toEqual([]); + }); + + it('still rejects errors elsewhere in the expression', () => { + expect(errorSpans('rate(up[$__interval]) +')).not.toEqual([]); + }); +}); diff --git a/packages/app/src/components/PromQLEditor/__tests__/variableCompletionSource.test.ts b/packages/app/src/components/PromQLEditor/__tests__/variableCompletionSource.test.ts index d801c1338d..f9154b25e4 100644 --- a/packages/app/src/components/PromQLEditor/__tests__/variableCompletionSource.test.ts +++ b/packages/app/src/components/PromQLEditor/__tests__/variableCompletionSource.test.ts @@ -68,6 +68,12 @@ describe('createVariableCompletionSource', () => { pos: 2, // $e|nv expected: { from: 0, to: 4, pattern: '$env' }, }, + { + name: 'macro inside a range selector stops before the bracket', + doc: 'rate(x[$__ra])', + pos: 12, // rate(x[$__ra|]) + expected: { from: 7, to: 12, pattern: '$__ra' }, + }, { name: 'bare reference followed by an adjacent reference stops at it', doc: '$env$other', diff --git a/packages/app/src/components/PromQLEditor/templateMaskedLanguage.ts b/packages/app/src/components/PromQLEditor/templateMaskedLanguage.ts new file mode 100644 index 0000000000..c2212f4036 --- /dev/null +++ b/packages/app/src/components/PromQLEditor/templateMaskedLanguage.ts @@ -0,0 +1,121 @@ +import { linter } from '@codemirror/lint'; +import { + isPromqlMacroName, + PROMQL_MACRO_NAMES, +} from '@hyperdx/common-utils/dist/core/promql'; +import { + scanTemplateTokens, + VARIABLE_MACRO_NAMES, +} from '@hyperdx/common-utils/dist/variables'; +import { + LanguageType, + PromQLExtension, + promQLLanguage, +} from '@prometheus-io/codemirror-promql'; + +type TemplateOptions = { enableMacros: boolean }; + +type TemplateRange = { from: number; to: number; isMacro: boolean }; + +/** + * The spans of an expression that are templates rather than PromQL: macros + * (when enabled) and variable references outside string literals. A macro's + * arguments are masked with it: substitution reports a bad argument count. + */ +export function findMaskedTemplateRanges( + expression: string, + { enableMacros }: TemplateOptions, +): TemplateRange[] { + const ranges: TemplateRange[] = []; + let offset = 0; + const tokens = scanTemplateTokens( + expression, + enableMacros + ? [...VARIABLE_MACRO_NAMES, ...PROMQL_MACRO_NAMES] + : VARIABLE_MACRO_NAMES, + { onMalformed: 'skip' }, + ); + for (const token of tokens) { + if (token.kind === 'text') { + offset += token.text.length; + continue; + } + const isMacro = token.kind === 'macro'; + const isMaskable = + !token.inStringLiteral && (!isMacro || isPromqlMacroName(token.name)); + if (isMaskable) { + ranges.push({ from: offset, to: offset + token.raw.length, isMacro }); + } + offset += token.raw.length; + } + return ranges; +} + +const basePromqlLanguage = promQLLanguage(LanguageType.PromQL); + +/** + * The expression with each template swapped for same-length filler, so the + * parse tree's offsets still line up with the original text. Macros are + * durations, so they become digits. Variables usually stand for names, as in + * `sum by (${labels:csv})`, so they become letters; one used as a number or + * duration is flagged. + */ +export function maskPromqlTemplates( + expression: string, + options: TemplateOptions, +): string { + let masked = ''; + let offset = 0; + for (const { from, to, isMacro } of findMaskedTemplateRanges( + expression, + options, + )) { + masked += + expression.slice(offset, from) + (isMacro ? '1' : 'a').repeat(to - from); + offset = to; + } + return masked + expression.slice(offset); +} + +/** + * The PromQL language, parsing the masked expression so highlighting, bracket + * matching, completion and linting all see a valid tree around templates. + */ +export const templateMaskedPromqlLanguage = (options: TemplateOptions) => + basePromqlLanguage.configure({ + wrap: (_inner, input, _fragments, ranges) => { + const masked = maskPromqlTemplates(input.read(0, input.length), options); + // No fragments: a mask can change outside the edited range (e.g. typing + // a quote), so reusing the old tree could leave stale nodes. + return basePromqlLanguage.parser.createParse( + { + length: masked.length, + chunk: from => masked.slice(from), + lineChunks: false, + read: (from, to) => masked.slice(from, to), + }, + [], + ranges, + ); + }, + }); + +const MASKED_LANGUAGES = { + macros: templateMaskedPromqlLanguage({ enableMacros: true }), + noMacros: templateMaskedPromqlLanguage({ enableMacros: false }), +}; + +/** + * Stands in for the `PromQLExtension.asExtension()` codemirror extension, providing + * linting and syntax highlighting that is variable- and macro-aware. + */ +export function promqlTemplateExtension( + promqlExtension: PromQLExtension, + { enableMacros }: TemplateOptions, +) { + const lintStrategy = promqlExtension.getLinter(); + return [ + enableMacros ? MASKED_LANGUAGES.macros : MASKED_LANGUAGES.noMacros, + linter(lintStrategy.promQL()), + ]; +} diff --git a/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts b/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts index 21956b49fa..fbb3fbfdb7 100644 --- a/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts +++ b/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts @@ -1,6 +1,9 @@ import { ChartVariable } from '@hyperdx/common-utils/dist/types'; -import { buildPromqlVariableCompletions } from '@/components/SQLEditor/variableCompletions'; +import { + buildPromqlVariableCompletions, + PROMQL_MACRO_COMPLETIONS, +} from '@/components/SQLEditor/variableCompletions'; const SERVICE: ChartVariable = { name: 'service', @@ -114,3 +117,15 @@ describe('buildPromqlVariableCompletions', () => { expect(info?.firstChild?.textContent).not.toContain('Expands to:'); }); }); + +describe('PROMQL_MACRO_COMPLETIONS', () => { + it('inserts each macro without an argument list', () => { + expect( + PROMQL_MACRO_COMPLETIONS.map(({ label, apply }) => [label, apply]), + ).toEqual([ + ['$__interval', '$__interval'], + ['$__range', '$__range'], + ['$__rate_interval', '$__rate_interval'], + ]); + }); +}); diff --git a/packages/app/src/components/SQLEditor/variableCompletions.tsx b/packages/app/src/components/SQLEditor/variableCompletions.tsx index a5b1ce1e21..653c7177e3 100644 --- a/packages/app/src/components/SQLEditor/variableCompletions.tsx +++ b/packages/app/src/components/SQLEditor/variableCompletions.tsx @@ -1,4 +1,5 @@ import { createContext, use, useCallback, useMemo } from 'react'; +import { PROMQL_MACROS } from '@hyperdx/common-utils/dist/core/promql'; import { MacroSuggestion, VARIABLE_MACRO_SUGGESTIONS, @@ -207,6 +208,10 @@ export function buildPromqlVariableCompletions( return variables.flatMap(getPromqlVariableCompletions); } +/** Completions for the macros a PromQL chart expression can use. */ +export const PROMQL_MACRO_COMPLETIONS: SQLCompletion[] = + PROMQL_MACROS.map(toMacroCompletion); + /** One bare `$name` suggestion for a Lucene input. */ export type LuceneVariableSuggestion = { value: string; diff --git a/packages/app/src/utils/promqlChartQuery.ts b/packages/app/src/utils/promqlChartQuery.ts index 6654a0188c..38949028ac 100644 --- a/packages/app/src/utils/promqlChartQuery.ts +++ b/packages/app/src/utils/promqlChartQuery.ts @@ -6,6 +6,7 @@ import { reducePromqlSamples, } from '@hyperdx/common-utils/dist/core/promql'; import { renderPromqlSeriesNames } from '@hyperdx/common-utils/dist/core/seriesNameTemplate'; +import { substitutePromqlChartConfigTemplates } from '@hyperdx/common-utils/dist/macros'; import { DisplayType, PrometheusMatrixResult, @@ -13,7 +14,6 @@ import { PromqlReducer, PromqlSeries, } from '@hyperdx/common-utils/dist/types'; -import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import { prometheusApi, PrometheusInstantQueryResponse } from '@/api'; import { MAX_TABLE_ROWS } from '@/HDXMultiSeriesTableChart'; @@ -261,8 +261,11 @@ export async function queryPromqlChartConfig( dateRange: [Date, Date], signal: AbortSignal, ): Promise { - // Expand dashboard variables in the expressions before sending to Prometheus. - const substituted = substitutePromqlChartConfigVariables(config); + // Expand dashboard variables and macros before sending to Prometheus. + const substituted = substitutePromqlChartConfigTemplates({ + ...config, + dateRange, + }); const results = await Promise.all( getQueriedPromqlSeries(substituted).map(series => diff --git a/packages/app/tests/e2e/features/promql-macros.spec.ts b/packages/app/tests/e2e/features/promql-macros.spec.ts new file mode 100644 index 0000000000..a4f67d85cf --- /dev/null +++ b/packages/app/tests/e2e/features/promql-macros.spec.ts @@ -0,0 +1,77 @@ +/** + * `$__interval`, `$__range` and `$__rate_interval` expand to concrete + * durations before a PromQL expression reaches Prometheus, in the chart + * explorer and on dashboards with no variables. An unexpanded macro is a + * PromQL parse error, so a rendered chart or value proves expansion. + */ +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import { ChartExplorerPage } from '../page-objects/ChartExplorerPage'; +import { DashboardPage } from '../page-objects/DashboardPage'; +import { expect, test } from '../utils/base-test'; +import { E2E_PROMQL_METRIC_NAME, PROMQL_SOURCE_NAME } from '../utils/constants'; + +const ONE_SERIES = `${E2E_PROMQL_METRIC_NAME}{service="accounting"}`; + +test.describe('PromQL macros', { tag: ['@charts', '@full-stack'] }, () => { + test('expands macros in the chart explorer', async ({ page }) => { + const chartExplorerPage = new ChartExplorerPage(page); + const editor = chartExplorerPage.chartEditor; + + await test.step('Query a PromQL chart that uses a macro', async () => { + await chartExplorerPage.goto(); + await expect(chartExplorerPage.form).toBeVisible(); + await editor.switchToPromqlMode(); + await editor.selectSource(PROMQL_SOURCE_NAME); + await editor.setChartType(DisplayType.Number); + await editor.replacePromqlExpression( + `last_over_time(${ONE_SERIES}[$__rate_interval])`, + ); + await editor.runQuery(false); + + // The seed writes 1 for every sample of this series. + await expect(page.getByTestId('number-chart-value')).toHaveText('1', { + timeout: 30000, + }); + }); + + await test.step('The preview shows the expanded duration', async () => { + await editor.openGeneratedPromql(); + await expect + .poll(() => editor.getGeneratedPromqlText()) + .toMatch(/^last_over_time\(.+\[\d+s\]\)$/); + }); + }); + + test('expands macros on a dashboard without variables', async ({ page }) => { + const dashboardPage = new DashboardPage(page); + const editor = dashboardPage.chartEditor; + const value = page.getByTestId('number-chart-value'); + + await test.step('Create a PromQL number tile that uses $__range', async () => { + await dashboardPage.goto(); + await dashboardPage.openNewTileEditor(); + await editor.switchToPromqlMode(); + await editor.selectSource(PROMQL_SOURCE_NAME); + await editor.setChartName('PromQL macro tile'); + await editor.setChartType(DisplayType.Number); + await editor.replacePromqlExpression( + `last_over_time(${ONE_SERIES}[$__range])`, + ); + await editor.runQuery(false); + + // The seed writes 1 for every sample of this series. + await expect(value).toHaveText('1', { timeout: 30000 }); + }); + + await test.step('The saved tile still queries successfully', async () => { + await editor.save(); + await expect(dashboardPage.getTiles()).toHaveCount(1, { + timeout: 10000, + }); + await expect(dashboardPage.getNumberTileValue()).toHaveText('1', { + timeout: 30000, + }); + }); + }); +}); diff --git a/packages/common-utils/src/__tests__/macros.test.ts b/packages/common-utils/src/__tests__/macros.test.ts index 151dc99d28..df0b1152f3 100644 --- a/packages/common-utils/src/__tests__/macros.test.ts +++ b/packages/common-utils/src/__tests__/macros.test.ts @@ -4,8 +4,17 @@ import { hasMacro, isMissingFiltersMacro, replaceMacros, + substitutePromqlChartConfigTemplates, } from '@/macros'; -import type { MetricTable } from '@/types'; +import type { ChartVariable, MetricTable } from '@/types'; + +const variable = (name: string, values: string[]): ChartVariable => ({ + name, + values, +}); + +const SERVICE = variable('service', ['api', 'web']); +const EMPTY_SERVICE = variable('service', []); const ALL_METRIC_TABLES: MetricTable = { gauge: 'otel_metrics_gauge', @@ -603,3 +612,152 @@ describe('replaceMacros with variables', () => { }); }); }); + +const PROMQL_DATE_RANGE: [Date, Date] = [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), +]; + +describe('substitutePromqlChartConfigTemplates', () => { + const promqlConfig = ( + promqlExpression: string, + variables?: ChartVariable[], + ) => ({ + configType: 'promql' as const, + promqlExpression, + connection: 'local', + variables, + dateRange: PROMQL_DATE_RANGE, + }); + + it('leaves references as written when there is no variable context', () => { + expect( + substitutePromqlChartConfigTemplates( + promqlConfig('up{service=~"$service"}'), + ).promqlExpression, + ).toBe('up{service=~"$service"}'); + }); + + it('expands the expression and consumes the variables', () => { + expect( + substitutePromqlChartConfigTemplates( + promqlConfig('up{service=~"$service"}', [SERVICE]), + ), + ).toMatchObject({ + promqlExpression: 'up{service=~"(api|web)"}', + variables: undefined, + }); + }); + + it('renders an empty selection as an unconstrained matcher', () => { + expect( + substitutePromqlChartConfigTemplates( + promqlConfig('up{service=~"$service"}', [EMPTY_SERVICE]), + ).promqlExpression, + ).toBe('up{service=~".*"}'); + }); + + it('expands every expression of a multi-expression config', () => { + expect( + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression: [ + { expression: 'up{service=~"$service"}', alias: 'up' }, + { expression: 'rate(errors{service=~"$service"}[5m])' }, + ], + variables: [SERVICE], + dateRange: PROMQL_DATE_RANGE, + }), + ).toMatchObject({ + promqlExpression: [ + { expression: 'up{service=~"(api|web)"}', alias: 'up' }, + { expression: 'rate(errors{service=~"(api|web)"}[5m])' }, + ], + variables: undefined, + }); + }); +}); + +describe('substitutePromqlChartConfigTemplates macros', () => { + const substitute = ( + promqlExpression: string, + { + variables, + granularity = '5 minute', + dateRange = PROMQL_DATE_RANGE, + }: { + variables?: ChartVariable[]; + granularity?: string; + dateRange?: [Date, Date]; + } = {}, + ) => + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression, + variables, + granularity, + dateRange, + }).promqlExpression; + + it('expands macros without a variable context', () => { + expect( + substitute( + 'rate(a[$__rate_interval]) + avg_over_time(b[$__interval]) + increase(c[$__range])', + ), + ).toBe('rate(a[315s]) + avg_over_time(b[300s]) + increase(c[3600s])'); + }); + + it('expands macros and variables together', () => { + expect( + substitute('rate(up{service=~"$service"}[$__rate_interval])', { + variables: [SERVICE], + }), + ).toBe('rate(up{service=~"(api|web)"}[315s])'); + }); + + it('expands every expression of a multi-expression config', () => { + expect( + substitutePromqlChartConfigTemplates({ + configType: 'promql' as const, + connection: 'local', + promqlExpression: [ + { expression: 'rate(a[$__interval])' }, + { expression: 'increase(b[$__range])' }, + ], + granularity: '1 minute', + dateRange: PROMQL_DATE_RANGE, + }).promqlExpression, + ).toEqual([ + { expression: 'rate(a[60s])' }, + { expression: 'increase(b[3600s])' }, + ]); + }); + + it('leaves a selected value that looks like a macro inert', () => { + expect( + substitute('up{path=~"${path:csv}"}', { + variables: [variable('path', ['$__interval'])], + }), + ).toBe('up{path=~"$__interval"}'); + }); + + it('leaves SQL-only and unknown macros as written', () => { + expect(substitute('up{a="$__filter($service)"} $__foo $__intervals')).toBe( + 'up{a="$__filter($service)"} $__foo $__intervals', + ); + }); + + it('throws for a macro given arguments', () => { + expect(() => substitute('rate(a[$__interval(1)])')).toThrow( + "Macro 'interval' expects 0 argument(s), but got 1", + ); + }); + + it('throws for a macro with an unclosed argument list', () => { + expect(() => substitute('up offset $__interval(')).toThrow( + MalformedMacroArgsError, + ); + }); +}); diff --git a/packages/common-utils/src/__tests__/variables.test.ts b/packages/common-utils/src/__tests__/variables.test.ts index 7a767d2be1..dd7dc12366 100644 --- a/packages/common-utils/src/__tests__/variables.test.ts +++ b/packages/common-utils/src/__tests__/variables.test.ts @@ -12,7 +12,6 @@ import { getVariableReferences, hasVariableMacro, substituteChartConfigVariables, - substitutePromqlChartConfigVariables, substituteVariables, validateVariableReferencesInTemplate, VariableContext, @@ -1679,62 +1678,6 @@ describe('substituteChartConfigVariables', () => { }); }); -describe('substitutePromqlChartConfigVariables', () => { - const promqlConfig = ( - promqlExpression: string, - variables?: ChartVariable[], - ) => ({ - configType: 'promql' as const, - promqlExpression, - connection: 'local', - variables, - }); - - it('returns the config untouched when there is no variable context', () => { - const config = promqlConfig('up{service=~"$service"}'); - expect(substitutePromqlChartConfigVariables(config)).toBe(config); - }); - - it('expands the expression and consumes the variables', () => { - expect( - substitutePromqlChartConfigVariables( - promqlConfig('up{service=~"$service"}', [SERVICE]), - ), - ).toMatchObject({ - promqlExpression: 'up{service=~"(api|web)"}', - variables: undefined, - }); - }); - - it('renders an empty selection as an unconstrained matcher', () => { - expect( - substitutePromqlChartConfigVariables( - promqlConfig('up{service=~"$service"}', [EMPTY_SERVICE]), - ).promqlExpression, - ).toBe('up{service=~".*"}'); - }); - - it('expands every expression of a multi-expression config', () => { - expect( - substitutePromqlChartConfigVariables({ - configType: 'promql' as const, - connection: 'local', - promqlExpression: [ - { expression: 'up{service=~"$service"}', alias: 'up' }, - { expression: 'rate(errors{service=~"$service"}[5m])' }, - ], - variables: [SERVICE], - }), - ).toMatchObject({ - promqlExpression: [ - { expression: 'up{service=~"(api|web)"}', alias: 'up' }, - { expression: 'rate(errors{service=~"(api|web)"}[5m])' }, - ], - variables: undefined, - }); - }); -}); - describe('validateVariableReferencesInTemplate', () => { const validate = validateVariableReferencesInTemplate; diff --git a/packages/common-utils/src/core/__tests__/promql.test.ts b/packages/common-utils/src/core/__tests__/promql.test.ts index 349929f703..1c631f2ba0 100644 --- a/packages/common-utils/src/core/__tests__/promql.test.ts +++ b/packages/common-utils/src/core/__tests__/promql.test.ts @@ -1,9 +1,11 @@ import { displayTypeSupportsInstantQuery, displayTypeSupportsReducer, + getPromqlMacroInputs, getPromqlSeries, getQueriedPromqlSeries, isRangeQuery, + PROMQL_MACROS, promqlSeriesQueryType, promqlStep, reducePromqlSamples, @@ -227,6 +229,45 @@ describe('promqlStep with a window', () => { }); }); +describe('PROMQL_MACROS', () => { + const hour: [Date, Date] = [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ]; + const expandMacro = ( + name: string, + granularity: string, + dateRange: [Date, Date], + ) => + PROMQL_MACROS.find(macro => macro.name === name)?.expand( + getPromqlMacroInputs(granularity, dateRange), + ); + + it('uses the step for $__interval', () => { + expect(expandMacro('interval', '5 minute', hour)).toBe('300s'); + expect(expandMacro('interval', 'auto', hour)).toBe( + promqlStep('auto', hour), + ); + }); + + it('rounds the window length to seconds for $__range', () => { + expect(expandMacro('range', '1 minute', hour)).toBe('3600s'); + expect( + expandMacro('range', '1 minute', [ + new Date('2024-01-01T00:00:00.000Z'), + new Date('2024-01-01T00:00:10.600Z'), + ]), + ).toBe('11s'); + expect(expandMacro('range', '1 minute', [hour[0], hour[0]])).toBe('1s'); + }); + + it('floors $__rate_interval at four scrape intervals', () => { + expect(expandMacro('rate_interval', '15 second', hour)).toBe('60s'); + expect(expandMacro('rate_interval', '1 minute', hour)).toBe('75s'); + expect(expandMacro('rate_interval', '5 minute', hour)).toBe('315s'); + }); +}); + describe('isRangeQuery', () => { it('is false while every queried expression is an instant query', () => { expect( diff --git a/packages/common-utils/src/core/promql.ts b/packages/common-utils/src/core/promql.ts index d7837009d2..9f35dcbcad 100644 --- a/packages/common-utils/src/core/promql.ts +++ b/packages/common-utils/src/core/promql.ts @@ -3,6 +3,7 @@ import { convertGranularityToSeconds, isTimeSeriesDisplayType, } from '@/core/utils'; +import type { MacroSuggestion } from '@/macros'; import { DisplayType, PromqlExpressionList, @@ -92,20 +93,95 @@ export function reducePromqlSamples( } } -/** A HyperDX granularity ("5 minute") as a Prometheus step ("300s"). */ -export const promqlStep = ( +/** A HyperDX granularity ("5 minute") as a Prometheus step, in seconds. */ +const promqlStepSeconds = ( granularity: string | undefined, dateRange?: [Date, Date], -): string => { +): number => { const resolved = (!granularity || granularity === 'auto') && dateRange ? convertDateRangeToGranularityString(dateRange) : granularity; - if (!resolved || resolved === 'auto') return '60s'; + if (!resolved || resolved === 'auto') return 60; // convertGranularityToSeconds returns 0 for units it doesn't recognize. - return `${convertGranularityToSeconds(resolved as SQLInterval) || 60}s`; + return convertGranularityToSeconds(resolved as SQLInterval) || 60; }; +/** A HyperDX granularity ("5 minute") as a Prometheus step ("300s"). */ +export const promqlStep = ( + granularity: string | undefined, + dateRange?: [Date, Date], +): string => `${promqlStepSeconds(granularity, dateRange)}s`; + +/** + * HyperDX doesn't know a PromQL source's scrape interval, so `$__rate_interval` + * assumes Prometheus' default. + * + * TODO (HDX-5512): Use the PromQL source's minGranularitySeconds setting instead. + */ +const PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS = 15; + +/** What the PromQL macros are computed from, for a chart's granularity and time range. */ +export type PromqlMacroInputs = { + intervalSeconds: number; + rangeSeconds: number; +}; + +export type PromqlMacro = MacroSuggestion & { + expand: (inputs: PromqlMacroInputs) => string; +}; + +export const PROMQL_MACROS = [ + { + name: 'interval', + minArgs: 0, + maxArgs: 0, + description: + 'The step between points in the chart, as a duration such as 60s. Use it as a range, e.g. avg_over_time(metric[$__interval]).', + expand: ({ intervalSeconds }) => `${intervalSeconds}s`, + }, + { + name: 'range', + minArgs: 0, + maxArgs: 0, + description: + 'The length of the selected time range, as a duration such as 3600s. e.g. increase(metric[$__range]).', + expand: ({ rangeSeconds }) => `${rangeSeconds}s`, + }, + { + name: 'rate_interval', + minArgs: 0, + maxArgs: 0, + description: `A safe range for rate() and increase(): the larger of $__interval + ${PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS}s and ${4 * PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS}s, assuming a ${PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS}s scrape interval. e.g. rate(metric[$__rate_interval]).`, + expand: ({ intervalSeconds }) => + `${Math.max( + intervalSeconds + PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS, + 4 * PROMQL_DEFAULT_SCRAPE_INTERVAL_SECONDS, + )}s`, + }, +] as const satisfies readonly PromqlMacro[]; + +export type PromqlMacroName = (typeof PROMQL_MACROS)[number]['name']; + +export const PROMQL_MACRO_NAMES = PROMQL_MACROS.map(({ name }) => name); + +export const isPromqlMacroName = (name: string): name is PromqlMacroName => + (PROMQL_MACRO_NAMES as readonly string[]).includes(name); + +export function getPromqlMacroInputs( + granularity: string | undefined, + dateRange: [Date, Date], +): PromqlMacroInputs { + const [start, end] = dateRange; + return { + intervalSeconds: promqlStepSeconds(granularity, dateRange), + rangeSeconds: Math.max( + 1, + Math.round((end.getTime() - start.getTime()) / 1000), + ), + }; +} + /** * Whether any expression this config queries is evaluated over a range, and so * reads the tile's granularity (step). diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 59835e6ec4..cbce10b168 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1685,7 +1685,7 @@ export function validateRawSqlChartConfig( } // Track macros this function has already described with an error, to avoid repetition. - const reportedMacros = new Set(); + const reportedMacros = new Set(); const pushError = (message: string, macro?: MacroName) => { errors.push(message); if (macro != null) reportedMacros.add(macro); diff --git a/packages/common-utils/src/macroErrors.ts b/packages/common-utils/src/macroErrors.ts index fcaf19f9ad..302806cd55 100644 --- a/packages/common-utils/src/macroErrors.ts +++ b/packages/common-utils/src/macroErrors.ts @@ -1,3 +1,4 @@ +import type { PromqlMacroName } from './core/promql'; import type { MacroName } from './macros'; /** Thrown for an unterminated macro argument list, e.g. `$__timeFilter(col`. */ @@ -11,7 +12,7 @@ export class MalformedMacroArgsError extends Error { /** Thrown when a macro cannot expand, tagged with the macro it came. */ export class MacroExpansionError extends Error { constructor( - public readonly macro: MacroName, + public readonly macro: MacroName | PromqlMacroName, message: string, ) { super(message); diff --git a/packages/common-utils/src/macros.ts b/packages/common-utils/src/macros.ts index 6bccd3ca6f..d1f1429e80 100644 --- a/packages/common-utils/src/macros.ts +++ b/packages/common-utils/src/macros.ts @@ -1,9 +1,17 @@ +import { + getPromqlMacroInputs, + PROMQL_MACRO_NAMES, + PROMQL_MACROS, + PromqlMacroName, +} from './core/promql'; import { splitAndTrimWithBracket } from './core/utils'; import { MacroExpansionError, MalformedMacroArgsError } from './macroErrors'; import { RawSqlQueryParam, renderQueryParam } from './rawSqlParams'; import { + ChartVariable, MetricsDataType, MetricsDataTypeSchema, + PromqlExpressionList, RawSqlChartConfig, } from './types'; import { @@ -17,7 +25,7 @@ import { } from './variables'; function expectArgs( - macroName: MacroName, + macroName: MacroName | PromqlMacroName, args: string[], minArgs: number, maxArgs: number, @@ -501,3 +509,66 @@ export function replaceMacros( variableContext ? expandVariableToken(token, variableContext) : token.raw, }); } + +/** + * Expand the macros (`$__interval`, `$__range`, `$__rate_interval`) and variable + * references in a PromQL config's expression(s), returning it with `variables` + * consumed. References to names that aren't variables are left as written, so + * `$1` in a `label_replace` replacement reaches Prometheus unchanged. + * + * Macros and variables expand in one pass, so a selected value that looks like + * `$__range` stays inert. Dropping `variables` from the result ensures variables + * can't be substituted twice. + */ +export function substitutePromqlChartConfigTemplates< + T extends { + promqlExpression: PromqlExpressionList; + variables?: ChartVariable[]; + granularity?: string; + dateRange: [Date, Date]; + }, +>(config: T): T { + const { promqlExpression, variables = [], granularity, dateRange } = config; + const variableContext: VariableContext = { + variables, + inputLanguage: 'promql', + }; + const macroInputs = getPromqlMacroInputs(granularity, dateRange); + const macrosByName = new Map( + PROMQL_MACROS.map(({ name, minArgs, maxArgs, description, expand }) => [ + name, + { + name, + minArgs, + maxArgs, + description, + replace: args => { + expectArgs(name, args, minArgs, maxArgs); + return expand(macroInputs); + }, + }, + ]), + ); + + const substitute = (expression: string) => + expandTemplate(expression, { + // `$__filter` / `$__conditionalAll` are scanned only so a reference in + // their arguments isn't expanded; they have no meaning in PromQL. + macroNames: [...VARIABLE_MACRO_NAMES, ...PROMQL_MACRO_NAMES], + expandMacro: token => + macrosByName.get(token.name)?.replace(token.args) ?? token.raw, + expandReference: token => expandVariableToken(token, variableContext), + }); + + return { + ...config, + promqlExpression: + typeof promqlExpression === 'string' + ? substitute(promqlExpression) + : promqlExpression.map(series => ({ + ...series, + expression: substitute(series.expression), + })), + variables: undefined, + }; +} diff --git a/packages/common-utils/src/variables.ts b/packages/common-utils/src/variables.ts index f90c1e397c..2f18e6f220 100644 --- a/packages/common-utils/src/variables.ts +++ b/packages/common-utils/src/variables.ts @@ -22,7 +22,6 @@ import { DASHBOARD_VARIABLE_NAME_PATTERN, DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, DisplayType, - PromqlExpressionList, SavedChartConfig, SearchConditionLanguage, SelectList, @@ -1021,38 +1020,6 @@ export function substituteChartConfigVariables< return { ...substituted, variables: undefined }; } -/** - * Expand the variable references in a PromQL config's expression(s), returning it - * with `variables` consumed. `variables` being undefined means this is a no-op. - * - * Dropping `variables` from the result ensures a config can't be substituted - * twice, the same way `substituteChartConfigVariables` does. - */ -export function substitutePromqlChartConfigVariables< - T extends { - promqlExpression: PromqlExpressionList; - variables?: ChartVariable[]; - }, ->(config: T): T { - const { promqlExpression, variables } = config; - if (variables == null) return config; - - const substitute = (expression: string) => - substituteVariables(expression, { variables, inputLanguage: 'promql' }); - - return { - ...config, - promqlExpression: - typeof promqlExpression === 'string' - ? substitute(promqlExpression) - : promqlExpression.map(series => ({ - ...series, - expression: substitute(series.expression), - })), - variables: undefined, - }; -} - /** * Every variable reference across a chart builder config's expressions. * Never throws: it runs over saved configs that may be mid-edit or malformed.