diff --git a/.changeset/markdown-tile-variables.md b/.changeset/markdown-tile-variables.md new file mode 100644 index 0000000000..1787c97f30 --- /dev/null +++ b/.changeset/markdown-tile-variables.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +feat: Substitute dashboard variables in markdown tiles diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index bc5dd16348..9532585409 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -790,7 +790,7 @@ describe('getTileVariableWarnings', () => { expect(warnings.join('\n')).toContain('(none)'); }); - it('ignores markdown tiles, which have no expressions to check', () => { + it('warns about an unknown variable in a markdown tile', () => { const markdownTile: ExternalDashboardTileWithId = { id: 'md', x: 0, @@ -798,11 +798,11 @@ describe('getTileVariableWarnings', () => { w: 12, h: 3, name: 'Notes', - config: { displayType: 'markdown', markdown: 'Pick a $service' }, + config: { displayType: 'markdown', markdown: '# $service on $env' }, }; - expect(getTileVariableWarnings([markdownTile], [variableFilter])).toEqual( - [], - ); + expect(getTileVariableWarnings([markdownTile], [variableFilter])).toEqual([ + 'Tile "Notes": The markdown references unknown variable $env. Available variables: service.', + ]); }); }); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 6f335870e1..ea8d2248c3 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -991,7 +991,7 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros: table 1 to 20 select items. Optional groupBy defines row groups. Per-series numberFormat lets one column render as a duration while a sibling count column stays a plain number. heatmap Exactly 1 select item with a non-empty valueExpression. No aggFn or alias on the select item (the chart-level displayType: "heatmap" is the discriminator). Trace sources only (no Log/Metric/Session). No groupBy. Optional where filter applied before bucketing. search No select items (select is a column list string). where is the filter. - markdown No select items. Set markdown field with content. + markdown No select items. Set markdown field with content, which may reference dashboard variables (see DASHBOARD VARIABLES). == METRIC SOURCES == @@ -1221,6 +1221,7 @@ Only add one when the default is wrong. \${service:sqlstring} is redundant in a lucene ("a" OR "b") ("") the default in Lucene inputs; ("") is a match-all, so no guard is needed regex (a|b) .* use with match() csv a,b use INSIDE a string literal + markdown a, b the default in markdown tiles; markdown syntax in values is escaped BUILDER TILES @@ -1234,6 +1235,11 @@ In a LUCENE input the macros have no meaning and are matched as literal text. Re select: [{ aggFn: "count", whereLanguage: "lucene", where: "ServiceName:$service" }] When in doubt on a variable-driven tile, set whereLanguage: "sql" and use $__filter. +MARKDOWN TILES + +The markdown field substitutes $variableName / \${variableName} / \${variableName:format} with the selected values as plain text, so a note can name what the dashboard is scoped to. Nothing is selected on a freshly-opened dashboard, so the reference renders as nothing until the user picks a value; write the surrounding text so it still reads. The macros have no meaning here and are left as written. + markdown: "Latency for the selected services: $service" + RAW SQL TILES (for advanced use-cases only) SELECT $__timeInterval(Timestamp) AS ts, count() AS errors diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 7fc6f6bbde..7d2eb818e7 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -18,11 +18,11 @@ import { DisplayType, QueryExpressionDashboardFilter, SavedChartConfig, - SearchConditionLanguage, } from '@hyperdx/common-utils/dist/types'; import { getVariableReferences, mapBuilderVariableTemplates, + TemplateLanguage, validateVariableReferencesInTemplate, } from '@hyperdx/common-utils/dist/variables'; @@ -244,6 +244,19 @@ export function getFilterVariableWarnings( return warnings; } +function variableIssueSubject(language: TemplateLanguage): string { + switch (language) { + case 'sql': + return 'SQL'; + case 'lucene': + return 'The Lucene filter'; + case 'promql': + return 'The PromQL expression'; + case 'markdown': + return 'The markdown'; + } +} + /** * Non-blocking checks on the dashboard variables a tile's expressions * reference, run against the variables the dashboard's filters actually @@ -273,9 +286,9 @@ export function getTileVariableWarnings( } const issues: string[] = []; - const validate = (template: string, language: SearchConditionLanguage) => { + const validate = (template: string, language: TemplateLanguage) => { const result = validateVariableReferencesInTemplate(template, variables, { - subject: language === 'lucene' ? 'The Lucene filter' : 'SQL', + subject: variableIssueSubject(language), language, }); issues.push(...result.errors, ...result.warnings); @@ -285,8 +298,10 @@ export function getTileVariableWarnings( validate(config.sqlTemplate, 'sql'); } else if ( isBuilderSavedChartConfig(config) && - config.displayType !== DisplayType.Markdown + config.displayType === DisplayType.Markdown ) { + if (config.markdown) validate(config.markdown, 'markdown'); + } else if (isBuilderSavedChartConfig(config)) { mapBuilderVariableTemplates(config, (template, language) => { validate(template, language); return template; diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 81441a79f5..7120338d20 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -596,6 +596,21 @@ const Tile = ({ [serializedTileVariables], ); + // Serialized for the same reason as `tileVariables`. Markdown tiles flag + // references to names outside this list, which `tileVariables` filters out. + const serializedVariableNames = useMemo( + () => + variables + ? JSON.stringify(variables.map(variable => variable.name)) + : undefined, + [variables], + ); + const variableNames = useMemo( + () => + serializedVariableNames ? JSON.parse(serializedVariableNames) : undefined, + [serializedVariableNames], + ); + // Serialized for the same reason as `tileVariables`: any change to the // dashboard's filters hands this tile a new array, and only a change to the // names this tile is blocked on should churn the render memo below. @@ -1377,6 +1392,8 @@ const Tile = ({ title={title} toolbarItems={toolbar} config={effectiveMarkdownConfig} + variables={tileVariables} + availableVariableNames={variableNames} /> )} {effectiveQueriedConfig?.displayType === DisplayType.Search && @@ -1495,6 +1512,8 @@ const Tile = ({ annotations, isLive, readOnly, + tileVariables, + variableNames, ], ); diff --git a/packages/app/src/HDXMarkdownChart.tsx b/packages/app/src/HDXMarkdownChart.tsx index 971d6f50b5..f4593f6fca 100644 --- a/packages/app/src/HDXMarkdownChart.tsx +++ b/packages/app/src/HDXMarkdownChart.tsx @@ -1,28 +1,82 @@ -import { memo } from 'react'; +import { memo, useMemo } from 'react'; import ReactMarkdown from 'react-markdown'; +import { ChartVariable } from '@hyperdx/common-utils/dist/types'; +import { + substituteVariables, + validateVariableReferencesInTemplate, +} from '@hyperdx/common-utils/dist/variables'; import ChartContainer from './components/charts/ChartContainer'; +import { + hasVariableIssues, + VariableIssueIndicator, +} from './components/SQLEditor/variableValidation'; const HDXMarkdownChart = memo( ({ config: { markdown }, title, toolbarItems, + variables: referencedVariables, + availableVariableNames, }: { title?: React.ReactNode; toolbarItems?: React.ReactNode[]; config: { markdown?: string; }; + /** The variables to substitute. Limited to the ones this chart references. */ + variables?: ChartVariable[]; + /** Every variable name in scope, for flagging unknown references. */ + availableVariableNames?: string[]; }) => { + const content = useMemo(() => { + if (!markdown || referencedVariables == null) return markdown ?? ''; + try { + return substituteVariables(markdown, { + variables: referencedVariables, + inputLanguage: 'markdown', + }); + } catch { + // An unknown `${name:format}` throws; show the markdown as written. + return markdown; + } + }, [markdown, referencedVariables]); + + const toolbar = useMemo(() => { + const names = + availableVariableNames ?? + referencedVariables?.map(variable => variable.name); + + // Without variables in scope nothing is substituted, so a `$word` is prose. + if (!markdown || !names?.length) return toolbarItems; + + // Only names are checked for markdown, so the selections don't matter. + const allVariables = names.map(name => ({ name, values: [] })); + const issues = validateVariableReferencesInTemplate( + markdown, + allVariables, + { + subject: 'Markdown', + language: 'markdown', + }, + ); + if (!hasVariableIssues(issues)) return toolbarItems; + + return [ + , + ...(toolbarItems ?? []), + ]; + }, [markdown, availableVariableNames, referencedVariables, toolbarItems]); + return (
- {markdown ?? ''} + {content}
); diff --git a/packages/app/src/__tests__/HDXMarkdownChart.test.tsx b/packages/app/src/__tests__/HDXMarkdownChart.test.tsx new file mode 100644 index 0000000000..fe38c34f1d --- /dev/null +++ b/packages/app/src/__tests__/HDXMarkdownChart.test.tsx @@ -0,0 +1,73 @@ +import React from 'react'; +import { screen } from '@testing-library/react'; + +import HDXMarkdownChart from '@/HDXMarkdownChart'; + +const service = { name: 'service', values: ['api', 'web'] }; + +describe('HDXMarkdownChart', () => { + it('substitutes variable references', () => { + renderWithMantine( + , + ); + expect(screen.getByText('# Services: api, web')).toBeInTheDocument(); + }); + + it('renders the markdown as written without variables', () => { + renderWithMantine( + , + ); + expect(screen.getByText('# Services: $service')).toBeInTheDocument(); + }); + + it('renders the markdown as written when a format is unknown', () => { + renderWithMantine( + , + ); + expect(screen.getByText('${service:nope}')).toBeInTheDocument(); + }); + + it('flags a reference to an unknown variable', () => { + renderWithMantine( + , + ); + expect(screen.getByTestId('variable-validation')).toHaveAccessibleName( + 'Markdown references unknown variable $nope. Available variables: service.', + ); + }); + + it('checks references against the available names when given', () => { + renderWithMantine( + , + ); + expect(screen.queryByTestId('variable-validation')).not.toBeInTheDocument(); + }); + + it('flags nothing when every reference is known', () => { + renderWithMantine( + , + ); + expect(screen.queryByTestId('variable-validation')).not.toBeInTheDocument(); + }); + + it('flags nothing without variables', () => { + renderWithMantine(); + expect(screen.queryByTestId('variable-validation')).not.toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 46cad955c2..fdb798e7e5 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -1090,6 +1090,7 @@ export default function EditTimeChartForm({ config={{ markdown: markdown || 'Preview', }} + variables={variables} /> diff --git a/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts b/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts index 68846b31e6..21956b49fa 100644 --- a/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts +++ b/packages/app/src/components/SQLEditor/__tests__/promqlVariableCompletions.test.ts @@ -65,11 +65,12 @@ describe('buildPromqlVariableCompletions', () => { expect(labels([SERVICE])).not.toContain('$__filter($service)'); }); - it('withholds the formats that emit SQL or Lucene syntax', () => { + it('withholds the formats that emit SQL, Lucene, or markdown syntax', () => { // Still honoured if hand-typed — just never suggested, because neither // `'api', 'web'` nor `("api" OR "web")` parses inside a PromQL expression. expect(labels([SERVICE])).not.toContain('${service:sqlstring}'); expect(labels([SERVICE])).not.toContain('${service:lucene}'); + expect(labels([SERVICE])).not.toContain('${service:markdown}'); }); it('offers the same set of forms for every variable', () => { diff --git a/packages/app/src/components/SQLEditor/__tests__/variableCompletions.test.ts b/packages/app/src/components/SQLEditor/__tests__/variableCompletions.test.ts index 735c36e783..eac445c96a 100644 --- a/packages/app/src/components/SQLEditor/__tests__/variableCompletions.test.ts +++ b/packages/app/src/components/SQLEditor/__tests__/variableCompletions.test.ts @@ -53,6 +53,10 @@ describe('buildSqlVariableCompletions', () => { ); }); + it('withholds the markdown format, which only applies to markdown tiles', () => { + expect(labels([SERVICE])).not.toContain('${service:markdown}'); + }); + it('withholds the macros that a chart builder input never expands', () => { // Only the variable macros are substituted into builder expressions, so // offering $__timeFilter and friends here would suggest SQL that reaches diff --git a/packages/app/src/components/SQLEditor/variableCompletions.tsx b/packages/app/src/components/SQLEditor/variableCompletions.tsx index 22c53246f4..a5b1ce1e21 100644 --- a/packages/app/src/components/SQLEditor/variableCompletions.tsx +++ b/packages/app/src/components/SQLEditor/variableCompletions.tsx @@ -7,21 +7,49 @@ import { ChartVariable } from '@hyperdx/common-utils/dist/types'; import { substituteVariables, TemplateLanguage, - VARIABLE_FORMATS, VariableFormat, } from '@hyperdx/common-utils/dist/variables'; import { type SQLCompletion } from './utils'; -/** What each `${name:format}` renders, for the completion's help text. */ -const VARIABLE_FORMAT_DESCRIPTIONS: Record = { - sqlstring: "Quoted and comma-separated, escaped for SQL. e.g. 'a', 'b', 'c'", - csv: 'Comma-separated and unquoted. Not SQL-escaped. e.g. a,b,c', - regex: 'A regex alternation. Regex escaped. e.g. (a|b|c)', - lucene: - 'An OR of quoted terms, for Lucene inputs. e.g. ("a" OR "b" OR "c"). Quote the reference (field:"$var") for exact-match behavior. Leave unquoted (field:$var) for substring matching.', +/** + * The `${name:format}` forms each language's completions offer, in display + * order, with their help text. + */ +const COMPLETION_FORMATS: Record< + TemplateLanguage, + Partial> +> = { + sql: { + sqlstring: + "Quoted and comma-separated, escaped for SQL. e.g. 'a', 'b', 'c'", + csv: 'Comma-separated and unquoted. Not SQL-escaped. e.g. a,b,c', + regex: 'A regex alternation. Regex escaped. e.g. (a|b|c)', + lucene: + 'An OR of quoted terms, for Lucene inputs. e.g. ("a" OR "b" OR "c"). Quote the reference (field:"$var") for exact-match behavior. Leave unquoted (field:$var) for substring matching.', + }, + promql: { + regex: + 'The default format, written out. A regex alternation, escaped for the string literal it sits in. e.g. (a|b|c)', + csv: 'Comma-separated and unquoted, with no escaping.', + }, + // Lucene inputs suggest only the bare `$name` reference. + lucene: {}, + // Markdown has no auto-complete + markdown: {}, }; +/** A completion for each of `formats`, for the given variable. */ +function formatCompletions( + name: string, + formats: Partial>, + buildCompletion: (label: string, description: string) => SQLCompletion, +): SQLCompletion[] { + return Object.entries(formats).map(([format, description]) => + buildCompletion(`\${${name}:${format}}`, description), + ); +} + /** What `snippet` expands to against the variable's current selection. */ function describeVariableExpansion( snippet: string, @@ -128,12 +156,7 @@ function getSqlVariableCompletions(variable: ChartVariable): SQLCompletion[] { `\${${name}}`, `The same as $${name}, but delimited — use it when the reference runs into following word characters, as in \${${name}}_total.`, ), - ...VARIABLE_FORMATS.map(format => - buildCompletion( - `\${${name}:${format}}`, - VARIABLE_FORMAT_DESCRIPTIONS[format], - ), - ), + ...formatCompletions(name, COMPLETION_FORMATS.sql, buildCompletion), ]; } @@ -168,14 +191,7 @@ function getPromqlVariableCompletions( `\${${name}}`, `The same as $${name}, but delimited — use it when the reference runs into following word characters.`, ), - reference( - `\${${name}:regex}`, - 'The default format, written out. A regex alternation, escaped for the string literal it sits in. e.g. (a|b|c)', - ), - reference( - `\${${name}:csv}`, - 'Comma-separated and unquoted, with no escaping. Use it to interpolate something that is not a matcher value, such as a metric or label name.', - ), + ...formatCompletions(name, COMPLETION_FORMATS.promql, reference), ]; } diff --git a/packages/app/tests/e2e/features/dashboard-static-filters.spec.ts b/packages/app/tests/e2e/features/dashboard-static-filters.spec.ts index fec1e3e1ea..9f14b47262 100644 --- a/packages/app/tests/e2e/features/dashboard-static-filters.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-static-filters.spec.ts @@ -197,6 +197,55 @@ test.describe( }); }); + test('substitutes the selected values into a markdown tile', async ({ + page, + }) => { + test.setTimeout(90000); + const chartName = `E2E Markdown Variables ${Date.now()}`; + + await test.step('Create a static Service filter and select one value', async () => { + await dashboardPage.createNewDashboard(); + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addStaticListFilterToDashboard( + 'Service', + ['frontend', 'accounting', 'ad'], + { variableName: 'svc' }, + ); + await dashboardPage.closeFiltersModal(); + await dashboardPage.toggleFilterValue('Service', 'accounting'); + }); + + await test.step('The editor preview shows the selected value', async () => { + await dashboardPage.addTile(); + await expect(dashboardPage.chartEditor.nameInput).toBeVisible(); + await dashboardPage.chartEditor.setChartType(DisplayType.Markdown); + await dashboardPage.chartEditor.setChartName(chartName); + const editor = page.getByTestId('tile-editor-form'); + const markdownInput = editor.locator('textarea[name="markdown"]'); + await markdownInput.fill('# Services: $nope'); + await expect(editor.getByTestId('variable-validation')).toBeVisible(); + + await markdownInput.fill('# Services: $svc'); + await expect(editor.getByTestId('variable-validation')).toBeHidden(); + await expect( + editor.getByRole('heading', { name: 'Services: accounting' }), + ).toBeVisible(); + await dashboardPage.saveTile(); + }); + + await test.step('The tile follows the selection', async () => { + const tile = dashboardPage.getTiles().filter({ hasText: chartName }); + await expect( + tile.getByRole('heading', { name: 'Services: accounting' }), + ).toBeVisible(); + + await dashboardPage.toggleFilterValue('Service', 'frontend'); + await expect( + tile.getByRole('heading', { name: 'Services: accounting, frontend' }), + ).toBeVisible(); + }); + }); + test('round-trips the options and variable name through an edit', async ({ page, }) => { diff --git a/packages/common-utils/src/__tests__/variables.test.ts b/packages/common-utils/src/__tests__/variables.test.ts index eba27788df..7a767d2be1 100644 --- a/packages/common-utils/src/__tests__/variables.test.ts +++ b/packages/common-utils/src/__tests__/variables.test.ts @@ -3,7 +3,7 @@ import { MalformedMacroArgsError, UnknownVariableError, } from '@/macroErrors'; -import type { BuilderChartConfig, ChartVariable } from '@/types'; +import { BuilderChartConfig, ChartVariable, DisplayType } from '@/types'; import { filterReferencedVariables, formatVariableValues, @@ -107,6 +107,31 @@ describe('formatVariableValues', () => { }); }); + describe('markdown', () => { + it('renders empty when nothing is selected', () => { + expect(formatVariableValues([], 'markdown')).toBe(''); + }); + + it('joins values with a comma and a space', () => { + expect(formatVariableValues(['api', 'web'], 'markdown')).toBe('api, web'); + }); + + it('escapes markdown syntax in values', () => { + expect( + formatVariableValues( + ['[x](https://e.com)', '*a_b* \\ `c`'], + 'markdown', + ), + ).toBe('\\[x\\]\\(https\\:\\/\\/e\\.com\\), \\*a\\_b\\* \\\\ \\`c\\`'); + }); + + it('replaces newlines with spaces', () => { + expect(formatVariableValues(['a\n# b\r\nc'], 'markdown')).toBe( + 'a \\# b c', + ); + }); + }); + describe('lucene', () => { it('renders an empty term when nothing is selected', () => { // Parenthesized so it stays a no-op in a field-scoped position; see the @@ -804,6 +829,63 @@ describe('substituteVariables per language', () => { }); }); +describe('substituteVariables for markdown', () => { + const markdown = (input: string, variables: ChartVariable[]) => + substituteVariables(input, { variables, inputLanguage: 'markdown' }); + + it('joins the selected values with a comma and a space by default', () => { + expect(markdown('Services: $service', [SERVICE])).toBe( + 'Services: api, web', + ); + }); + + it('renders nothing when no values are selected', () => { + expect(markdown('Services: ${service}', [EMPTY_SERVICE])).toBe( + 'Services: ', + ); + }); + + it('substitutes in headings and lines SQL would treat as comments', () => { + expect(markdown('# $service\n-- $service /* $service */', [SERVICE])).toBe( + '# api, web\n-- api, web /* api, web */', + ); + }); + + it('substitutes after an apostrophe', () => { + expect(markdown("Today's services: $service", [SERVICE])).toBe( + "Today's services: api, web", + ); + }); + + it('substitutes inside link URLs', () => { + expect(markdown('[logs](/search?where=${service:csv})', [SERVICE])).toBe( + '[logs](/search?where=api,web)', + ); + }); + + it('honors an explicit format', () => { + expect(markdown('${service:regex}', [SERVICE])).toBe('(api|web)'); + }); + + it('escapes markdown syntax in values by default', () => { + expect(markdown('$service', [variable('service', ['*prod*'])])).toBe( + '\\*prod\\*', + ); + }); + + it('leaves values unescaped with the csv format', () => { + expect( + markdown('`${service:csv}`', [variable('service', ['*prod*'])]), + ).toBe('`*prod*`'); + }); + + it('leaves unknown references and variable macros as written', () => { + expect(markdown('$nope $__filter(ServiceName, $service)', [SERVICE])).toBe( + '$nope $__filter(ServiceName, $service)', + ); + }); +}); + describe('substituteVariables for promql', () => { const promql = (input: string, variables: ChartVariable[]) => substituteVariables(input, { variables, inputLanguage: 'promql' }); @@ -1248,6 +1330,15 @@ describe('getReferencedVariableNames', () => { 'env', ]); }); + + it('skips SQL comments for SQL but not for markdown', () => { + const template = '# $service\n$env'; + expect(getReferencedVariableNames(template)).toEqual(['env']); + expect(getReferencedVariableNames(template, 'markdown')).toEqual([ + 'service', + 'env', + ]); + }); }); describe('filterReferencedVariables', () => { @@ -1332,6 +1423,30 @@ describe('filterReferencedVariables', () => { ).toEqual(variables); }); + it('keeps the variables a markdown tile references', () => { + expect( + filterReferencedVariables( + builderConfig({ + displayType: DisplayType.Markdown, + markdown: '# Service $service', + }), + variables, + ), + ).toEqual([SERVICE]); + }); + + it('ignores leftover markdown on a tile that is not markdown', () => { + expect( + filterReferencedVariables( + builderConfig({ + displayType: DisplayType.Line, + markdown: '$service', + }), + variables, + ), + ).toEqual([]); + }); + it('returns an empty array when a builder config references none of them', () => { expect( filterReferencedVariables( @@ -1940,6 +2055,42 @@ describe('validateVariableReferencesInTemplate', () => { }); }); }); + + describe('for markdown', () => { + const markdown = (template: string, variables: ChartVariable[]) => + validate(template, variables, { + subject: 'Markdown', + language: 'markdown', + }); + + it('says nothing when every reference is known', () => { + expect(markdown("# $service\nToday's ${service:csv}", [SERVICE])).toEqual( + { errors: [], warnings: [] }, + ); + }); + + it('checks references in headings', () => { + expect(markdown('# $srvice', [SERVICE]).warnings).toEqual([ + 'Markdown references unknown variable $srvice. Available variables: service.', + ]); + }); + + it('warns about an unknown format', () => { + expect(markdown('${service:nope}', [SERVICE]).warnings).toEqual([ + '${service:nope} uses an unknown format, so no variables are substituted. ' + + 'Expected one of: sqlstring, regex, csv, lucene, markdown.', + ]); + }); + + it('warns that a macro is left as written', () => { + expect( + markdown('$__filter(ServiceName, $service)', [SERVICE]).warnings, + ).toEqual([ + '$__filter has no meaning in markdown — it is left as written. ' + + 'Reference the variable directly, as in $service.', + ]); + }); + }); }); describe('macros naming an unknown variable', () => { diff --git a/packages/common-utils/src/variables.ts b/packages/common-utils/src/variables.ts index ada287378d..f90c1e397c 100644 --- a/packages/common-utils/src/variables.ts +++ b/packages/common-utils/src/variables.ts @@ -21,6 +21,7 @@ import { ChartVariable, DASHBOARD_VARIABLE_NAME_PATTERN, DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, + DisplayType, PromqlExpressionList, SavedChartConfig, SearchConditionLanguage, @@ -34,6 +35,7 @@ export const VARIABLE_FORMATS = [ 'regex', 'csv', 'lucene', + 'markdown', ] as const; export type VariableFormat = (typeof VARIABLE_FORMATS)[number]; @@ -51,6 +53,15 @@ const escapeLuceneValue = (value: string) => const escapePromqlStringValue = (value: string) => value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +/** + * Backslash-escape every ASCII punctuation character (CommonMark renders each + * escape as the literal character), so a value renders as plain text rather + * than as markdown syntax. Newlines become spaces so a value can't split the + * surrounding block. + */ +const escapeMarkdownValue = (value: string) => + value.replace(/\r\n|\r|\n/g, ' ').replace(/[!-/:-@[-`{-~]/g, '\\$&'); + /** * Render a variable's selected values in the requested format. Every format * has an "empty selection" rendering that keeps the surrounding query valid. @@ -75,6 +86,8 @@ export function formatVariableValues( return values.length === 0 ? '("")' : `(${values.map(value => `"${escapeLuceneValue(value)}"`).join(' OR ')})`; + case 'markdown': + return values.map(escapeMarkdownValue).join(', '); default: format satisfies never; // Unreachable throw new Error(`Unknown variable format '${format}'`); @@ -197,7 +210,14 @@ export function findBalancedParens(input: string, start: number): number { export function scanTemplateTokens( input: string, macroNames: readonly string[], - { onMalformed = 'throw' }: { onMalformed?: 'throw' | 'skip' } = {}, + { + onMalformed = 'throw', + skipSqlComments = true, + }: { + onMalformed?: 'throw' | 'skip'; + /** Whether SQL comments are skipped over rather than scanned for references. */ + skipSqlComments?: boolean; + } = {}, ): TemplateToken[] { // Longest name first so `$__filters` isn't matched as `$__filter` + `s`, etc. const sortedMacroNames = [...macroNames].sort((a, b) => b.length - a.length); @@ -219,7 +239,7 @@ export function scanTemplateTokens( let i = 0; while (i < input.length) { // Consume any comments starting at this position - if (!inSingleQuote && !inDoubleQuote) { + if (skipSqlComments && !inSingleQuote && !inDoubleQuote) { const commentEnd = findCommentEnd(input, i); if (commentEnd > i) { text += input.slice(i, commentEnd); @@ -405,7 +425,9 @@ export function expandTemplate( // -- Variable expansion ----------------------------------------------------- -export type TemplateLanguage = NonNullable; +export type TemplateLanguage = + | NonNullable + | 'markdown'; export type VariableContext = { variables: ChartVariable[]; @@ -427,6 +449,11 @@ type LanguageSettings = { * themselves, and `csv` is the raw escape hatch that carries identifiers. */ escapeRegexForLiteral?: (rendered: string) => string; + /** + * Whether SQL comments (`--`, `#`, block comments) are skipped over rather + * than scanned for references. + */ + skipSqlComments?: boolean; }; /** Settings controlling how variables and templates are expanded for each template language. */ @@ -441,6 +468,12 @@ const LANGUAGE_SETTINGS: Record = { disableMacros: true, escapeRegexForLiteral: escapePromqlStringValue, }, + markdown: { + defaultFormat: 'markdown', + disableMacros: true, + // Markdown doesn't have SQL comments, we wouldn't want to skip # ... (a markdown header) + skipSqlComments: false, + }, }; const SETTINGS_BY_LANGUAGE = new Map(Object.entries(LANGUAGE_SETTINGS)); @@ -859,9 +892,11 @@ export function substituteVariables( ); } - if (languageSettings(ctx.inputLanguage).disableMacros) { + const settings = languageSettings(ctx.inputLanguage); + if (settings.disableMacros) { return scanTemplateTokens(input, VARIABLE_MACRO_NAMES, { onMalformed: 'skip', + skipSqlComments: settings.skipSqlComments, }) .map(token => expandVariableOnly(token, ctx)) .join(''); @@ -1061,7 +1096,10 @@ export type VariableReference = { * template written that way is reported as using the macro; expansion is what * rejects it. */ -export function getVariableReferences(input: string): VariableReference[] { +export function getVariableReferences( + input: string, + { skipSqlComments }: { skipSqlComments?: boolean } = {}, +): VariableReference[] { const references: VariableReference[] = []; /** Returns the referenced name, or undefined when the argument isn't one. */ @@ -1086,6 +1124,7 @@ export function getVariableReferences(input: string): VariableReference[] { const visit = (text: string, guardedBy?: string) => { for (const token of scanTemplateTokens(text, VARIABLE_MACRO_NAMES, { onMalformed: 'skip', + skipSqlComments, })) { if (token.kind === 'text') continue; if (token.kind !== 'macro') { @@ -1172,13 +1211,16 @@ export function validateVariableReferencesInTemplate( /** The sentence subject of each message, e.g. `SQL references ...`. */ subject?: string; /** The language the renderer parses this template as. */ - language?: SearchConditionLanguage; + language?: TemplateLanguage; } = {}, ): VariableReferenceIssues { const errors: string[] = []; const warnings: string[] = []; - const references = getVariableReferences(template); + const settings = languageSettings(language); + const references = getVariableReferences(template, { + skipSqlComments: settings.skipSqlComments, + }); // Attempt to expand macros, so that errors during expansion can be surfaced. if (variables != null && hasVariableMacro(template)) { @@ -1227,19 +1269,36 @@ export function validateVariableReferencesInTemplate( ); } - const settings = languageSettings(language); - // A macro-less language leaves a macro exactly as written, so writing one can // only ever have been a mistake. if (settings.disableMacros && macroReferences.length > 0) { const [{ name }] = macroReferences; + const macros = formatReferenceList(macroReferences); warnings.push( language === 'promql' - ? `${formatReferenceList(macroReferences)} has no meaning in a PromQL expression — it is left as written and sent to Prometheus verbatim. Reference the variable directly, as in {