diff --git a/.changeset/webhook-group-by-vars.md b/.changeset/webhook-group-by-vars.md new file mode 100644 index 0000000000..c6429a60c2 --- /dev/null +++ b/.changeset/webhook-group-by-vars.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +feat: expose group-by values to webhook body templates as `{{var.}}`, and keep them on resolve notifications diff --git a/docs/alert-webhook-template-variables.md b/docs/alert-webhook-template-variables.md index 7443a8ea57..2a78f461a6 100644 --- a/docs/alert-webhook-template-variables.md +++ b/docs/alert-webhook-template-variables.md @@ -21,6 +21,7 @@ a HyperDX alert without parsing the human-readable message body. | `{{thresholdMax}}` | number | Upper bound of a `between`/`outside` condition. Always unset for every other comparator, even if the alert was once a range, so guard it (see below). | | `{{value}}` | number | The value that triggered or resolved the alert. | | `{{groupKey}}` | string | The breaching group, for a grouped alert. | +| `{{var}}` | object | The breaching group's values by group-by column or SQL alias. Use one value at a time, e.g. `{{var.customer_id}}`, or `{{var.[service.name]}}` for a name with dots. Empty when not grouped, and on the resolve of a group that has no rows left. | | `{{sourceQuery}}` | string | The condition behind the alert — search expression or SQL (see below). | | `{{teamId}}` | string | Team the alert belongs to. | | `{{note}}` | string | The alert's freeform note — commonly a runbook link. | diff --git a/packages/api/src/routers/api/__tests__/webhooks.int.test.ts b/packages/api/src/routers/api/__tests__/webhooks.int.test.ts index 55520e121f..1bd86f6bfc 100644 --- a/packages/api/src/routers/api/__tests__/webhooks.int.test.ts +++ b/packages/api/src/routers/api/__tests__/webhooks.int.test.ts @@ -1383,6 +1383,7 @@ describe('webhooks router', () => { thresholdMax: 10, value: 7, groupKey: 'test-group', + groupValues: { group: 'test-group' }, sourceQuery: 'SeverityText: "error"', note: 'Test webhook — no runbook', teamId: expect.any(String), diff --git a/packages/api/src/routers/api/webhooks.ts b/packages/api/src/routers/api/webhooks.ts index db9bdb80b7..4973dff1d8 100644 --- a/packages/api/src/routers/api/webhooks.ts +++ b/packages/api/src/routers/api/webhooks.ts @@ -491,6 +491,7 @@ router.post( thresholdMax: 10, value: 7, groupKey: 'test-group', + groupValues: { group: 'test-group' }, sourceQuery: 'SeverityText: "error"', teamId: teamId.toString(), note: 'Test webhook — no runbook', diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 9b9953ffad..8ef7412f5c 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -7223,6 +7223,68 @@ describe('checkAlerts', () => { expect(resolutionCall).toBeDefined(); }); + it('Group-by alerts keep the group values on the resolve notification', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + name: 'Errors for {{attributes.ServiceName}}', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + const error = (Timestamp: Date) => ({ + ServiceName: 'service-b', + Timestamp, + SeverityText: 'error', + Body: 'Error from service-b', + }); + const firstWindow = new Date('2023-11-16T22:05:00.000Z'); + // One error in the next window: still present in the data, but below threshold. + await bulkInsertLogs([ + error(firstWindow), + error(firstWindow), + error(new Date('2023-11-16T22:10:00.000Z')), + ]); + + for (const at of [ + '2023-11-16T22:12:00.000Z', + '2023-11-16T22:16:00.000Z', + ]) { + await processAlertAtTime( + new Date(at), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + } + + const texts = jest + .mocked(slack.postMessageToWebhook) + .mock.calls.map(call => call[1].text); + expect(texts).toHaveLength(2); + expect(texts[1]).toBe('✅ Errors for service-b'); + }); + it('TILE alert (metrics) - slack webhook', async () => { const { team, webhook, connection, teamWebhooksById, clickhouseClient } = await setupSavedSearchAlertTest(); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 1b481f789d..17a6b66c35 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -562,6 +562,7 @@ const fireChannelEvent = async ({ teamId, teamWebhooksById, sampleLines, + groupValues: attributes, }); return { failures, timings, dispatchDurationMs }; }; @@ -1286,6 +1287,8 @@ export const processAlert = async ( string, { value: number; attributes: Record; startTime: Date } >(); + // ponytail: a group that vanished from the data resolves without its values. + const groupAttributes = new Map>(); // Helper to get or create history for a group const getOrCreateHistory = (groupKey: string): IAlertHistory => { @@ -1471,6 +1474,7 @@ export const processAlert = async ( group: groupKey, totalCount: lastValue?.count || 0, startTime: lastValue?.startTime || nowInMinsRoundDown, + attributes: groupAttributes.get(groupKey), }); } }; @@ -1635,6 +1639,7 @@ export const processAlert = async ( ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') : ''; const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; + groupAttributes.set(groupKey, attributes); const exceeds = doesExceedThreshold(alert, value); diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index dad60cdeb4..9b31304d4d 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -617,6 +617,7 @@ export const renderAlertTemplate = async ({ teamWebhooksById, dispatcher = inlineNotificationDispatcher, sampleLines, + groupValues, }: { alertProvider: AlertProvider; clickhouseClient: ClickhouseClient; @@ -634,6 +635,8 @@ export const renderAlertTemplate = async ({ * per group; without one they are fetched here. */ sampleLines?: () => Promise; + // Flat, unlike view.attributes, so a dotted column name stays one key. + groupValues?: Record; }): Promise => { // Internal mutable view with __hdx_query_results__ populated on the // saved-search path. Untrusted values must flow through the view so @@ -752,6 +755,7 @@ export const renderAlertTemplate = async ({ : undefined, value, groupKey: group ?? '', + groupValues: groupValues ?? {}, sourceQuery: truncateString( describeSourceQuery(alert, savedSearch, dashboard), MAX_SOURCE_QUERY_LENGTH, diff --git a/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts b/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts index 1fd36cee1f..1f99435526 100644 --- a/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts +++ b/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts @@ -129,6 +129,21 @@ describe('buildWebhookTemplateVariables', () => { expect(vars.sourceQuery).toBe('Body: \\"error\\"'); }); + it('exposes group-by values as JSON-escaped var. entries', () => { + const vars = buildWebhookTemplateVariables({ + ...message, + groupValues: { customer_id: 'acme "eu"' }, + }); + const rendered = createHandlebarsWithHelpers().compile( + '{"summary": "{{var.customer_id}} has an elevated error rate"}', + { noEscape: true }, + )(vars); + expect(JSON.parse(rendered).summary).toBe( + 'acme "eu" has an elevated error rate', + ); + expect(buildWebhookTemplateVariables(message).var).toEqual({}); + }); + it('renders enriched fields as empty strings (never "undefined") when absent', () => { const vars = buildWebhookTemplateVariables(message); expect(vars.alertId).toBe(''); diff --git a/packages/api/src/tasks/checkAlerts/transports/generic.ts b/packages/api/src/tasks/checkAlerts/transports/generic.ts index 5a1d23cdda..2dd60b983c 100644 --- a/packages/api/src/tasks/checkAlerts/transports/generic.ts +++ b/packages/api/src/tasks/checkAlerts/transports/generic.ts @@ -5,6 +5,7 @@ import { WebhookTemplateVariable, } from '@hyperdx/common-utils/dist/types'; import Handlebars from 'handlebars'; +import _ from 'lodash'; import { performance } from 'perf_hooks'; import { serializeError } from 'serialize-error'; @@ -96,6 +97,7 @@ export const buildWebhookTemplateVariables = (message: Message) => alertType: escapeJsonString(message.alertType ?? ''), comparator: escapeJsonString(message.comparator ?? ''), groupKey: escapeJsonString(message.groupKey ?? ''), + var: _.mapValues(message.groupValues ?? {}, v => escapeJsonString(v)), note: escapeJsonString(message.note ?? ''), sourceQuery: escapeJsonString(message.sourceQuery ?? ''), status: escapeJsonString(message.status ?? ''), @@ -103,7 +105,10 @@ export const buildWebhookTemplateVariables = (message: Message) => threshold: message.threshold, thresholdMax: message.thresholdMax, value: message.value, - }) satisfies Record; + }) satisfies Record< + WebhookTemplateVariable, + string | number | undefined | Record + >; /** * Creates a Handlebars instance with common helpers registered. diff --git a/packages/api/src/tasks/checkAlerts/transports/types.ts b/packages/api/src/tasks/checkAlerts/transports/types.ts index 5c12bd7739..7ed52698cd 100644 --- a/packages/api/src/tasks/checkAlerts/transports/types.ts +++ b/packages/api/src/tasks/checkAlerts/transports/types.ts @@ -23,6 +23,8 @@ export interface Message { thresholdMax?: number; value?: number; // the value that triggered/resolved the alert groupKey?: string; + // The group's values keyed by group-by column, exposed as `{{var.}}`. + groupValues?: Record; sourceQuery?: string; // the search expr / SQL that defines the alert teamId?: string; note?: string; // freeform alert note (markdown); commonly holds a runbook link diff --git a/packages/app/src/components/TeamSettings/WebhookForm.tsx b/packages/app/src/components/TeamSettings/WebhookForm.tsx index 727083a051..4aac573677 100644 --- a/packages/app/src/components/TeamSettings/WebhookForm.tsx +++ b/packages/app/src/components/TeamSettings/WebhookForm.tsx @@ -58,6 +58,7 @@ const describeTemplateVariable = ( thresholdMax: 'Upper bound of a between/outside range (number)', value: 'Value that triggered or resolved the alert (number)', groupKey: 'The breaching group, if grouped', + var: 'Group-by values by column, e.g. {{var.customer_id}}', sourceQuery: 'Search expression or SQL behind the alert', startTime: 'Window start, Unix ms (number)', endTime: 'Window end, Unix ms (number)', diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 5685542c80..4c71dbdd34 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -648,6 +648,7 @@ export const WEBHOOK_TEMPLATE_VARIABLES = [ 'thresholdMax', 'value', 'groupKey', + 'var', 'sourceQuery', 'teamId', 'note',